From c1cc7d2cd258af4815536c76f13675b68241c0fa Mon Sep 17 00:00:00 2001 From: chenht2022 Date: Thu, 8 Aug 2024 09:04:36 +0000 Subject: [PATCH 1/8] 1) Linear and MLP operators support qlen>1; 2) All operators now share a single memory buffer; 3) Refactor CPUInfer submit/sync logic. --- .../ktransformers_ext/CMakeLists.txt | 3 +- .../ktransformers_ext/bench/bench_linear.py | 52 +-- .../bench/bench_linear_torch.py | 44 ++- .../ktransformers_ext/bench/bench_mlp.py | 52 +-- .../bench/bench_mlp_torch.py | 86 ++--- .../ktransformers_ext/bench/bench_moe.py | 64 ++-- .../bench/bench_moe_torch.py | 147 ++++---- .../ktransformers_ext/cpu_backend/cpuinfer.h | 36 +- .../ktransformers_ext/examples/test_linear.py | 65 ++-- .../ktransformers_ext/examples/test_mlp.py | 86 ++--- .../ktransformers_ext/examples/test_moe.py | 138 ++++---- .../ktransformers_ext/ext_bindings.cpp | 321 +++++++----------- .../operators/llamafile/linear.cpp | 50 ++- .../operators/llamafile/linear.h | 18 +- .../operators/llamafile/mlp.cpp | 90 +++-- .../operators/llamafile/mlp.h | 28 +- .../operators/llamafile/moe.cpp | 95 ++---- .../operators/llamafile/moe.h | 5 +- .../operators/llamafile/shared_mem_buffer.cpp | 55 +++ .../operators/llamafile/shared_mem_buffer.h | 37 ++ ktransformers/operators/experts.py | 8 +- 21 files changed, 749 insertions(+), 731 deletions(-) create mode 100644 ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp create mode 100644 ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.h diff --git a/ktransformers/ktransformers_ext/CMakeLists.txt b/ktransformers/ktransformers_ext/CMakeLists.txt index 02e6a04..c3d4f5b 100644 --- a/ktransformers/ktransformers_ext/CMakeLists.txt +++ b/ktransformers/ktransformers_ext/CMakeLists.txt @@ -22,14 +22,13 @@ option(LLAMA_AVX2 "llama: enable AVX2" option(LLAMA_AVX512 "llama: enable AVX512" OFF) option(LLAMA_AVX512_VBMI "llama: enable AVX512-VBMI" OFF) option(LLAMA_AVX512_VNNI "llama: enable AVX512-VNNI" OFF) +option(LLAMA_AVX512_BF16 "llama: enable AVX512-BF16" OFF) option(LLAMA_FMA "llama: enable FMA" OFF) # in MSVC F16C is implied with AVX2/AVX512 if (NOT MSVC) option(LLAMA_F16C "llama: enable F16C" OFF) endif() option(LLAMA_AVX512_FANCY_SIMD "llama: enable AVX512-VL, AVX512-BW, AVX512-DQ, AVX512-VNNI" OFF) -option(LLAMA_AVX512_BF16 "llama: enable AVX512-BF16" OFF) - # Architecture specific # TODO: probably these flags need to be tweaked on some architectures diff --git a/ktransformers/ktransformers_ext/bench/bench_linear.py b/ktransformers/ktransformers_ext/bench/bench_linear.py index 0a4de3a..3189afd 100644 --- a/ktransformers/ktransformers_ext/bench/bench_linear.py +++ b/ktransformers/ktransformers_ext/bench/bench_linear.py @@ -6,7 +6,7 @@ Date : 2024-07-25 10:31:59 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:32:51 +LastEditTime : 2024-08-06 10:35:35 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,15 +15,18 @@ import cpuinfer_ext import torch +input_size = 16384 +output_size = 5120 +stride = 16 +group_max_len = 1024 +layer_num = 10 +qlen = 1 +CPUInfer = cpuinfer_ext.CPUInfer(64) +warm_up_iter = 1000 +test_iter = 10000 + def bench_linear(quant_mode: str): with torch.inference_mode(mode=True): - input_size = 16384 - output_size = 5120 - stride = 16 - layer_num = 10 - CPUInfer = cpuinfer_ext.CPUInfer(64) - warm_up_iter = 1000 - test_iter = 10000 hidden_type = 30 # ggml_type::GGML_TYPE_BF16 if quant_mode == "fp32": @@ -66,30 +69,37 @@ def bench_linear(quant_mode: str): projs = [] for _ in range(layer_num): proj = torch.randn((output_size, input_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() - config = cpuinfer_ext.linear.LinearConfig(input_size, output_size, stride, proj.data_ptr(), proj_type, hidden_type) + config = cpuinfer_ext.linear.LinearConfig(input_size, output_size, stride, group_max_len, proj.data_ptr(), proj_type, hidden_type) linear = cpuinfer_ext.linear.Linear(config) projs.append(proj) linears.append(linear) + input = torch.randn((layer_num, qlen, input_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() + output = torch.empty((layer_num, qlen, output_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - linear = linears[i % layer_num] - input = torch.randn((1, input_size), dtype=torch.bfloat16).contiguous() - output = torch.empty((1, output_size), dtype=torch.bfloat16).contiguous() - CPUInfer.submit(linear.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + linears[i % layer_num].forward( + qlen, + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() # test - total_time = 0 + start = time.perf_counter() for i in range(test_iter): - linear = linears[i % layer_num] - input = torch.randn((1, input_size), dtype=torch.bfloat16).contiguous() - output = torch.empty((1, output_size), dtype=torch.bfloat16).contiguous() - start = time.perf_counter() - CPUInfer.submit(linear.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + linears[i % layer_num].forward( + qlen, + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() - end = time.perf_counter() - total_time += end - start + end = time.perf_counter() + total_time = end - start print('Quant mode: ', quant_mode) print('Time(s): ', total_time) print('Iteration: ', test_iter) diff --git a/ktransformers/ktransformers_ext/bench/bench_linear_torch.py b/ktransformers/ktransformers_ext/bench/bench_linear_torch.py index cb3e4ef..72e0e75 100644 --- a/ktransformers/ktransformers_ext/bench/bench_linear_torch.py +++ b/ktransformers/ktransformers_ext/bench/bench_linear_torch.py @@ -14,14 +14,17 @@ import torch import torch.nn.quantized as nnq +scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset + +input_size = 16384 +output_size = 5120 +layer_num = 10 +qlen = 1 +warm_up_iter = 1000 +test_iter = 10000 + def bench_linear(quant_mode: str): with torch.inference_mode(mode=True): - input_size = 16384 - output_size = 5120 - layer_num = 10 - warm_up_iter = 1000 - test_iter = 10000 - if quant_mode == "fp32": proj_type = torch.float32 bytes_per_elem = 4.000000 @@ -41,37 +44,32 @@ def bench_linear(quant_mode: str): for _ in range(layer_num): proj = torch.randn((output_size, input_size), dtype = torch.float32, device = "cuda").to("cpu").contiguous() if quant_mode == "qint8": - scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset proj_q = torch.quantize_per_tensor(proj, scale, zero_point, torch.qint8) quantized_layer = nnq.Linear(input_size, output_size) quantized_layer.set_weight_bias(proj_q, None) projs.append(quantized_layer) else: projs.append(proj.to(proj_type)) + input = torch.randn((layer_num, qlen, input_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - input = torch.randn((1, input_size), dtype=torch.float32).contiguous() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - quantized_layer = projs[i % layer_num] - t_output = quantized_layer(input_q) + if isinstance(projs[i % layer_num], nnq.Linear): + input_q = torch.quantize_per_tensor(input[i % layer_num].to(torch.float32), scale, zero_point, torch.quint8) + t_output = projs[i % layer_num](input_q) else: - t_output = torch.mm(input.to(proj_type), projs[i % layer_num].t()) + t_output = torch.mm(input[i % layer_num].to(proj_type), projs[i % layer_num].t()) # test - total_time = 0 + start = time.perf_counter() for i in range(test_iter): - input = torch.randn((1, input_size), dtype=torch.float32).contiguous() - start = time.perf_counter() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - quantized_layer = projs[i % layer_num] - t_output = quantized_layer(input_q) + if isinstance(projs[i % layer_num], nnq.Linear): + input_q = torch.quantize_per_tensor(input[i % layer_num].to(torch.float32), scale, zero_point, torch.quint8) + t_output = projs[i % layer_num](input_q) else: - t_output = torch.mm(input.to(proj_type), projs[i % layer_num].t()) - end = time.perf_counter() - total_time += end - start + t_output = torch.mm(input[i % layer_num].to(proj_type), projs[i % layer_num].t()) + end = time.perf_counter() + total_time = end - start print('Quant mode: ', quant_mode) print('Time(s): ', total_time) print('Iteration: ', test_iter) diff --git a/ktransformers/ktransformers_ext/bench/bench_mlp.py b/ktransformers/ktransformers_ext/bench/bench_mlp.py index 5680a9b..690f9d9 100644 --- a/ktransformers/ktransformers_ext/bench/bench_mlp.py +++ b/ktransformers/ktransformers_ext/bench/bench_mlp.py @@ -6,7 +6,7 @@ Date : 2024-07-16 10:43:18 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:32:55 +LastEditTime : 2024-08-06 10:36:04 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,15 +15,18 @@ import cpuinfer_ext import torch +hidden_size = 5120 +intermediate_size = 3072 +stride = 16 +group_max_len = 1024 +layer_num = 10 +qlen = 1 +CPUInfer = cpuinfer_ext.CPUInfer(64) +warm_up_iter = 1000 +test_iter = 10000 + def bench_mlp(quant_mode: str): with torch.inference_mode(mode=True): - hidden_size = 5120 - intermediate_size = 3072 - stride = 16 - layer_num = 10 - CPUInfer = cpuinfer_ext.CPUInfer(64) - warm_up_iter = 1000 - test_iter = 10000 hidden_type = 30 # ggml_type::GGML_TYPE_BF16 if quant_mode == "fp32": @@ -93,32 +96,39 @@ def bench_mlp(quant_mode: str): gate_proj = torch.randn((intermediate_size, hidden_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() up_proj = torch.randn((intermediate_size, hidden_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() down_proj = torch.randn((hidden_size, intermediate_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() - config = cpuinfer_ext.mlp.MLPConfig(hidden_size, intermediate_size, stride, gate_proj.data_ptr(), up_proj.data_ptr(), down_proj.data_ptr(), gate_type, up_type, down_type, hidden_type) + config = cpuinfer_ext.mlp.MLPConfig(hidden_size, intermediate_size, stride, group_max_len, gate_proj.data_ptr(), up_proj.data_ptr(), down_proj.data_ptr(), gate_type, up_type, down_type, hidden_type) mlp = cpuinfer_ext.mlp.MLP(config) gate_projs.append(gate_proj) up_projs.append(up_proj) down_projs.append(down_proj) mlps.append(mlp) + input = torch.randn((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() + output = torch.empty((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - mlp = mlps[i % layer_num] - input = torch.randn((1, hidden_size), dtype=torch.bfloat16).contiguous() - output = torch.empty((1, hidden_size), dtype=torch.bfloat16).contiguous() - CPUInfer.submit(mlp.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + mlps[i % layer_num].forward( + qlen, + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() # test - total_time = 0 + start = time.perf_counter() for i in range(test_iter): - mlp = mlps[i % layer_num] - input = torch.randn((1, hidden_size), dtype=torch.bfloat16).contiguous() - output = torch.empty((1, hidden_size), dtype=torch.bfloat16).contiguous() - start = time.perf_counter() - CPUInfer.submit(mlp.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + mlps[i % layer_num].forward( + qlen, + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() - end = time.perf_counter() - total_time += end - start + end = time.perf_counter() + total_time = end - start print('Quant mode: ', quant_mode) print('Time(s): ', total_time) print('Iteration: ', test_iter) diff --git a/ktransformers/ktransformers_ext/bench/bench_mlp_torch.py b/ktransformers/ktransformers_ext/bench/bench_mlp_torch.py index 3aad58c..7b811d8 100644 --- a/ktransformers/ktransformers_ext/bench/bench_mlp_torch.py +++ b/ktransformers/ktransformers_ext/bench/bench_mlp_torch.py @@ -14,17 +14,38 @@ import torch import torch.nn.quantized as nnq +scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset + +hidden_size = 5120 +intermediate_size = 3072 +layer_num = 10 +qlen = 1 +warm_up_iter = 1000 +test_iter = 10000 + def act_fn(x): return x / (1.0 + torch.exp(-x)) +def mlp_torch(input, gate_proj, up_proj, down_proj): + if isinstance(gate_proj, nnq.Linear): + input_q = torch.quantize_per_tensor(input.to(torch.float32), scale, zero_point, torch.quint8) + gate_buf = gate_proj(input_q) + up_buf = up_proj(input_q) + gate_buf = gate_buf.dequantize() + up_buf = up_buf.dequantize() + intermediate = act_fn(gate_buf) * up_buf + intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) + expert_output = down_proj(intermediate_q) + ret = expert_output.dequantize() + else: + gate_buf = torch.mm(input.to(gate_proj.dtype), gate_proj.t()) + up_buf = torch.mm(input.to(up_proj.dtype), up_proj.t()) + intermediate = act_fn(gate_buf) * up_buf + ret = torch.mm(intermediate.to(down_proj.dtype), down_proj.t()) + return ret + def bench_mlp(quant_mode: str): with torch.inference_mode(mode=True): - hidden_size = 5120 - intermediate_size = 3072 - layer_num = 10 - warm_up_iter = 1000 - test_iter = 10000 - if quant_mode == "fp32": proj_type = torch.float32 bytes_per_elem = 4.000000 @@ -48,7 +69,6 @@ def bench_mlp(quant_mode: str): up_proj = torch.randn((intermediate_size, hidden_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() down_proj = torch.randn((hidden_size, intermediate_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() if quant_mode == "qint8": - scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset gate_proj_q = torch.quantize_per_tensor(gate_proj, scale, zero_point, torch.qint8) quantized_gate = nnq.Linear(hidden_size, intermediate_size) quantized_gate.set_weight_bias(gate_proj_q, None) @@ -65,58 +85,18 @@ def bench_mlp(quant_mode: str): gate_projs.append(gate_proj.to(proj_type)) up_projs.append(up_proj.to(proj_type)) down_projs.append(down_proj.to(proj_type)) + input = torch.randn((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - input = torch.randn((1, hidden_size), dtype=torch.float32).contiguous() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - quantized_gate = gate_projs[i % layer_num] - gate_buf = quantized_gate(input_q) - quantized_up = up_projs[i % layer_num] - up_buf = quantized_gate(input_q) - gate_buf = gate_buf.dequantize() - up_buf = up_buf.dequantize() - intermediate = act_fn(gate_buf) * up_buf - intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) - quantized_down = down_projs[i % layer_num] - t_output = quantized_down(intermediate_q) - else: - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - gate_buf = torch.mm(input.to(proj_type), gate_proj.t()) - up_buf = torch.mm(input.to(proj_type), up_proj.t()) - intermediate = act_fn(gate_buf) * up_buf - t_output = torch.mm(intermediate.to(proj_type), down_proj.t()) + mlp_torch(input[i % layer_num], gate_projs[i % layer_num], up_projs[i % layer_num], down_projs[i % layer_num]) # test - total_time = 0 + start = time.perf_counter() for i in range(test_iter): - input = torch.randn((1, hidden_size), dtype=torch.float32).contiguous() - start = time.perf_counter() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - quantized_gate = gate_projs[i % layer_num] - gate_buf = quantized_gate(input_q) - quantized_up = up_projs[i % layer_num] - up_buf = quantized_gate(input_q) - gate_buf = gate_buf.dequantize() - up_buf = up_buf.dequantize() - intermediate = act_fn(gate_buf) * up_buf - intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) - quantized_down = down_projs[i % layer_num] - t_output = quantized_down(intermediate_q) - else: - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - gate_buf = torch.mm(input.to(proj_type), gate_proj.t()) - up_buf = torch.mm(input.to(proj_type), up_proj.t()) - intermediate = act_fn(gate_buf) * up_buf - t_output = torch.mm(intermediate.to(proj_type), down_proj.t()) - end = time.perf_counter() - total_time += end - start + mlp_torch(input[i % layer_num], gate_projs[i % layer_num], up_projs[i % layer_num], down_projs[i % layer_num]) + end = time.perf_counter() + total_time = end - start print('Quant mode: ', quant_mode) print('Time(s): ', total_time) print('Iteration: ', test_iter) diff --git a/ktransformers/ktransformers_ext/bench/bench_moe.py b/ktransformers/ktransformers_ext/bench/bench_moe.py index 909f029..6d617b7 100644 --- a/ktransformers/ktransformers_ext/bench/bench_moe.py +++ b/ktransformers/ktransformers_ext/bench/bench_moe.py @@ -6,7 +6,7 @@ Date : 2024-07-25 10:32:05 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:33:00 +LastEditTime : 2024-08-06 10:41:28 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,21 +15,21 @@ import cpuinfer_ext import torch +expert_num = 160 +hidden_size = 5120 +intermediate_size = 1536 +stride = 16 +group_min_len = 10 +group_max_len = 1024 +n_routed_experts = 6 +layer_num = 10 +qlen = 1 +CPUInfer = cpuinfer_ext.CPUInfer(64) +warm_up_iter = 1000 +test_iter = 10000 + def bench_moe(quant_mode: str): with torch.inference_mode(mode=True): - expert_num = 10 - hidden_size = 5120 - intermediate_size = 1536 - stride = 16 - group_min_len = 10 - group_max_len = 1024 - n_routed_experts = 6 - layer_num = 10 - qlen = 1 - CPUInfer = cpuinfer_ext.CPUInfer(64) - warm_up_iter = 1000 - test_iter = 10000 - hidden_type = 30 # ggml_type::GGML_TYPE_BF16 if quant_mode == "fp32": gate_type = 0 # ggml_type::GGML_TYPE_F32 @@ -104,32 +104,38 @@ def bench_moe(quant_mode: str): up_projs.append(up_proj) down_projs.append(down_proj) moes.append(moe) - expert_ids = torch.randint(0, expert_num, (layer_num, qlen, n_routed_experts), dtype=torch.int64, device = "cuda").to("cpu").contiguous() + expert_ids = torch.stack([torch.stack([torch.randperm(expert_num, dtype=torch.int64, device = "cuda")[:n_routed_experts] for _ in range(qlen)]) for _ in range(layer_num)]).to("cpu").contiguous() weights = torch.rand((layer_num, qlen, n_routed_experts), dtype=torch.float32, device = "cuda").to("cpu").contiguous() input = torch.randn((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() output = torch.empty((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - CPUInfer.submit(moes[i % layer_num].forward, - qlen, - n_routed_experts, - expert_ids[i % layer_num].data_ptr(), - weights[i % layer_num].data_ptr(), - input[i % layer_num].data_ptr(), - output[i % layer_num].data_ptr()) + CPUInfer.submit( + moes[i % layer_num].forward( + qlen, + n_routed_experts, + expert_ids[i % layer_num].data_ptr(), + weights[i % layer_num].data_ptr(), + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() # test start = time.perf_counter() for i in range(test_iter): - CPUInfer.submit(moes[i % layer_num].forward, - qlen, - n_routed_experts, - expert_ids[i % layer_num].data_ptr(), - weights[i % layer_num].data_ptr(), - input[i % layer_num].data_ptr(), - output[i % layer_num].data_ptr()) + CPUInfer.submit( + moes[i % layer_num].forward( + qlen, + n_routed_experts, + expert_ids[i % layer_num].data_ptr(), + weights[i % layer_num].data_ptr(), + input[i % layer_num].data_ptr(), + output[i % layer_num].data_ptr() + ) + ) CPUInfer.sync() end = time.perf_counter() total_time = end - start diff --git a/ktransformers/ktransformers_ext/bench/bench_moe_torch.py b/ktransformers/ktransformers_ext/bench/bench_moe_torch.py index 5075636..1aecf40 100644 --- a/ktransformers/ktransformers_ext/bench/bench_moe_torch.py +++ b/ktransformers/ktransformers_ext/bench/bench_moe_torch.py @@ -14,19 +14,71 @@ import torch import torch.nn.quantized as nnq +scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset + +expert_num = 160 +hidden_size = 5120 +intermediate_size = 1536 +n_routed_experts = 6 +layer_num = 10 +qlen = 1 +warm_up_iter = 1000 +test_iter = 10000 + def act_fn(x): return x / (1.0 + torch.exp(-x)) +def mlp_torch(input, gate_proj, up_proj, down_proj): + if isinstance(gate_proj, nnq.Linear): + input_q = torch.quantize_per_tensor(input.to(torch.float32), scale, zero_point, torch.quint8) + gate_buf = gate_proj(input_q) + up_buf = up_proj(input_q) + gate_buf = gate_buf.dequantize() + up_buf = up_buf.dequantize() + intermediate = act_fn(gate_buf) * up_buf + intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) + expert_output = down_proj(intermediate_q) + ret = expert_output.dequantize() + else: + gate_buf = torch.mm(input.to(gate_proj.dtype), gate_proj.t()) + up_buf = torch.mm(input.to(up_proj.dtype), up_proj.t()) + intermediate = act_fn(gate_buf) * up_buf + ret = torch.mm(intermediate.to(down_proj.dtype), down_proj.t()) + return ret + +def moe_torch(input, expert_ids, weights, gate_proj, up_proj, down_proj): + cnts = expert_ids.new_zeros((expert_ids.shape[0], expert_num)) + cnts.scatter_(1, expert_ids, 1) + tokens_per_expert = cnts.sum(dim=0) + idxs = expert_ids.view(-1).argsort() + sorted_tokens = input[idxs // expert_ids.shape[1]] + + outputs = [] + start_idx = 0 + for i, num_tokens in enumerate(tokens_per_expert): + end_idx = start_idx + num_tokens + if num_tokens == 0: + continue + tokens_for_this_expert = sorted_tokens[start_idx:end_idx] + expert_out = mlp_torch(tokens_for_this_expert, gate_proj[i], up_proj[i], down_proj[i]) + outputs.append(expert_out) + start_idx = end_idx + + outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0) + + new_x = torch.empty_like(outs) + new_x[idxs] = outs + t_output = ( + new_x.view(*expert_ids.shape, -1) + .type(weights.dtype) + .mul_(weights.unsqueeze(dim=-1)) + .sum(dim=1) + .type(new_x.dtype) + ) + return t_output + def bench_moe(quant_mode: str): with torch.inference_mode(mode=True): - expert_num = 10 - hidden_size = 5120 - intermediate_size = 1536 - n_routed_experts = 6 - layer_num = 10 - warm_up_iter = 1000 - test_iter = 10000 - if quant_mode == "fp32": proj_type = torch.float32 bytes_per_elem = 4.000000 @@ -50,7 +102,6 @@ def bench_moe(quant_mode: str): up_proj = torch.randn((expert_num, intermediate_size, hidden_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() down_proj = torch.randn((expert_num, hidden_size, intermediate_size), dtype=torch.float32, device = "cuda").to("cpu").contiguous() if quant_mode == "qint8": - scale, zero_point = 0.1, 0 # Adjust scale and zero_point based on your dataset quantized_gate_proj = [] quantized_up_proj = [] quantized_down_proj = [] @@ -74,82 +125,20 @@ def bench_moe(quant_mode: str): gate_projs.append(gate_proj.to(proj_type)) up_projs.append(up_proj.to(proj_type)) down_projs.append(down_proj.to(proj_type)) + expert_ids = torch.stack([torch.stack([torch.randperm(expert_num, dtype=torch.int64, device = "cuda")[:n_routed_experts] for _ in range(qlen)]) for _ in range(layer_num)]).to("cpu").contiguous() + weights = torch.rand((layer_num, qlen, n_routed_experts), dtype=torch.float32, device = "cuda").to("cpu").contiguous() + input = torch.randn((layer_num, qlen, hidden_size), dtype=torch.bfloat16, device = "cuda").to("cpu").contiguous() # warm up for i in range(warm_up_iter): - expert_ids = torch.randint(0, expert_num, (n_routed_experts,), dtype=torch.int64).contiguous() - weights = torch.rand((n_routed_experts,), dtype=torch.float32).contiguous() - input = torch.randn((1, hidden_size), dtype=torch.float32).contiguous() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - t_output = torch.zeros((1, hidden_size), dtype=torch.float32).contiguous() - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - for i, expert_id in enumerate(expert_ids): - quantized_gate = gate_proj[expert_id] - gate_buf = quantized_gate(input_q) - quantized_up = up_proj[expert_id] - up_buf = quantized_up(input_q) - gate_buf = gate_buf.dequantize() - up_buf = up_buf.dequantize() - intermediate = act_fn(gate_buf) * up_buf - intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) - quantized_down = down_proj[expert_id] - expert_output = quantized_down(intermediate_q) - expert_output = expert_output.dequantize() - t_output += weights[i] * expert_output - else: - t_output = torch.zeros((1, hidden_size), dtype=proj_type).contiguous() - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - for i, expert_id in enumerate(expert_ids): - gate_buf = torch.mm(input.to(proj_type), gate_proj[expert_id].t()) - up_buf = torch.mm(input.to(proj_type), up_proj[expert_id].t()) - intermediate = act_fn(gate_buf) * up_buf - expert_output = torch.mm(intermediate.to(proj_type), down_proj[expert_id].t()) - t_output += weights[i] * expert_output + moe_torch(input[i % layer_num], expert_ids[i % layer_num], weights[i % layer_num], gate_projs[i % layer_num], up_projs[i % layer_num], down_projs[i % layer_num]) # test - total_time = 0 + start = time.perf_counter() for i in range(test_iter): - expert_ids = torch.randint(0, expert_num, (n_routed_experts,), dtype=torch.int64).contiguous() - weights = torch.rand((n_routed_experts,), dtype=torch.float32).contiguous() - input = torch.randn((1, hidden_size), dtype=torch.float32).contiguous() - start = time.perf_counter() - if quant_mode == "qint8": - input_q = torch.quantize_per_tensor(input, scale, zero_point, torch.quint8) - t_output = torch.zeros((1, hidden_size), dtype=torch.float32).contiguous() - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - for i, expert_id in enumerate(expert_ids): - quantized_gate = gate_proj[expert_id] - gate_buf = quantized_gate(input_q) - quantized_up = up_proj[expert_id] - up_buf = quantized_up(input_q) - gate_buf = gate_buf.dequantize() - up_buf = up_buf.dequantize() - intermediate = act_fn(gate_buf) * up_buf - intermediate_q = torch.quantize_per_tensor(intermediate, scale, zero_point, torch.quint8) - quantized_down = down_proj[expert_id] - expert_output = quantized_down(intermediate_q) - expert_output = expert_output.dequantize() - t_output += weights[i] * expert_output - else: - t_output = torch.zeros((1, hidden_size), dtype=proj_type).contiguous() - gate_proj = gate_projs[i%layer_num] - up_proj = up_projs[i%layer_num] - down_proj = down_projs[i%layer_num] - for i, expert_id in enumerate(expert_ids): - gate_buf = torch.mm(input.to(proj_type), gate_proj[expert_id].t()) - up_buf = torch.mm(input.to(proj_type), up_proj[expert_id].t()) - intermediate = act_fn(gate_buf) * up_buf - expert_output = torch.mm(intermediate.to(proj_type), down_proj[expert_id].t()) - t_output += weights[i] * expert_output - end = time.perf_counter() - total_time += end - start + moe_torch(input[i % layer_num], expert_ids[i % layer_num], weights[i % layer_num], gate_projs[i % layer_num], up_projs[i % layer_num], down_projs[i % layer_num]) + end = time.perf_counter() + total_time = end - start print('Quant mode: ', quant_mode) print('Time(s): ', total_time) print('Iteration: ', test_iter) diff --git a/ktransformers/ktransformers_ext/cpu_backend/cpuinfer.h b/ktransformers/ktransformers_ext/cpu_backend/cpuinfer.h index eae6f90..9618e6b 100644 --- a/ktransformers/ktransformers_ext/cpu_backend/cpuinfer.h +++ b/ktransformers/ktransformers_ext/cpu_backend/cpuinfer.h @@ -1,12 +1,12 @@ /** - * @Description : + * @Description : * @Author : chenht2022 * @Date : 2024-07-16 10:43:18 * @Version : 1.0.0 - * @LastEditors : chenht2022 - * @LastEditTime : 2024-07-25 10:33:42 - * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. -**/ + * @LastEditors : chenht2022 + * @LastEditTime : 2024-08-07 09:47:43 + * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. + **/ #ifndef CPUINFER_CPUINFER_H #define CPUINFER_CPUINFER_H @@ -17,6 +17,7 @@ #include #include #include +#include "cuda_runtime.h" #include "backend.h" #include "task_queue.h" @@ -39,16 +40,39 @@ class CPUInfer { } template - void submit(Func f, Obj* obj, Args... args) { + void enqueue(Func f, Obj* obj, Args... args) { task_queue_->enqueue([=]() { std::invoke(f, *obj, args..., backend_); }); } + void submit(std::pair params) { + void (*func)(void*) = (void (*)(void*))params.first; + void* args = (void*)params.second; + *((CPUInfer**)args) = this; + func(args); + } + void sync() { task_queue_->sync(); } + void submit_with_cuda_stream(intptr_t user_cuda_stream, std::pair params) { + void (*func)(void*) = (void (*)(void*))params.first; + void* args = (void*)params.second; + *((CPUInfer**)args) = this; + cudaLaunchHostFunc((cudaStream_t)user_cuda_stream, (cudaHostFn_t)func, args); + } + + static void sync_(void* cpu_infer_ptr) { + CPUInfer* cpuinfer = (CPUInfer*)cpu_infer_ptr; + cpuinfer->sync(); + } + + void sync_with_cuda_stream(intptr_t user_cuda_stream) { + cudaLaunchHostFunc((cudaStream_t)user_cuda_stream, (cudaHostFn_t)&sync_, (void*)this); + } + public: Backend* backend_; TaskQueue* task_queue_; diff --git a/ktransformers/ktransformers_ext/examples/test_linear.py b/ktransformers/ktransformers_ext/examples/test_linear.py index 6cb8d0c..7a331db 100644 --- a/ktransformers/ktransformers_ext/examples/test_linear.py +++ b/ktransformers/ktransformers_ext/examples/test_linear.py @@ -6,7 +6,7 @@ Date : 2024-07-25 10:32:05 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:34:00 +LastEditTime : 2024-08-06 10:36:59 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,23 +15,23 @@ import cpuinfer_ext import torch -with torch.inference_mode(mode=True): - input_size = 16384 - output_size = 5120 - stride = 32 - proj_type = 1 # ggml_type::GGML_TYPE_F16 - hidden_type = 1 # ggml_type::GGML_TYPE_F16 - layer_num = 10 - CPUInfer = cpuinfer_ext.CPUInfer(48) - validation_iter = 100 - warm_up_iter = 1000 - test_iter = 10000 +input_size = 16384 +output_size = 5120 +stride = 32 +group_max_len = 1024 +proj_type = 1 # ggml_type::GGML_TYPE_F16 +hidden_type = 1 # ggml_type::GGML_TYPE_F16 +qlen = 30 +layer_num = 10 +CPUInfer = cpuinfer_ext.CPUInfer(48) +validation_iter = 100 +with torch.inference_mode(mode=True): linears = [] projs = [] for _ in range(layer_num): proj = torch.randn((output_size, input_size), dtype=torch.float16, device = "cuda").to("cpu").contiguous() - config = cpuinfer_ext.linear.LinearConfig(input_size, output_size, stride, proj.data_ptr(), proj_type, hidden_type) + config = cpuinfer_ext.linear.LinearConfig(input_size, output_size, stride, group_max_len, proj.data_ptr(), proj_type, hidden_type) linear = cpuinfer_ext.linear.Linear(config) projs.append(proj) linears.append(linear) @@ -39,11 +39,17 @@ # validation for i in range(validation_iter): linear = linears[i % layer_num] - input = torch.randn((1, input_size), dtype=torch.float16).contiguous() - output = torch.empty((1, output_size), dtype=torch.float16).contiguous() + input = torch.randn((qlen, input_size), dtype=torch.float16).contiguous() + output = torch.empty((qlen, output_size), dtype=torch.float16).contiguous() input = input / 100 - CPUInfer.submit(linear.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + linear.forward( + qlen, + input.data_ptr(), + output.data_ptr() + ) + ) CPUInfer.sync() # print('cpuinfer output', output) @@ -54,30 +60,3 @@ diff = torch.mean(torch.abs(output - t_output)) / torch.mean(torch.abs(t_output)) print('diff = ', diff) assert(diff < 0.001) - - # warm up - for i in range(warm_up_iter): - linear = linears[i % layer_num] - input = torch.randn((1, input_size), dtype=torch.float16).contiguous() - output = torch.empty((1, output_size), dtype=torch.float16).contiguous() - input = input / 100 - CPUInfer.submit(linear.forward, input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - - # test - total_time = 0 - for i in range(test_iter): - linear = linears[i % layer_num] - input = torch.randn((1, input_size), dtype=torch.float16).contiguous() - output = torch.empty((1, output_size), dtype=torch.float16).contiguous() - input = input / 100 - start = time.perf_counter() - CPUInfer.submit(linear.forward, input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - end = time.perf_counter() - total_time += end - start - print('Time: ', total_time) - print('Iteration: ', test_iter) - print('Time per iteration: ', total_time / test_iter) - print('Bandwidth: ', input_size * output_size * 2 * test_iter / total_time / 1000 / 1000 / 1000, 'GB/s') - print("All tasks completed.") \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/examples/test_mlp.py b/ktransformers/ktransformers_ext/examples/test_mlp.py index d965877..9805e72 100644 --- a/ktransformers/ktransformers_ext/examples/test_mlp.py +++ b/ktransformers/ktransformers_ext/examples/test_mlp.py @@ -6,7 +6,7 @@ Date : 2024-07-25 10:32:05 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:34:03 +LastEditTime : 2024-08-06 10:37:28 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,20 +15,30 @@ import cpuinfer_ext import torch -with torch.inference_mode(mode=True): - hidden_size = 5120 - intermediate_size = 3072 - stride = 32 - gate_type = 1 # ggml_type::GGML_TYPE_F16 - up_type = 1 # ggml_type::GGML_TYPE_F16 - down_type = 1 # ggml_type::GGML_TYPE_F16 - hidden_type = 1 # ggml_type::GGML_TYPE_F16 - layer_num = 10 - CPUInfer = cpuinfer_ext.CPUInfer(48) - validation_iter = 100 - warm_up_iter = 1000 - test_iter = 10000 +hidden_size = 5120 +intermediate_size = 3072 +stride = 32 +group_max_len = 1024 +gate_type = 1 # ggml_type::GGML_TYPE_F16 +up_type = 1 # ggml_type::GGML_TYPE_F16 +down_type = 1 # ggml_type::GGML_TYPE_F16 +hidden_type = 1 # ggml_type::GGML_TYPE_F16 +qlen = 30 +layer_num = 10 +CPUInfer = cpuinfer_ext.CPUInfer(48) +validation_iter = 100 + +def act_fn(x): + return x / (1.0 + torch.exp(-x)) + +def mlp_torch(input, gate_proj, up_proj, down_proj): + gate_buf = torch.mm(input, gate_proj.t()) + up_buf = torch.mm(input, up_proj.t()) + intermediate = act_fn(gate_buf) * up_buf + ret = torch.mm(intermediate, down_proj.t()) + return ret +with torch.inference_mode(mode=True): mlps = [] gate_projs = [] up_projs = [] @@ -37,7 +47,7 @@ gate_proj = torch.randn((intermediate_size, hidden_size), dtype=torch.float16, device = "cuda").to("cpu").contiguous() up_proj = torch.randn((intermediate_size, hidden_size), dtype=torch.float16, device = "cuda").to("cpu").contiguous() down_proj = torch.randn((hidden_size, intermediate_size), dtype=torch.float16, device = "cuda").to("cpu").contiguous() - config = cpuinfer_ext.mlp.MLPConfig(hidden_size, intermediate_size, stride, gate_proj.data_ptr(), up_proj.data_ptr(), down_proj.data_ptr(), gate_type, up_type, down_type, hidden_type) + config = cpuinfer_ext.mlp.MLPConfig(hidden_size, intermediate_size, stride, group_max_len, gate_proj.data_ptr(), up_proj.data_ptr(), down_proj.data_ptr(), gate_type, up_type, down_type, hidden_type) mlp = cpuinfer_ext.mlp.MLP(config) gate_projs.append(gate_proj) up_projs.append(up_proj) @@ -47,52 +57,26 @@ # validation for i in range(validation_iter): mlp = mlps[i % layer_num] - input = torch.randn((1, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((1, hidden_size), dtype=torch.float16).contiguous() + input = torch.randn((qlen, hidden_size), dtype=torch.float16).contiguous() + output = torch.empty((qlen, hidden_size), dtype=torch.float16).contiguous() input = input / 100 - CPUInfer.submit(mlp.forward, input.data_ptr(), output.data_ptr()) + CPUInfer.submit( + mlp.forward( + qlen, + input.data_ptr(), + output.data_ptr() + ) + ) CPUInfer.sync() # print('cpuinfer output', output) - def act_fn(x): - return x / (1.0 + torch.exp(-x)) gate_proj = gate_projs[i%layer_num] up_proj = up_projs[i%layer_num] down_proj = down_projs[i%layer_num] - gate_buf = torch.mm(input, gate_proj.t()) - up_buf = torch.mm(input, up_proj.t()) - intermediate = act_fn(gate_buf) * up_buf - t_output = torch.mm(intermediate, down_proj.t()) + t_output = mlp_torch(input, gate_proj, up_proj, down_proj) # print('torch output', t_output) diff = torch.mean(torch.abs(output - t_output)) / torch.mean(torch.abs(t_output)) print('diff = ', diff) assert(diff < 0.001) - - # warm up - for i in range(warm_up_iter): - mlp = mlps[i % layer_num] - input = torch.randn((1, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((1, hidden_size), dtype=torch.float16).contiguous() - input = input / 100 - CPUInfer.submit(mlp.forward, input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - - # test - total_time = 0 - for i in range(test_iter): - mlp = mlps[i % layer_num] - input = torch.randn((1, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((1, hidden_size), dtype=torch.float16).contiguous() - input = input / 100 - start = time.time() - CPUInfer.submit(mlp.forward, input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - end = time.time() - total_time += end - start - print('Time: ', total_time) - print('Iteration: ', test_iter) - print('Time per iteration: ', total_time / test_iter) - print('Bandwidth: ', hidden_size * intermediate_size * 3 * 2 * test_iter / total_time / 1024 / 1024 / 1024, 'GB/s') - print("All tasks completed.") \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/examples/test_moe.py b/ktransformers/ktransformers_ext/examples/test_moe.py index 0597811..3fa4dbd 100644 --- a/ktransformers/ktransformers_ext/examples/test_moe.py +++ b/ktransformers/ktransformers_ext/examples/test_moe.py @@ -6,7 +6,7 @@ Date : 2024-07-25 10:32:05 Version : 1.0.0 LastEditors : chenht2022 -LastEditTime : 2024-07-25 10:34:06 +LastEditTime : 2024-08-06 10:38:05 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' import os, sys @@ -15,25 +15,64 @@ import cpuinfer_ext import torch -with torch.inference_mode(mode=True): - expert_num = 10 - hidden_size = 5120 - intermediate_size = 1536 - stride = 32 - group_min_len = 10 - group_max_len = 1024 - gate_type = 1 # ggml_type::GGML_TYPE_F16 - up_type = 1 # ggml_type::GGML_TYPE_F16 - down_type = 1 # ggml_type::GGML_TYPE_F16 - hidden_type = 1 # ggml_type::GGML_TYPE_F16 - n_routed_experts = 6 - qlen = 30 - layer_num = 10 - CPUInfer = cpuinfer_ext.CPUInfer(48) - validation_iter = 100 - warm_up_iter = 1000 - test_iter = 10000 +expert_num = 160 +hidden_size = 5120 +intermediate_size = 1536 +stride = 32 +group_min_len = 10 +group_max_len = 1024 +gate_type = 1 # ggml_type::GGML_TYPE_F16 +up_type = 1 # ggml_type::GGML_TYPE_F16 +down_type = 1 # ggml_type::GGML_TYPE_F16 +hidden_type = 1 # ggml_type::GGML_TYPE_F16 +n_routed_experts = 6 +qlen = 30 +layer_num = 10 +CPUInfer = cpuinfer_ext.CPUInfer(48) +validation_iter = 100 + +def act_fn(x): + return x / (1.0 + torch.exp(-x)) + +def mlp_torch(input, gate_proj, up_proj, down_proj): + gate_buf = torch.mm(input, gate_proj.t()) + up_buf = torch.mm(input, up_proj.t()) + intermediate = act_fn(gate_buf) * up_buf + ret = torch.mm(intermediate, down_proj.t()) + return ret + +def moe_torch(input, expert_ids, weights, gate_proj, up_proj, down_proj): + cnts = expert_ids.new_zeros((expert_ids.shape[0], expert_num)) + cnts.scatter_(1, expert_ids, 1) + tokens_per_expert = cnts.sum(dim=0) + idxs = expert_ids.view(-1).argsort() + sorted_tokens = input[idxs // expert_ids.shape[1]] + + outputs = [] + start_idx = 0 + for i, num_tokens in enumerate(tokens_per_expert): + end_idx = start_idx + num_tokens + if num_tokens == 0: + continue + tokens_for_this_expert = sorted_tokens[start_idx:end_idx] + expert_out = mlp_torch(tokens_for_this_expert, gate_proj[i], up_proj[i], down_proj[i]) + outputs.append(expert_out) + start_idx = end_idx + + outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0) + + new_x = torch.empty_like(outs) + new_x[idxs] = outs + t_output = ( + new_x.view(*expert_ids.shape, -1) + .type(weights.dtype) + .mul_(weights.unsqueeze(dim=-1)) + .sum(dim=1) + .type(new_x.dtype) + ) + return t_output +with torch.inference_mode(mode=True): moes = [] gate_projs = [] up_projs = [] @@ -51,63 +90,32 @@ # validation for i in range(validation_iter): - moe = moes[i % layer_num] - expert_ids = torch.randint(0, expert_num, (qlen, n_routed_experts), dtype=torch.int64).contiguous() + expert_ids = torch.stack([torch.randperm(expert_num)[:n_routed_experts] for _ in range(qlen)]).contiguous() weights = torch.rand((qlen, n_routed_experts), dtype=torch.float32).contiguous() - input = torch.randn((qlen, 1, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((qlen, 1, hidden_size), dtype=torch.float16).contiguous() + input = torch.randn((qlen, hidden_size), dtype=torch.float16).contiguous() + output = torch.empty((qlen, hidden_size), dtype=torch.float16).contiguous() input = input / 100 - CPUInfer.submit(moe.forward, qlen, n_routed_experts, expert_ids.data_ptr(), weights.data_ptr(), input.data_ptr(), output.data_ptr()) + moe = moes[i % layer_num] + CPUInfer.submit( + moe.forward( + qlen, + n_routed_experts, + expert_ids.data_ptr(), + weights.data_ptr(), + input.data_ptr(), + output.data_ptr() + ) + ) CPUInfer.sync() # print('cpuinfer output', output) - def act_fn(x): - return x / (1.0 + torch.exp(-x)) - t_output = torch.zeros((qlen, 1, hidden_size), dtype=torch.float32).contiguous() gate_proj = gate_projs[i%layer_num] up_proj = up_projs[i%layer_num] down_proj = down_projs[i%layer_num] - for token_idx in range(qlen): - for i, expert_id in enumerate(expert_ids[token_idx]): - gate_buf = torch.mm(input[token_idx], gate_proj[expert_id].t()) - up_buf = torch.mm(input[token_idx], up_proj[expert_id].t()) - intermediate = act_fn(gate_buf) * up_buf - expert_output = torch.mm(intermediate, down_proj[expert_id].t()) - t_output[token_idx] += weights[token_idx][i] * expert_output + t_output = moe_torch(input, expert_ids, weights, gate_proj, up_proj, down_proj) # print('torch output', t_output) diff = torch.mean(torch.abs(output - t_output)) / torch.mean(torch.abs(t_output)) print('diff = ', diff) assert(diff < 0.001) - - # warm up - for i in range(warm_up_iter): - moe = moes[i % layer_num] - expert_ids = torch.randint(0, expert_num, (qlen, n_routed_experts), dtype=torch.int64).contiguous() - weights = torch.rand((qlen, n_routed_experts), dtype=torch.float32).contiguous() - input = torch.randn((qlen, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((qlen, hidden_size), dtype=torch.float16).contiguous() - input = input / 100 - CPUInfer.submit(moe.forward, qlen, n_routed_experts, expert_ids.data_ptr(), weights.data_ptr(), input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - - # test - total_time = 0 - for i in range(test_iter): - moe = moes[i % layer_num] - expert_ids = torch.randint(0, expert_num, (qlen, n_routed_experts), dtype=torch.int64).contiguous() - weights = torch.rand((qlen, n_routed_experts), dtype=torch.float32).contiguous() - input = torch.randn((qlen, hidden_size), dtype=torch.float16).contiguous() - output = torch.empty((qlen, hidden_size), dtype=torch.float16).contiguous() - input = input / 100 - start = time.perf_counter() - CPUInfer.submit(moe.forward, qlen, n_routed_experts, expert_ids.data_ptr(), weights.data_ptr(), input.data_ptr(), output.data_ptr()) - CPUInfer.sync() - end = time.perf_counter() - total_time += end - start - print('Time: ', total_time) - print('Iteration: ', test_iter) - print('Time per iteration: ', total_time / test_iter) - print('Bandwidth: ', hidden_size * intermediate_size * 3 * n_routed_experts * 2 * test_iter / total_time / 1000 / 1000 / 1000, 'GB/s') - print("All tasks completed.") \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/ext_bindings.cpp b/ktransformers/ktransformers_ext/ext_bindings.cpp index 0aeead3..ef30037 100644 --- a/ktransformers/ktransformers_ext/ext_bindings.cpp +++ b/ktransformers/ktransformers_ext/ext_bindings.cpp @@ -3,8 +3,8 @@ * @Author : chenht2022 * @Date : 2024-07-22 02:03:22 * @Version : 1.0.0 - * @LastEditors : chenht2022 - * @LastEditTime : 2024-07-25 10:34:23 + * @LastEditors : chenht2022 + * @LastEditTime : 2024-08-07 10:39:37 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ // Python bindings @@ -12,7 +12,6 @@ #include #include #include "cpu_backend/cpuinfer.h" -#include "cuda_runtime.h" #include "device_launch_parameters.h" #include "llamafile/flags.h" #include "operators/llamafile/linear.h" @@ -26,239 +25,155 @@ namespace py = pybind11; using namespace pybind11::literals; -// Binding functions for the Linear class class LinearBindings { public: - static void bind_forward(CPUInfer& cpuinfer, Linear* linear, py::args args, py::kwargs kwargs) { - auto input = args[0].cast(); - auto output = args[1].cast(); - cpuinfer.submit(&Linear::forward, linear, - (const void*)input, (void*)output); - } - - static void bind_warm_up(CPUInfer& cpuinfer, Linear* linear, py::args args, py::kwargs kwargs) { - cpuinfer.submit(&Linear::warm_up, linear); - } - - static void bind_functions(CPUInfer& cpuinfer, py::object func, py::args args, py::kwargs kwargs) { - auto linear = func.attr("__self__").cast(); - std::string func_name = py::str(func.attr("__func__").attr("__name__")); - - if (func_name == "forward") { - bind_forward(cpuinfer, linear, args, kwargs); - } else if (func_name == "warm_up") { - bind_warm_up(cpuinfer, linear, args, kwargs); - } else { - throw py::value_error("Unsupported function: " + - std::string(func_name)); + class WarmUpBindinds { + public: + struct Args { + CPUInfer* cpuinfer; + Linear* linear; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&Linear::warm_up, args_->linear); + } + static std::pair interface(Linear& linear) { + Args* args = new Args{nullptr, &linear}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; + class ForwardBindings { + public: + struct Args { + CPUInfer* cpuinfer; + Linear* linear; + int qlen; + const void* input; + void* output; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&Linear::forward, args_->linear, args_->qlen, args_->input, args_->output); } - } + static std::pair interface(Linear& linear, int qlen, intptr_t input, intptr_t output) { + Args* args = new Args{nullptr, &linear, qlen, (const void*)input, (void*)output}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; }; -// Binding functions for the MLP class class MLPBindings { public: - static void bind_forward(CPUInfer& cpuinfer, MLP* mlp, py::args args, py::kwargs kwargs) { - auto input = args[0].cast(); - auto output = args[1].cast(); - cpuinfer.submit(&MLP::forward, mlp, - (const void*)input, (void*)output); - } - - static void bind_warm_up(CPUInfer& cpuinfer, MLP* mlp, py::args args, py::kwargs kwargs) { - cpuinfer.submit(&MLP::warm_up, mlp); - } - - static void bind_functions(CPUInfer& cpuinfer, py::object func, py::args args, py::kwargs kwargs) { - auto mlp = func.attr("__self__").cast(); - std::string func_name = py::str(func.attr("__func__").attr("__name__")); - - if (func_name == "forward") { - bind_forward(cpuinfer, mlp, args, kwargs); - } else if (func_name == "warm_up") { - bind_warm_up(cpuinfer, mlp, args, kwargs); - } else { - throw py::value_error("Unsupported function: " + - std::string(func_name)); + class WarmUpBindinds { + public: + struct Args { + CPUInfer* cpuinfer; + MLP* mlp; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&MLP::warm_up, args_->mlp); } - } + static std::pair interface(MLP& mlp) { + Args* args = new Args{nullptr, &mlp}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; + class ForwardBindings { + public: + struct Args { + CPUInfer* cpuinfer; + MLP* mlp; + int qlen; + const void* input; + void* output; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&MLP::forward, args_->mlp, args_->qlen, args_->input, args_->output); + } + static std::pair interface(MLP& mlp, int qlen, intptr_t input, intptr_t output) { + Args* args = new Args{nullptr, &mlp, qlen, (const void*)input, (void*)output}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; }; -// Binding functions for the MOE class class MOEBindings { public: - static void bind_forward(CPUInfer& cpuinfer, MOE* moe, py::args args, py::kwargs kwargs) { - int qlen = args[0].cast(); - int k = args[1].cast(); - auto expert_ids = args[2].cast(); - auto weights = args[3].cast(); - auto input = args[4].cast(); - auto output = args[5].cast(); - cpuinfer.submit(&MOE::forward, moe, - qlen, k, (const uint64_t*)expert_ids, (const float*)weights, (const void*)input, (void*)output); - } - - static void bind_warm_up(CPUInfer& cpuinfer, MOE* moe, py::args args, py::kwargs kwargs) { - cpuinfer.submit(&MOE::warm_up, moe); - } - - static void bind_functions(CPUInfer& cpuinfer, py::object func, py::args args, py::kwargs kwargs) { - auto moe = func.attr("__self__").cast(); - std::string func_name = py::str(func.attr("__func__").attr("__name__")); - - if (func_name == "forward") { - bind_forward(cpuinfer, moe, args, kwargs); - } else if (func_name == "warm_up") { - bind_warm_up(cpuinfer, moe, args, kwargs); - } else { - throw py::value_error("Unsupported function: " + - std::string(func_name)); + class WarmUpBindinds { + public: + struct Args { + CPUInfer* cpuinfer; + MOE* moe; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&MOE::warm_up, args_->moe); } - } -}; - -struct MOEForwardArgs { - CPUInfer* cpuinfer; - MOE* moe; - int qlen; - int k; - uint64_t* expert_ids; - float* weights; - void* input; - void* output; + static std::pair interface(MOE& moe) { + Args* args = new Args{nullptr, &moe}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; + class ForwardBindings { + public: + struct Args { + CPUInfer* cpuinfer; + MOE* moe; + int qlen; + int k; + const uint64_t* expert_ids; + const float* weights; + const void* input; + void* output; + }; + static void inner(void* args) { + Args* args_ = (Args*)args; + args_->cpuinfer->enqueue(&MOE::forward, args_->moe, args_->qlen, args_->k, args_->expert_ids, args_->weights, args_->input, args_->output); + } + static std::pair interface(MOE& moe, int qlen, int k, intptr_t expert_ids, intptr_t weights, intptr_t input, intptr_t output) { + Args* args = new Args{nullptr, &moe, qlen, k, (const uint64_t*)expert_ids, (const float*)weights, (const void*)input, (void*)output}; + return std::make_pair((intptr_t)&inner, (intptr_t)args); + } + }; }; -void submit_moe_forward_with_host_args_ptr(void* host_args_ptr) { - MOEForwardArgs* host_args = (MOEForwardArgs*)host_args_ptr; - host_args->cpuinfer->submit(&MOE::forward, host_args->moe, - host_args->qlen, host_args->k, host_args->expert_ids, host_args->weights, host_args->input, host_args->output); -} - -void cpuinfer_sync(void* host_args_ptr) { - CPUInfer* cpuinfer = (CPUInfer*)host_args_ptr; - cpuinfer->sync(); -} - PYBIND11_MODULE(cpuinfer_ext, m) { - auto linear_module = m.def_submodule("linear"); + py::class_(m, "CPUInfer") + .def(py::init()) + .def("submit", &CPUInfer::submit) + .def("submit_with_cuda_stream", &CPUInfer::submit_with_cuda_stream) + .def("sync", &CPUInfer::sync) + .def("sync_with_cuda_stream", &CPUInfer::sync_with_cuda_stream); + auto linear_module = m.def_submodule("linear"); py::class_(linear_module, "LinearConfig") - .def(py::init([](int hidden_size, int intermediate_size, int stride, intptr_t proj, int proj_type, int hidden_type) { - return LinearConfig(hidden_size, intermediate_size, stride, (void*)proj, (ggml_type)proj_type, (ggml_type)hidden_type); + .def(py::init([](int hidden_size, int intermediate_size, int stride, int group_max_len, intptr_t proj, int proj_type, int hidden_type) { + return LinearConfig(hidden_size, intermediate_size, stride, group_max_len, (void*)proj, (ggml_type)proj_type, (ggml_type)hidden_type); })); - py::class_(linear_module, "Linear") .def(py::init()) - .def("warm_up", [](Linear& linear) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }) - .def("forward", [](Linear& linear, intptr_t input, intptr_t output) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }); + .def("warm_up", &LinearBindings::WarmUpBindinds::interface) + .def("forward", &LinearBindings::ForwardBindings::interface); auto mlp_module = m.def_submodule("mlp"); - py::class_(mlp_module, "MLPConfig") - .def(py::init([](int hidden_size, int intermediate_size, int stride, intptr_t gate_proj, intptr_t up_proj, intptr_t down_proj, int gate_type, int up_type, int down_type, int hidden_type) { - return MLPConfig(hidden_size, intermediate_size, stride, (void*)gate_proj, (void*)up_proj, (void*)down_proj, (ggml_type)gate_type, (ggml_type)up_type, (ggml_type)down_type, (ggml_type)hidden_type); + .def(py::init([](int hidden_size, int intermediate_size, int stride, int group_max_len, intptr_t gate_proj, intptr_t up_proj, intptr_t down_proj, int gate_type, int up_type, int down_type, int hidden_type) { + return MLPConfig(hidden_size, intermediate_size, stride, group_max_len, (void*)gate_proj, (void*)up_proj, (void*)down_proj, (ggml_type)gate_type, (ggml_type)up_type, (ggml_type)down_type, (ggml_type)hidden_type); })); - py::class_(mlp_module, "MLP") .def(py::init()) - .def("warm_up", [](MLP& mlp) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }) - .def("forward", [](MLP& mlp, intptr_t input, intptr_t output) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }); + .def("warm_up", &MLPBindings::WarmUpBindinds::interface) + .def("forward", &MLPBindings::ForwardBindings::interface); auto moe_module = m.def_submodule("moe"); - py::class_(moe_module, "MOEConfig") .def(py::init([](int expert_num, int routed_expert_num, int hidden_size, int intermediate_size, int stride, int group_min_len, int group_max_len, intptr_t gate_proj, intptr_t up_proj, intptr_t down_proj, int gate_type, int up_type, int down_type, int hidden_type) { return MOEConfig(expert_num, routed_expert_num, hidden_size, intermediate_size, stride, group_min_len, group_max_len, (void*)gate_proj, (void*)up_proj, (void*)down_proj, (ggml_type)gate_type, (ggml_type)up_type, (ggml_type)down_type, (ggml_type)hidden_type); })); - py::class_(moe_module, "MOE") .def(py::init()) - .def("warm_up", [](MOE& moe) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }) - .def("forward", [](MOE& moe, int k, uint64_t expert_ids, intptr_t weights, intptr_t input, intptr_t output) { - throw std::runtime_error("!!! Doing nothing, please use CPUInfer.submit to call it!!!\n"); - }); - - py::class_(m, "CPUInfer") - .def(py::init()) - .def("submit", - [linear_module, mlp_module, moe_module](CPUInfer& cpuinfer, py::object func, py::args args, py::kwargs kwargs) { - if (py::hasattr(func, "__self__") && - py::hasattr(func, "__func__")) { - std::string class_name = py::str(func.attr("__self__") - .attr("__class__") - .attr("__name__")); - if (class_name == "Linear") { - LinearBindings::bind_functions(cpuinfer, func, - args, kwargs); - } else if (class_name == "MLP") { - MLPBindings::bind_functions(cpuinfer, func, - args, kwargs); - } else if (class_name == "MOE") { - MOEBindings::bind_functions(cpuinfer, func, - args, kwargs); - } else { - // handle other classes - throw py::type_error("Unsupported class type: " + - class_name); - } - } else { - // handle cases where func does not have __self__ or - // __func__ - throw py::type_error( - "Invalid function object: missing " - "__self__ or __func__ attribute."); - } - }) - .def("submit_with_cuda_stream", - [linear_module, mlp_module, moe_module](CPUInfer& cpuinfer, intptr_t user_cuda_stream, py::object func, py::args args, py::kwargs kwargs) { - if (py::hasattr(func, "__self__") && - py::hasattr(func, "__func__")) { - std::string class_name = py::str(func.attr("__self__") - .attr("__class__") - .attr("__name__")); - if (class_name == "MOE") { - std::string func_name = py::str(func.attr("__func__").attr("__name__")); - if (func_name == "forward") { - auto moe = func.attr("__self__").cast(); - int qlen = args[0].cast(); - int k = args[1].cast(); - auto expert_ids = args[2].cast(); - auto weights = args[3].cast(); - auto input = args[4].cast(); - auto output = args[5].cast(); - MOEForwardArgs* moe_forward_args = new MOEForwardArgs{&cpuinfer, moe, qlen, k, (uint64_t*)expert_ids, (float*)weights, (void*)input, (void*)output}; - // submit_moe_forward_with_host_args_ptr(moe_forward_args); - cudaLaunchHostFunc((cudaStream_t)user_cuda_stream, (cudaHostFn_t)submit_moe_forward_with_host_args_ptr, moe_forward_args); - } else { - throw py::value_error("Unsupported function: " + - std::string(func_name)); - } - } else { - // handle other classes - throw py::type_error("Unsupported class type: " + - class_name); - } - } else { - // handle cases where func does not have __self__ or - // __func__ - throw py::type_error( - "Invalid function object: missing " - "__self__ or __func__ attribute."); - } - }) - .def("sync_with_cuda_stream", [](CPUInfer& cpuinfer, intptr_t user_cuda_stream) { - // cpuinfer_sync((void*)(&cpuinfer)); - cudaLaunchHostFunc((cudaStream_t)user_cuda_stream, (cudaHostFn_t)cpuinfer_sync, (void*)(&cpuinfer)); - }) - .def("sync", &CPUInfer::sync); + .def("warm_up", &MOEBindings::WarmUpBindinds::interface) + .def("forward", &MOEBindings::ForwardBindings::interface); } diff --git a/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp b/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp index bf1935e..7dcba57 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp @@ -3,7 +3,7 @@ * @Author : chenht2022 * @Date : 2024-07-12 10:07:58 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:34:58 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -13,9 +13,15 @@ Linear::Linear(LinearConfig config) { config_ = config; proj_ = config_.proj; - input_fp32_.resize(config_.input_size); - proj_input_.resize(config_.input_size * 4); - proj_output_.resize(config_.output_size); + std::vector> mem_requests; + mem_requests.push_back({(void**)&input_fp32_, sizeof(float) * config_.group_max_len * config_.input_size}); + mem_requests.push_back({(void**)&proj_input_, config_.group_max_len * config_.input_size * ggml_type_size(ggml_internal_get_type_traits(config_.proj_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.proj_type).vec_dot_type)}); + mem_requests.push_back({(void**)&proj_output_, sizeof(float) * config_.group_max_len * config_.output_size}); + shared_mem_buffer.alloc(this, mem_requests); +} + +Linear::~Linear() { + shared_mem_buffer.dealloc(this); } void Linear::warm_up(Backend* backend) { @@ -26,22 +32,42 @@ void Linear::warm_up(Backend* backend) { input_fp32[i] = 0; } from_float(input_fp32.data(), input.data(), config_.input_size, config_.hidden_type); - forward(input.data(), output.data(), backend); + forward_many(1, input.data(), output.data(), backend); } -void Linear::forward(const void* input, void* output, Backend* backend) { +void Linear::forward_many(int qlen, const void* input, void* output, Backend* backend) { const void* proj_input_ptr; if (config_.hidden_type == ggml_internal_get_type_traits(config_.proj_type).vec_dot_type) { proj_input_ptr = input; } else { - to_float(input, input_fp32_.data(), config_.input_size, config_.hidden_type); - from_float(input_fp32_.data(), proj_input_.data(), config_.input_size, ggml_internal_get_type_traits(config_.proj_type).vec_dot_type); - proj_input_ptr = proj_input_.data(); + to_float(input, input_fp32_, qlen * config_.input_size, config_.hidden_type); + from_float(input_fp32_, proj_input_, qlen * config_.input_size, ggml_internal_get_type_traits(config_.proj_type).vec_dot_type); + proj_input_ptr = proj_input_; } int nth = config_.output_size / config_.stride; backend->do_work_stealing_job(nth, [&](int task_id) { - int ith = task_id % nth; - llamafile_sgemm(config_.output_size, 1, config_.input_size / ggml_blck_size(config_.proj_type), proj_, config_.input_size / ggml_blck_size(config_.proj_type), proj_input_ptr, config_.input_size / ggml_blck_size(config_.proj_type), proj_output_.data(), config_.output_size, ith, nth, GGML_TASK_TYPE_COMPUTE, config_.proj_type, ggml_internal_get_type_traits(config_.proj_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); + int ith = task_id; + void* proj_ptr = proj_ + ith * config_.stride * config_.input_size * ggml_type_size(config_.proj_type) / ggml_blck_size(config_.proj_type); + float* proj_output_ptr = proj_output_ + ith * config_.stride; + llamafile_sgemm(config_.stride, qlen, config_.input_size / ggml_blck_size(config_.proj_type), proj_ptr, config_.input_size / ggml_blck_size(config_.proj_type), proj_input_ptr, config_.input_size / ggml_blck_size(config_.proj_type), proj_output_ptr, config_.output_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.proj_type, ggml_internal_get_type_traits(config_.proj_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); + if (config_.stride % ggml_blck_size(config_.hidden_type) == 0) { + for (int i = 0; i < qlen; i++) { + float* output_fp32_ptr = proj_output_ + i * config_.output_size + ith * config_.stride; + void* output_ptr = output + i * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + from_float(output_fp32_ptr, output_ptr, config_.stride, config_.hidden_type); + } + } }); - from_float(proj_output_.data(), output, config_.output_size, config_.hidden_type); + if (config_.stride % ggml_blck_size(config_.hidden_type) != 0) { + from_float(proj_output_, output, qlen * config_.output_size, config_.hidden_type); + } +} + +void Linear::forward(int qlen, const void* input, void* output, Backend* backend) { + if (qlen <= 0) { + return; + } + int forward_len = std::min(qlen, config_.group_max_len); + forward_many(forward_len, input, output, backend); + forward(qlen - forward_len, input + forward_len * config_.input_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + forward_len * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); } \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/linear.h b/ktransformers/ktransformers_ext/operators/llamafile/linear.h index 4285551..fd856f9 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/linear.h +++ b/ktransformers/ktransformers_ext/operators/llamafile/linear.h @@ -3,7 +3,7 @@ * @Author : chenht2022 * @Date : 2024-07-12 10:07:58 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:35:00 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -22,34 +22,38 @@ #include "llama.cpp/ggml-quants.h" #include "llama.cpp/ggml.h" #include "llamafile/sgemm.h" +#include "shared_mem_buffer.h" struct LinearConfig { int input_size; int output_size; int stride; + int group_max_len; void* proj; ggml_type proj_type; ggml_type hidden_type; LinearConfig() {} - LinearConfig(int input_size, int output_size, int stride, void* proj, ggml_type proj_type, ggml_type hidden_type) - : input_size(input_size), output_size(output_size), stride(stride), proj(proj), proj_type(proj_type), hidden_type(hidden_type) {} + LinearConfig(int input_size, int output_size, int stride, int group_max_len, void* proj, ggml_type proj_type, ggml_type hidden_type) + : input_size(input_size), output_size(output_size), stride(stride), group_max_len(group_max_len), proj(proj), proj_type(proj_type), hidden_type(hidden_type) {} }; class Linear { public: Linear(LinearConfig); + ~Linear(); void warm_up(Backend* backend); - void forward(const void* input, void* output, Backend* backend); + void forward_many(int qlen, const void* input, void* output, Backend* backend); + void forward(int qlen, const void* input, void* output, Backend* backend); private: LinearConfig config_; void* proj_; // [output_size * input_size ( /32 if quantized)] - std::vector input_fp32_; // [input_size] - std::vector proj_input_; // [input_size * 4] - std::vector proj_output_; // [output_size] + float* input_fp32_; // [group_max_len * input_size] + uint8_t* proj_input_; // [group_max_len * input_size * ggml_type_size(ggml_internal_get_type_traits(proj_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(proj_type).vec_dot_type)] + float* proj_output_; // [group_max_len * output_size] }; #endif \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp b/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp index 632c210..8ef092f 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp @@ -3,7 +3,7 @@ * @Author : chenht2022 * @Date : 2024-07-16 10:43:18 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:35:04 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -15,14 +15,20 @@ MLP::MLP(MLPConfig config) { up_proj_ = config_.up_proj; down_proj_ = config_.down_proj; - input_fp32_.resize(config_.hidden_size); - gate_input_.resize(config_.hidden_size * 4); - up_input_.resize(config_.hidden_size * 4); - gate_output_.resize(config_.intermediate_size); - up_output_.resize(config_.intermediate_size); - intermediate_fp32_.resize(config_.intermediate_size); - down_input_.resize(config_.intermediate_size * 4); - down_output_.resize(config_.hidden_size); + std::vector> mem_requests; + mem_requests.push_back({(void**)&input_fp32_, sizeof(float) * config_.group_max_len * config_.hidden_size}); + mem_requests.push_back({(void**)&gate_input_, config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type)}); + mem_requests.push_back({(void**)&up_input_, config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type)}); + mem_requests.push_back({(void**)&gate_output_, sizeof(float) * config_.group_max_len * config_.intermediate_size}); + mem_requests.push_back({(void**)&up_output_, sizeof(float) * config_.group_max_len * config_.intermediate_size}); + mem_requests.push_back({(void**)&intermediate_fp32_, sizeof(float) * config_.group_max_len * config_.intermediate_size}); + mem_requests.push_back({(void**)&down_input_, config_.group_max_len * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type)}); + mem_requests.push_back({(void**)&down_output_, sizeof(float) * config_.group_max_len * config_.hidden_size}); + shared_mem_buffer.alloc(this, mem_requests); +} + +MLP::~MLP() { + shared_mem_buffer.dealloc(this); } void MLP::warm_up(Backend* backend) { @@ -33,33 +39,33 @@ void MLP::warm_up(Backend* backend) { input_fp32[i] = 0; } from_float(input_fp32.data(), input.data(), config_.hidden_size, config_.hidden_type); - forward(input.data(), output.data(), backend); + forward_many(1, input.data(), output.data(), backend); } static float act_fn(float x) { return x / (1.0f + expf(-x)); } -void MLP::forward(const void* input, void* output, Backend* backend) { +void MLP::forward_many(int qlen, const void* input, void* output, Backend* backend) { const void* gate_input_ptr; const void* up_input_ptr; if (config_.hidden_type == ggml_internal_get_type_traits(config_.gate_type).vec_dot_type && config_.hidden_type == ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { gate_input_ptr = up_input_ptr = input; } else { - to_float(input, input_fp32_.data(), config_.hidden_size, config_.hidden_type); + to_float(input, input_fp32_, qlen * config_.hidden_size, config_.hidden_type); if (ggml_internal_get_type_traits(config_.gate_type).vec_dot_type == ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { - from_float(input_fp32_.data(), gate_input_.data(), config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - gate_input_ptr = up_input_ptr = gate_input_.data(); + from_float(input_fp32_, gate_input_, qlen * config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); + gate_input_ptr = up_input_ptr = gate_input_; } else { if (config_.hidden_type != ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) { - from_float(input_fp32_.data(), gate_input_.data(), config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - gate_input_ptr = gate_input_.data(); + from_float(input_fp32_, gate_input_, qlen * config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); + gate_input_ptr = gate_input_; } else { gate_input_ptr = input; } if (config_.hidden_type != ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { - from_float(input_fp32_.data(), up_input_.data(), config_.hidden_size, ggml_internal_get_type_traits(config_.up_type).vec_dot_type); - up_input_ptr = up_input_.data(); + from_float(input_fp32_, up_input_, qlen * config_.hidden_size, ggml_internal_get_type_traits(config_.up_type).vec_dot_type); + up_input_ptr = up_input_; } else { up_input_ptr = input; } @@ -69,35 +75,49 @@ void MLP::forward(const void* input, void* output, Backend* backend) { backend->do_work_stealing_job(nth, [&](int task_id) { int ith = task_id; void* gate_proj_ptr = gate_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); - float* gate_output_ptr = gate_output_.data() + ith * config_.stride; - llamafile_sgemm(config_.stride, 1, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_proj_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_input_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.gate_type, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); + float* gate_output_ptr = gate_output_ + ith * config_.stride; + llamafile_sgemm(config_.stride, qlen, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_proj_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_input_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.gate_type, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); void* up_proj_ptr = up_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); - float* up_output_ptr = up_output_.data() + ith * config_.stride; - llamafile_sgemm(config_.stride, 1, config_.hidden_size / ggml_blck_size(config_.up_type), up_proj_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_input_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.up_type, ggml_internal_get_type_traits(config_.up_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); - for (int i = ith * config_.stride; i < (ith + 1) * config_.stride; i++) { - intermediate_fp32_[i] = act_fn(gate_output_[i]) * up_output_[i]; - } - if (config_.stride % ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) == 0) { - float* intermediate_fp32_ptr = intermediate_fp32_.data() + ith * config_.stride; - void* down_input_ptr = down_input_.data() + ith * config_.stride * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); - from_float(intermediate_fp32_ptr, down_input_ptr, config_.stride, ggml_internal_get_type_traits(config_.down_type).vec_dot_type); + float* up_output_ptr = up_output_ + ith * config_.stride; + llamafile_sgemm(config_.stride, qlen, config_.hidden_size / ggml_blck_size(config_.up_type), up_proj_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_input_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.up_type, ggml_internal_get_type_traits(config_.up_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); + for (int i = 0; i < qlen; i++) { + for (int j = ith * config_.stride; j < (ith + 1) * config_.stride; j++) { + intermediate_fp32_[i * config_.intermediate_size + j] = act_fn(gate_output_[i * config_.intermediate_size + j]) * up_output_[i * config_.intermediate_size + j]; + } + if (config_.stride % ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) == 0) { + float* intermediate_fp32_ptr = intermediate_fp32_ + i * config_.intermediate_size + ith * config_.stride; + void* down_input_ptr = down_input_ + i * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) + ith * config_.stride * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); + from_float(intermediate_fp32_ptr, down_input_ptr, config_.stride, ggml_internal_get_type_traits(config_.down_type).vec_dot_type); + } } }); if (config_.stride % ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) != 0) { - from_float(intermediate_fp32_.data(), down_input_.data(), config_.intermediate_size, ggml_internal_get_type_traits(config_.down_type).vec_dot_type); + from_float(intermediate_fp32_, down_input_, qlen * config_.intermediate_size, ggml_internal_get_type_traits(config_.down_type).vec_dot_type); } nth = config_.hidden_size / config_.stride; backend->do_work_stealing_job(nth, [&](int task_id) { int ith = task_id; void* down_proj_ptr = down_proj_ + ith * config_.stride * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); - float* down_output_ptr = down_output_.data() + ith * config_.stride; - llamafile_sgemm(config_.stride, 1, config_.intermediate_size / ggml_blck_size(config_.down_type), down_proj_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), down_input_.data(), config_.intermediate_size / ggml_blck_size(config_.down_type), down_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.down_type, ggml_internal_get_type_traits(config_.down_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); + float* down_output_ptr = down_output_ + ith * config_.stride; + llamafile_sgemm(config_.stride, qlen, config_.intermediate_size / ggml_blck_size(config_.down_type), down_proj_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), down_input_, config_.intermediate_size / ggml_blck_size(config_.down_type), down_output_ptr, config_.hidden_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.down_type, ggml_internal_get_type_traits(config_.down_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); if (config_.stride % ggml_blck_size(config_.hidden_type) == 0) { - void* output_ptr = output + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); - from_float(down_output_ptr, output_ptr, config_.stride, config_.hidden_type); + for (int i = 0; i < qlen; i++) { + float* output_fp32_ptr = down_output_ + i * config_.hidden_size + ith * config_.stride; + void* output_ptr = output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + from_float(output_fp32_ptr, output_ptr, config_.stride, config_.hidden_type); + } } }); if (config_.stride % ggml_blck_size(config_.hidden_type) != 0) { - from_float(down_output_.data(), output, config_.hidden_size, config_.hidden_type); + from_float(down_output_, output, qlen * config_.hidden_size, config_.hidden_type); } } + +void MLP::forward(int qlen, const void* input, void* output, Backend* backend) { + if (qlen <= 0) { + return; + } + int forward_len = std::min(qlen, config_.group_max_len); + forward_many(forward_len, input, output, backend); + forward(qlen - forward_len, input + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); +} \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/mlp.h b/ktransformers/ktransformers_ext/operators/llamafile/mlp.h index 604db77..eb93294 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/mlp.h +++ b/ktransformers/ktransformers_ext/operators/llamafile/mlp.h @@ -3,7 +3,7 @@ * @Author : chenht2022 * @Date : 2024-07-12 10:07:58 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:35:06 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -22,11 +22,13 @@ #include "llama.cpp/ggml-quants.h" #include "llama.cpp/ggml.h" #include "llamafile/sgemm.h" +#include "shared_mem_buffer.h" struct MLPConfig { int hidden_size; int intermediate_size; int stride; + int group_max_len; void* gate_proj; void* up_proj; void* down_proj; @@ -37,15 +39,17 @@ struct MLPConfig { MLPConfig() {} - MLPConfig(int hidden_size, int intermediate_size, int stride, void* gate_proj, void* up_proj, void* down_proj, ggml_type gate_type, ggml_type up_type, ggml_type down_type, ggml_type hidden_type) - : hidden_size(hidden_size), intermediate_size(intermediate_size), stride(stride), gate_proj(gate_proj), up_proj(up_proj), down_proj(down_proj), gate_type(gate_type), up_type(up_type), down_type(down_type), hidden_type(hidden_type) {} + MLPConfig(int hidden_size, int intermediate_size, int stride, int group_max_len, void* gate_proj, void* up_proj, void* down_proj, ggml_type gate_type, ggml_type up_type, ggml_type down_type, ggml_type hidden_type) + : hidden_size(hidden_size), intermediate_size(intermediate_size), stride(stride), group_max_len(group_max_len), gate_proj(gate_proj), up_proj(up_proj), down_proj(down_proj), gate_type(gate_type), up_type(up_type), down_type(down_type), hidden_type(hidden_type) {} }; class MLP { public: MLP(MLPConfig); + ~MLP(); void warm_up(Backend* backend); - void forward(const void* input, void* output, Backend* backend); + void forward_many(int qlen, const void* input, void* output, Backend* backend); + void forward(int qlen, const void* input, void* output, Backend* backend); private: MLPConfig config_; @@ -53,14 +57,14 @@ class MLP { void* up_proj_; // [intermediate_size * hidden_size ( /32 if quantized)] void* down_proj_; // [hidden_size * intermediate_size ( /32 if quantized)] - std::vector input_fp32_; // [hidden_size] - std::vector gate_input_; // [hidden_size * 4] - std::vector up_input_; // [hidden_size * 4] - std::vector gate_output_; // [intermediate_size] - std::vector up_output_; // [intermediate_size] - std::vector intermediate_fp32_; // [intermediate_size] - std::vector down_input_; // [intermediate_size * 4] - std::vector down_output_; // [hidden_size] + float* input_fp32_; // [group_max_len * hidden_size] + uint8_t* gate_input_; // [group_max_len * hidden_size * ggml_type_size(ggml_internal_get_type_traits(gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(gate_type).vec_dot_type)] + uint8_t* up_input_; // [group_max_len * hidden_size * ggml_type_size(ggml_internal_get_type_traits(up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(up_type).vec_dot_type)] + float* gate_output_; // [group_max_len * intermediate_size] + float* up_output_; // [group_max_len * intermediate_size] + float* intermediate_fp32_; // [group_max_len * intermediate_size] + uint8_t* down_input_; // [group_max_len * intermediate_size * ggml_type_size(ggml_internal_get_type_traits(down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(down_type).vec_dot_type)] + float* down_output_; // [group_max_len * hidden_size] }; #endif \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp b/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp index aaea4a7..8010f54 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp @@ -1,97 +1,62 @@ /** - * @Description : + * @Description : * @Author : chenht2022 * @Date : 2024-07-22 02:03:22 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:35:07 - * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. -**/ + * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. + **/ #include "moe.h" #include #include "unistd.h" -void* MOE::buffer_ = nullptr; - MOE::MOE(MOEConfig config) { config_ = config; gate_proj_ = config_.gate_proj; up_proj_ = config_.up_proj; down_proj_ = config_.down_proj; - if (MOE::buffer_ == nullptr) { - uint64_t buffer_size = 0; - buffer_size += sizeof(float) * config_.group_max_len * config_.hidden_size; - buffer_size += config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - buffer_size += config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type); - buffer_size += config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - buffer_size += config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type); - buffer_size += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - buffer_size += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - buffer_size += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - buffer_size += config_.routed_expert_num * config_.group_max_len * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); - buffer_size += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.hidden_size; - buffer_size += sizeof(float) * config_.group_max_len * config_.hidden_size; - buffer_ = malloc(buffer_size); - } - - uint64_t offset = 0; - s_input_fp32_ = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.hidden_size; - s_gate_input_ = (uint8_t*)(buffer_ + offset); - offset += config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - s_up_input_ = (uint8_t*)(buffer_ + offset); - offset += config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type); + std::vector> s_mem_requests; + s_mem_requests.push_back({(void**)&s_input_fp32_, sizeof(float) * config_.hidden_size}); + s_mem_requests.push_back({(void**)&s_gate_input_, config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type)}); + s_mem_requests.push_back({(void**)&s_up_input_, config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type)}); s_gate_output_.resize(config_.routed_expert_num); s_up_output_.resize(config_.routed_expert_num); s_intermediate_fp32_.resize(config_.routed_expert_num); s_down_input_.resize(config_.routed_expert_num); s_down_output_.resize(config_.routed_expert_num); for (int i = 0; i < config_.routed_expert_num; i++) { - s_gate_output_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.intermediate_size; - s_up_output_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.intermediate_size; - s_intermediate_fp32_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.intermediate_size; - s_down_input_[i] = (uint8_t*)(buffer_ + offset); - offset += config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); - s_down_output_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.hidden_size; + s_mem_requests.push_back({(void**)&s_gate_output_[i], sizeof(float) * config_.intermediate_size}); + s_mem_requests.push_back({(void**)&s_up_output_[i], sizeof(float) * config_.intermediate_size}); + s_mem_requests.push_back({(void**)&s_intermediate_fp32_[i], sizeof(float) * config_.intermediate_size}); + s_mem_requests.push_back({(void**)&s_down_input_[i], config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type)}); + s_mem_requests.push_back({(void**)&s_down_output_[i], sizeof(float) * config_.hidden_size}); } - s_output_fp32_ = (float*)(buffer_ + offset); + s_mem_requests.push_back({(void**)&s_output_fp32_, sizeof(float) * config_.hidden_size}); + shared_mem_buffer.alloc(this, s_mem_requests); - offset = 0; + std::vector> m_mem_requests; m_input_fp32_.resize(config_.group_max_len); m_gate_input_.resize(config_.group_max_len); m_up_input_.resize(config_.group_max_len); for (int i = 0; i < config_.group_max_len; i++) { - m_input_fp32_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.hidden_size; - m_gate_input_[i] = (uint8_t*)(buffer_ + offset); - offset += config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - m_up_input_[i] = (uint8_t*)(buffer_ + offset); - offset += config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type); + m_mem_requests.push_back({(void**)&m_input_fp32_[i], sizeof(float) * config_.hidden_size}); + m_mem_requests.push_back({(void**)&m_gate_input_[i], config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type)}); + m_mem_requests.push_back({(void**)&m_up_input_[i], config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type)}); } - m_local_gate_input_ = (uint8_t*)(buffer_ + offset); - offset += config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); - m_local_up_input_ = (uint8_t*)(buffer_ + offset); - offset += config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type); - m_local_gate_output_ = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - m_local_up_output_ = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - m_local_intermediate_fp32_ = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size; - m_local_down_input_ = (uint8_t*)(buffer_ + offset); - offset += config_.routed_expert_num * config_.group_max_len * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); - m_local_down_output_ = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.hidden_size; + m_mem_requests.push_back({(void**)&m_local_gate_input_, config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.gate_type).vec_dot_type)}); + m_mem_requests.push_back({(void**)&m_local_up_input_, config_.routed_expert_num * config_.group_max_len * config_.hidden_size * ggml_type_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.up_type).vec_dot_type)}); + m_mem_requests.push_back({(void**)&m_local_gate_output_, sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size}); + m_mem_requests.push_back({(void**)&m_local_up_output_, sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size}); + m_mem_requests.push_back({(void**)&m_local_intermediate_fp32_, sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.intermediate_size}); + m_mem_requests.push_back({(void**)&m_local_down_input_, config_.routed_expert_num * config_.group_max_len * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type)}); + m_mem_requests.push_back({(void**)&m_local_down_output_, sizeof(float) * config_.routed_expert_num * config_.group_max_len * config_.hidden_size}); m_output_fp32_.resize(config_.group_max_len); for (int i = 0; i < config_.group_max_len; i++) { - m_output_fp32_[i] = (float*)(buffer_ + offset); - offset += sizeof(float) * config_.hidden_size; + m_mem_requests.push_back({(void**)&m_output_fp32_[i], sizeof(float) * config_.hidden_size}); } + shared_mem_buffer.alloc(this, m_mem_requests); m_local_pos_.resize(config_.group_max_len); for (int i = 0; i < config_.group_max_len; i++) { @@ -107,6 +72,10 @@ MOE::MOE(MOEConfig config) { m_local_down_output_ptr_.resize(config_.expert_num); } +MOE::~MOE() { + shared_mem_buffer.dealloc(this); +} + void MOE::warm_up(Backend* backend) { std::vector input_fp32(config_.hidden_size); std::vector input(config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type)); diff --git a/ktransformers/ktransformers_ext/operators/llamafile/moe.h b/ktransformers/ktransformers_ext/operators/llamafile/moe.h index 0d279fe..a1470aa 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/moe.h +++ b/ktransformers/ktransformers_ext/operators/llamafile/moe.h @@ -3,7 +3,7 @@ * @Author : chenht2022 * @Date : 2024-07-22 02:03:22 * @Version : 1.0.0 - * @LastEditors : chenht2022 + * @LastEditors : chenht2022 * @LastEditTime : 2024-07-25 10:35:10 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -22,6 +22,7 @@ #include "llama.cpp/ggml-quants.h" #include "llama.cpp/ggml.h" #include "llamafile/sgemm.h" +#include "shared_mem_buffer.h" struct MOEConfig { int expert_num; @@ -48,13 +49,13 @@ struct MOEConfig { class MOE { public: MOE(MOEConfig); + ~MOE(); void warm_up(Backend* backend); void forward_one(int k, const uint64_t* expert_ids, const float* weights, const void* input, void* output, Backend* backend); void forward_many(int qlen, int k, const uint64_t* expert_ids, const float* weights, const void* input, void* output, Backend* backend); void forward(int qlen, int k, const uint64_t* expert_ids, const float* weights, const void* input, void* output, Backend* backend); private: - static void* buffer_; MOEConfig config_; void* gate_proj_; // [expert_num * intermediate_size * hidden_size ( /32 if quantized)] void* up_proj_; // [expert_num * intermediate_size * hidden_size ( /32 if quantized)] diff --git a/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp new file mode 100644 index 0000000..b1599da --- /dev/null +++ b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp @@ -0,0 +1,55 @@ +/** + * @Description : + * @Author : chenht2022 + * @Date : 2024-08-05 04:49:08 + * @Version : 1.0.0 + * @LastEditors : chenht2022 + * @LastEditTime : 2024-08-05 09:21:29 + * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. + **/ +#include "shared_mem_buffer.h" +#include + +SharedMemBuffer::SharedMemBuffer() { + buffer_ = nullptr; + size_ = 0; +} + +SharedMemBuffer::~SharedMemBuffer() { + if (buffer_) { + free(buffer_); + } +} + +void SharedMemBuffer::alloc(void* object, std::vector> requests) { + uint64_t size = 0; + for (auto& request : requests) { + size += request.second; + } + if (size > size_) { + if (buffer_) { + free(buffer_); + } + buffer_ = malloc(size); + size_ = size; + for (auto& obj_requests : hist_requests_) { + for (auto& requests : obj_requests.second) { + arrange(requests); + } + } + } + arrange(requests); + hist_requests_[object].push_back(requests); +} + +void SharedMemBuffer::dealloc(void* object) { + hist_requests_.erase(object); +} + +void SharedMemBuffer::arrange(std::vector> requests) { + uint64_t offset = 0; + for (auto& request : requests) { + *(request.first) = buffer_ + offset; + offset += request.second; + } +} diff --git a/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.h b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.h new file mode 100644 index 0000000..eeaccd4 --- /dev/null +++ b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.h @@ -0,0 +1,37 @@ +/** + * @Description : + * @Author : chenht2022 + * @Date : 2024-08-05 04:49:08 + * @Version : 1.0.0 + * @LastEditors : chenht2022 + * @LastEditTime : 2024-08-05 06:36:41 + * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. + **/ + +#ifndef CPUINFER_SHAREDMEMBUFFER_H +#define CPUINFER_SHAREDMEMBUFFER_H + +#include +#include +#include +#include + +class SharedMemBuffer { + public: + SharedMemBuffer(); + ~SharedMemBuffer(); + + void alloc(void* object, std::vector> requests); + void dealloc(void* object); + + private: + void* buffer_; + uint64_t size_; + std::map>>> hist_requests_; + + void arrange(std::vector> requests); +}; + +static SharedMemBuffer shared_mem_buffer; + +#endif \ No newline at end of file diff --git a/ktransformers/operators/experts.py b/ktransformers/operators/experts.py index 6adb657..0369f5f 100644 --- a/ktransformers/operators/experts.py +++ b/ktransformers/operators/experts.py @@ -155,7 +155,7 @@ def load(self, w: dict | nn.Parameter | tuple | None = None, device:str|None = N self.moe = MOE(moe_config) self.cpu_infer = MLPCPUExperts.CPU_INFER if warmup: - self.cpu_infer.submit(self.moe.warm_up) + self.cpu_infer.submit(self.moe.warm_up()) self.cpu_infer.sync() if MLPCPUExperts.output_gpu == None: MLPCPUExperts.input_tensor_cpu = torch.empty((self.config.hidden_size), device="cpu", pin_memory=True) @@ -168,7 +168,7 @@ def submit_for_one_decode(self, input_tensor, expert_ids, weights): MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) - self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward, 1, expert_ids.size(0), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr()) + self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(0), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) def sync_for_one_decode(self): self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) @@ -183,7 +183,7 @@ def forward(self, input_tensor, expert_ids, weights): MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) - self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward, 1, expert_ids.size(1), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr()) + self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(1), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) MLPCPUExperts.output_gpu.copy_(MLPCPUExperts.output_cpu, non_blocking=True) #print("capturing experts finish") @@ -193,7 +193,7 @@ def forward(self, input_tensor, expert_ids, weights): expert_ids = expert_ids.contiguous().cpu() weights = weights.contiguous().to(torch.float32).cpu() output = torch.empty_like(input_tensor).contiguous() - self.cpu_infer.submit(self.moe.forward, expert_ids.size(0), expert_ids.size(1), expert_ids.data_ptr(), weights.data_ptr(), input_tensor.data_ptr(), output.data_ptr()) + self.cpu_infer.submit(self.moe.forward(expert_ids.size(0), expert_ids.size(1), expert_ids.data_ptr(), weights.data_ptr(), input_tensor.data_ptr(), output.data_ptr())) self.cpu_infer.sync() return output.to(device=object.__getattribute__(self, "device")) From f5f79f5c0ec07ffd0d1b039534c45b8f6b0d939f Mon Sep 17 00:00:00 2001 From: chenxl Date: Mon, 12 Aug 2024 11:17:29 +0000 Subject: [PATCH 2/8] [ADD] support multi-gpu qlen>1 q5_k --- .gitignore | 5 +- install.bat | 16 + install.sh | 2 +- .../ktransformers_ext/CMakeLists.txt | 21 +- .../cpu_backend/task_queue.h | 44 +- .../ktransformers_ext/cuda/binding.cpp | 6 +- .../cuda/custom_gguf/binding.cpp | 3 + .../cuda/custom_gguf/custom_ggml.h | 39 - .../cuda/custom_gguf/dequant.cu | 56 +- .../ktransformers_ext/cuda/custom_gguf/ops.h | 5 +- .../cuda/gptq_marlin/gptq_marlin.cu | 77 +- ktransformers/ktransformers_ext/cuda/setup.py | 28 +- .../ktransformers_ext/ext_bindings.cpp | 24 +- .../operators/custom_marlin/quantize/gptq.py | 206 -- .../custom_marlin/quantize/gptq_marlin.py | 458 ----- .../custom_marlin/quantize/quantizer.py | 140 -- .../custom_marlin/quantize/repack.py | 99 - .../quantize/utils/marlin_utils.py | 4 +- .../operators/llamafile/linear.cpp | 6 +- .../operators/llamafile/mlp.cpp | 12 +- .../operators/llamafile/moe.cpp | 32 +- .../operators/llamafile/shared_mem_buffer.cpp | 2 +- ktransformers/local_chat.py | 13 +- ktransformers/models/custom_cache.py | 16 +- ktransformers/models/modeling_deepseek.py | 4 +- ktransformers/models/modeling_mixtral.py | 1735 +++++++++++++++++ ktransformers/operators/RoPE.py | 19 +- ktransformers/operators/experts.py | 292 ++- ktransformers/operators/layer_wise_prefill.py | 74 +- ktransformers/operators/linear.py | 12 +- ktransformers/optimize/optimize.py | 53 +- .../DeepSeek-V2-Chat-multi-gpu-4.yaml | 228 +++ .../DeepSeek-V2-Chat-multi-gpu.yaml | 126 ++ .../optimize_rules/DeepSeek-V2-Chat.yaml | 19 +- .../DeepSeek-V2-Lite-Chat-multi-gpu.yaml | 126 ++ .../optimize/optimize_rules/Mixtral.yaml | 45 + .../Qwen2-57B-A14B-Instruct-multi-gpu.yaml | 111 ++ .../Qwen2-57B-A14B-Instruct.yaml | 18 +- ktransformers/tests/dequant_gpu.py | 33 +- ktransformers/tests/dequant_gpu_t.py | 2 +- ktransformers/util/cuda_graph_runner.py | 18 +- ktransformers/util/custom_gguf.py | 143 +- ktransformers/util/utils.py | 90 +- pyproject.toml | 3 +- setup.py | 17 +- third_party/llamafile/iqk_mul_mat.inc | 7 +- .../llamafile/iqk_mul_mat_amd_avx2.cpp | 2 +- .../llamafile/iqk_mul_mat_amd_zen4.cpp | 2 +- third_party/llamafile/sgemm.cpp | 19 +- third_party/llamafile/tinyblas_cpu.h | 2 +- .../llamafile/tinyblas_cpu_mixmul_amd_avx.cpp | 2 +- .../tinyblas_cpu_mixmul_amd_avx2.cpp | 2 +- .../tinyblas_cpu_mixmul_amd_avx512f.cpp | 2 +- .../tinyblas_cpu_mixmul_amd_avxvnni.cpp | 2 +- .../llamafile/tinyblas_cpu_mixmul_amd_fma.cpp | 2 +- .../tinyblas_cpu_mixmul_amd_zen4.cpp | 2 +- third_party/llamafile/tinyblas_cpu_sgemm.inc | 4 +- .../llamafile/tinyblas_cpu_sgemm_amd_avx.cpp | 2 +- .../llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp | 2 +- .../tinyblas_cpu_sgemm_amd_avx512f.cpp | 2 +- .../tinyblas_cpu_sgemm_amd_avxvnni.cpp | 2 +- .../llamafile/tinyblas_cpu_sgemm_amd_fma.cpp | 2 +- .../llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp | 2 +- 63 files changed, 3264 insertions(+), 1278 deletions(-) create mode 100644 install.bat delete mode 100644 ktransformers/ktransformers_ext/cuda/custom_gguf/custom_ggml.h delete mode 100644 ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq.py delete mode 100644 ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq_marlin.py delete mode 100644 ktransformers/ktransformers_ext/operators/custom_marlin/quantize/quantizer.py delete mode 100644 ktransformers/ktransformers_ext/operators/custom_marlin/quantize/repack.py mode change 100644 => 100755 ktransformers/local_chat.py create mode 100644 ktransformers/models/modeling_mixtral.py create mode 100644 ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml create mode 100644 ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml create mode 100644 ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml create mode 100644 ktransformers/optimize/optimize_rules/Mixtral.yaml create mode 100644 ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml diff --git a/.gitignore b/.gitignore index 718ea55..1bb8666 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ node_modules .DS_Store compile_commands.json *.egg-info* -*dist/ \ No newline at end of file +*dist/ +ktransformers/server/local_store/ +ktransformers/server_test1.db +*.patch \ No newline at end of file diff --git a/install.bat b/install.bat new file mode 100644 index 0000000..dc429e4 --- /dev/null +++ b/install.bat @@ -0,0 +1,16 @@ +@echo off + +REM clear build dirs +rmdir /S /Q ktransformers\ktransformers_ext\build +rmdir /S /Q ktransformers\ktransformers_ext\cuda\build +rmdir /S /Q ktransformers\ktransformers_ext\cuda\dist +rmdir /S /Q ktransformers\ktransformers_ext\out +del /F /Q ktransformers\ktransformers_ext\cuda\*.egg-info + +echo Installing python dependencies from requirements.txt +pip install -r requirements-local_chat.txt + +echo Installing ktransformers +set KTRANSFORMERS_FORCE_BUILD=TRUE +pip install . --no-build-isolation +echo Installation completed successfully \ No newline at end of file diff --git a/install.sh b/install.sh index fa5ba18..ffb7aca 100644 --- a/install.sh +++ b/install.sh @@ -11,5 +11,5 @@ echo "Installing python dependencies from requirements.txt" pip install -r requirements-local_chat.txt echo "Installing ktransformers" -pip install . --no-build-isolation +KTRANSFORMERS_FORCE_BUILD=TRUE pip install . --no-build-isolation echo "Installation completed successfully" \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/CMakeLists.txt b/ktransformers/ktransformers_ext/CMakeLists.txt index c3d4f5b..e6e0518 100644 --- a/ktransformers/ktransformers_ext/CMakeLists.txt +++ b/ktransformers/ktransformers_ext/CMakeLists.txt @@ -189,7 +189,13 @@ else() message(STATUS "Unknown architecture") endif() -find_package(CUDA REQUIRED) +# message(STATUS "CUDAToolkit_ROOT:${CUDAToolkit_ROOT}") +# find_package(FindCUDAToolkit REQUIRED) +# if(CUDAToolkit_FOUND) +# message(STATUS "Found CUDA cudart lib at:${CUDAToolkit_LIBRARY_DIR}") +# else() +# message(STATUS "Can't found CUDA lib") +# endif() add_compile_options("$<$:${ARCH_FLAGS}>") add_compile_options("$<$:${ARCH_FLAGS}>") @@ -198,7 +204,12 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/pybind11 ${CMAKE_ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/llama.cpp ${CMAKE_CURRENT_BINARY_DIR}/third_party/llama.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../third_party) -include_directories("${CUDA_INCLUDE_DIRS}") +if (WIN32) + include_directories("$ENV{CUDA_PATH}/include") +elseif (UNIX) + find_package(CUDA REQUIRED) + include_directories("${CUDA_INCLUDE_DIRS}") +endif() aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_DIR1) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/cpu_backend SOURCE_DIR2) @@ -209,4 +220,8 @@ message(STATUS "ALL_SOURCES: ${ALL_SOURCES}") pybind11_add_module(${PROJECT_NAME} MODULE ${ALL_SOURCES}) target_link_libraries(${PROJECT_NAME} PRIVATE llama) -target_link_libraries(${PROJECT_NAME} PRIVATE "/usr/local/cuda/lib64/libcudart.so") \ No newline at end of file +if(WIN32) + target_link_libraries(${PROJECT_NAME} PRIVATE "$ENV{CUDA_PATH}/lib/x64/cudart.lib")#CUDA::cudart +elseif(UNIX) + target_link_libraries(${PROJECT_NAME} PRIVATE "$ENV{CUDA_HOME}/lib64/libcudart.so") +endif() \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/cpu_backend/task_queue.h b/ktransformers/ktransformers_ext/cpu_backend/task_queue.h index b4212fd..d4e6d8a 100644 --- a/ktransformers/ktransformers_ext/cpu_backend/task_queue.h +++ b/ktransformers/ktransformers_ext/cpu_backend/task_queue.h @@ -3,8 +3,8 @@ * @Author : chenht2022 * @Date : 2024-07-16 10:43:18 * @Version : 1.0.0 - * @LastEditors : chenht2022 - * @LastEditTime : 2024-07-25 10:33:47 + * @LastEditors : chenxl + * @LastEditTime : 2024-08-08 04:23:51 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ #ifndef CPUINFER_TASKQUEUE_H @@ -17,6 +17,44 @@ #include #include #include +#ifdef _WIN32 +#include +#endif + +class custom_mutex { +private: +#ifdef _WIN32 + HANDLE global_mutex; +#else + std::mutex global_mutex; +#endif + +public: + custom_mutex() + { +#ifdef _WIN32 + HANDLE global_mutex; +#endif + } + + void lock() + { +#ifdef _WIN32 + WaitForSingleObject(global_mutex, INFINITE); +#else + global_mutex.lock(); +#endif + } + + void unlock() + { +#ifdef _WIN32 + ReleaseMutex(global_mutex); +#else + global_mutex.lock(); +#endif + } +}; class TaskQueue { public: @@ -32,7 +70,7 @@ class TaskQueue { std::queue> tasks; std::thread worker; - std::mutex mutex; + custom_mutex mutex; std::atomic sync_flag; std::atomic exit_flag; }; diff --git a/ktransformers/ktransformers_ext/cuda/binding.cpp b/ktransformers/ktransformers_ext/cuda/binding.cpp index 2d5da68..f17382d 100644 --- a/ktransformers/ktransformers_ext/cuda/binding.cpp +++ b/ktransformers/ktransformers_ext/cuda/binding.cpp @@ -3,8 +3,8 @@ * @Author : Azure-Tang * @Date : 2024-07-25 13:38:30 * @Version : 1.0.0 - * @LastEditors : Azure - * @LastEditTime : 2024-07-26 08:36:03 + * @LastEditors : kkk1nak0 + * @LastEditTime : 2024-08-09 01:45:02 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -23,6 +23,8 @@ PYBIND11_MODULE(KTransformersOps, m) { py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q6_k", &dequantize_q6_k, "Function to dequantize q6_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q5_k", &dequantize_q5_k, "Function to dequantize q5_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q4_k", &dequantize_q4_k, "Function to dequantize q4_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("gptq_marlin_gemm", &gptq_marlin_gemm, "Function to perform GEMM using Marlin quantization.", diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp b/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp index ea52e8f..2cb46fc 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp @@ -12,12 +12,15 @@ int test(){ } torch::Tensor dequantize_q6_k(torch::Tensor data, int blk_size, torch::Device device); +torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device device); PYBIND11_MODULE(cudaops, m) { m.def("dequantize_q8_0", &dequantize_q8_0, "Function to dequantize q8_0 data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q6_k", &dequantize_q6_k, "Function to dequantize q6_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q5_k", &dequantize_q5_k, "Function to dequantize q5_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q4_k", &dequantize_q4_k, "Function to dequantize q4_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("test", &test, "Function to test."); diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/custom_ggml.h b/ktransformers/ktransformers_ext/cuda/custom_gguf/custom_ggml.h deleted file mode 100644 index 333dc69..0000000 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/custom_ggml.h +++ /dev/null @@ -1,39 +0,0 @@ - - - -#include - - -__device__ float ggml_compute_fp16_to_fp32(uint16_t h) { - return __uint2float_rd(h); -} - -static inline float ggml_compute_fp16_to_fp32(uint16_t h) { - uint16_t tmp; - memcpy(&tmp, &h, sizeof(ggml_fp16_t)); - return (float)tmp; -} - -// define the global table for fp16 to fp32 conversion -__device__ float ggml_table_f32_f16[1 << 16]; - -// CUDA Kernel to init the table -__global__ void init_fp16_to_fp32_table() { - int idx = blockIdx.x * blockDim.x + threadIdx.x; - for (auto blk_id = idx; blk_id<(1 << 16); blk_id+=blockDim.x * gridDim.x){ - ggml_table_f32_f16[blk_id] = GGML_COMPUTE_FP16_TO_FP32(blk_id); - } -} - -#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) - -extern __device__ float ggml_table_f32_f16[1 << 16]; // Declare as __device__ if used within device code - -// This version of the function is designed to be called from within a CUDA kernel -#if !defined(GGML_FP16_TO_FP32) -__device__ float ggml_lookup_fp16_to_fp32(uint16_t f) { - return ggml_table_f32_f16[f]; -} - -#define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x) -#endif \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu b/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu index 38f4842..aaa6453 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu @@ -3,8 +3,8 @@ * @Author : Azure-Tang, Boxin Zhang * @Date : 2024-07-25 13:38:30 * @Version : 1.0.0 - * @LastEditors : Azure - * @LastEditTime : 2024-07-26 11:58:50 + * @LastEditors : kkk1nak0 + * @LastEditTime : 2024-08-09 07:57:06 * Adapted from https://github.com/ggerganov/ggml/blob/fca1caafea7de9fbd7efc733b9818f9cf2da3050/src/ggml-quants.c * Copyright (c) 2023-2024 The ggml authors * Copyright (c) 2024 by KVCache.AI, All Rights Reserved. @@ -14,6 +14,7 @@ #include #include #include +#include __global__ void dequantize_q8_0_kernel(float* output, const float* scales, const int8_t* qs, int num_blocks, int blk_size) { int global_idx = blockIdx.x * blockDim.x + threadIdx.x; @@ -59,6 +60,35 @@ __global__ void dequantize_q4_k_kernel(int8_t* data, float* output, int blk_size } } +__global__ void dequantize_q5_k_kernel(int8_t* data, float* output, int blk_size, int num_blocks) { + int global_idx = blockIdx.x * blockDim.x + threadIdx.x; + for (auto block_id=global_idx; block_id(data + block_id * blk_size + 0))); + const float min = __half2float(*(reinterpret_cast(data + block_id * blk_size + 2))); + + const uint8_t * __restrict__ qh = (uint8_t*)(data + block_id * blk_size + 16); + const uint8_t * __restrict__ ql = (uint8_t*)(data + block_id * blk_size + 48); + + int is = 0; + uint8_t sc, m; + uint8_t u1 = 1, u2 = 2; + uint8_t* scales = (uint8_t*)(data + block_id * blk_size + 4); + + for (int j = 0; j < 256; j += 64) { + get_scale_min_k4(is + 0, scales, &sc, &m); + const float d1 = d * sc; const float m1 = min * m; + get_scale_min_k4(is + 1, scales, &sc, &m); + const float d2 = d * sc; const float m2 = min * m; + for (int l = 0; l < 32; ++l) *output_blk++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1; + for (int l = 0; l < 32; ++l) *output_blk++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2; + ql += 32; is += 2; + u1 <<= 2; u2 <<= 2; + } + } +} + __global__ void dequantize_q6_k_kernel(int8_t* data, float* output, int blk_size, int num_blocks) { int global_idx = blockIdx.x * blockDim.x + threadIdx.x; for (auto block_id=global_idx; block_id>>(data_gpu.data_ptr(), output.data_ptr(), blk_size, num_blocks); + + cudaDeviceSynchronize(); + return output; +} \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h b/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h index 9af8f30..f5fde87 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h @@ -3,8 +3,8 @@ * @Author : Azure-Tang * @Date : 2024-07-22 09:27:55 * @Version : 1.0.0 - * @LastEditors : Azure - * @LastEditTime : 2024-07-26 08:38:20 + * @LastEditors : kkk1nak0 + * @LastEditTime : 2024-08-09 01:44:21 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ #pragma once @@ -15,4 +15,5 @@ torch::Tensor dequantize_q8_0(torch::Tensor data, int blk_size, torch::Device device); torch::Tensor dequantize_q6_k(torch::Tensor data, int blk_size, torch::Device device); +torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device device); torch::Tensor dequantize_q4_k(torch::Tensor data, int blk_size, torch::Device device); \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/cuda/gptq_marlin/gptq_marlin.cu b/ktransformers/ktransformers_ext/cuda/gptq_marlin/gptq_marlin.cu index e8e5153..54e538a 100644 --- a/ktransformers/ktransformers_ext/cuda/gptq_marlin/gptq_marlin.cu +++ b/ktransformers/ktransformers_ext/cuda/gptq_marlin/gptq_marlin.cu @@ -23,7 +23,7 @@ */ #include "gptq_marlin.cuh" #include "gptq_marlin_dtypes.cuh" - +#include #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ static_assert(std::is_same::value || \ std::is_same::value, \ @@ -1703,28 +1703,63 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, thread_m_blocks = exec_cfg.max_m_blocks; } + + // Define kernel configurations - if (false) { +#define undefined_error TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " + \ + str(prob_n) + ", " + str(prob_k) + "]" + \ + ", has_act_order = " + str(has_act_order) + \ + ", num_groups = " + str(num_groups) + \ + ", group_size = " + str(group_size) + \ + ", thread_m_blocks = " + str(thread_m_blocks) + \ + ", thread_n_blocks = " + str(thread_n_blocks) + \ + ", thread_k_blocks = " + str(thread_k_blocks)); + + + if (num_bits == 4 && num_threads == 256) + { + if (false) { + } + CALL_IF(4, 32, 2, 256) + CALL_IF(4, 16, 4, 256) + CALL_IF(4, 8, 8, 256) + else { + undefined_error + } + } + else if (num_bits == 4 && num_threads == 128) + { + if (false) { + } + CALL_IF(4, 8, 4, 128) + CALL_IF(4, 4, 8, 128) + else { + undefined_error + } + } + else if (num_bits == 8 && num_threads == 256) + { + if (false) { + } + CALL_IF(8, 32, 2, 256) + CALL_IF(8, 16, 4, 256) + CALL_IF(8, 8, 8, 256) + else { + undefined_error + } + } + else if (num_bits == 8 && num_threads == 128) + { + if (false) { + } + CALL_IF(8, 8, 4, 128) + CALL_IF(8, 4, 8, 128) + else { + undefined_error + } } - CALL_IF(4, 32, 2, 256) - CALL_IF(4, 16, 4, 256) - CALL_IF(4, 8, 8, 256) - CALL_IF(4, 8, 4, 128) - CALL_IF(4, 4, 8, 128) - CALL_IF(8, 32, 2, 256) - CALL_IF(8, 16, 4, 256) - CALL_IF(8, 8, 8, 256) - CALL_IF(8, 8, 4, 128) - CALL_IF(8, 4, 8, 128) else { - TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " + - str(prob_n) + ", " + str(prob_k) + "]" + - ", has_act_order = " + str(has_act_order) + - ", num_groups = " + str(num_groups) + - ", group_size = " + str(group_size) + - ", thread_m_blocks = " + str(thread_m_blocks) + - ", thread_n_blocks = " + str(thread_n_blocks) + - ", thread_k_blocks = " + str(thread_k_blocks)); + undefined_error } A_ptr += 16 * thread_m_blocks * (prob_k / 8) * par; @@ -1739,6 +1774,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight, torch::Tensor& perm, torch::Tensor& workspace, int64_t num_bits, int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); // Verify num_bits TORCH_CHECK(num_bits == 4 || num_bits == 8, "num_bits must be 4 or 8. Got = ", num_bits); @@ -1781,7 +1817,6 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight, TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); // Alloc buffers - const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); torch::Tensor c = torch::empty({size_m, size_n}, options); torch::Tensor a_tmp = torch::empty({size_m, size_k}, options); diff --git a/ktransformers/ktransformers_ext/cuda/setup.py b/ktransformers/ktransformers_ext/cuda/setup.py index baf0808..156bb0e 100644 --- a/ktransformers/ktransformers_ext/cuda/setup.py +++ b/ktransformers/ktransformers_ext/cuda/setup.py @@ -2,17 +2,25 @@ from setuptools import setup, Extension from torch.utils import cpp_extension from torch.utils.cpp_extension import BuildExtension, CUDAExtension - -# setup marlin gemm -setup(name='KTransformersOps', - ext_modules=[ - CUDAExtension('KTransformersOps', [ +setup( + name='KTransformersOps', + ext_modules=[ + CUDAExtension( + 'KTransformersOps', [ 'custom_gguf/dequant.cu', 'binding.cpp', 'gptq_marlin/gptq_marlin.cu', # 'gptq_marlin_repack.cu', - ]) - ], - cmdclass={'build_ext': BuildExtension -}) - + ], + extra_compile_args={ + 'cxx': ['-O3'], + 'nvcc': [ + '-O3', + '--use_fast_math', + '-Xcompiler', '-fPIC', + ] + }, + ) + ], + cmdclass={'build_ext': BuildExtension} +) \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/ext_bindings.cpp b/ktransformers/ktransformers_ext/ext_bindings.cpp index ef30037..c220a9b 100644 --- a/ktransformers/ktransformers_ext/ext_bindings.cpp +++ b/ktransformers/ktransformers_ext/ext_bindings.cpp @@ -37,7 +37,7 @@ class LinearBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&Linear::warm_up, args_->linear); } - static std::pair interface(Linear& linear) { + static std::pair cpuinfer_interface(Linear& linear) { Args* args = new Args{nullptr, &linear}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -55,7 +55,7 @@ class LinearBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&Linear::forward, args_->linear, args_->qlen, args_->input, args_->output); } - static std::pair interface(Linear& linear, int qlen, intptr_t input, intptr_t output) { + static std::pair cpuinfer_interface(Linear& linear, int qlen, intptr_t input, intptr_t output) { Args* args = new Args{nullptr, &linear, qlen, (const void*)input, (void*)output}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -74,7 +74,7 @@ class MLPBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&MLP::warm_up, args_->mlp); } - static std::pair interface(MLP& mlp) { + static std::pair cpuinfer_interface(MLP& mlp) { Args* args = new Args{nullptr, &mlp}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -92,7 +92,7 @@ class MLPBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&MLP::forward, args_->mlp, args_->qlen, args_->input, args_->output); } - static std::pair interface(MLP& mlp, int qlen, intptr_t input, intptr_t output) { + static std::pair cpuinfer_interface(MLP& mlp, int qlen, intptr_t input, intptr_t output) { Args* args = new Args{nullptr, &mlp, qlen, (const void*)input, (void*)output}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -111,7 +111,7 @@ class MOEBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&MOE::warm_up, args_->moe); } - static std::pair interface(MOE& moe) { + static std::pair cpuinfer_interface(MOE& moe) { Args* args = new Args{nullptr, &moe}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -132,7 +132,7 @@ class MOEBindings { Args* args_ = (Args*)args; args_->cpuinfer->enqueue(&MOE::forward, args_->moe, args_->qlen, args_->k, args_->expert_ids, args_->weights, args_->input, args_->output); } - static std::pair interface(MOE& moe, int qlen, int k, intptr_t expert_ids, intptr_t weights, intptr_t input, intptr_t output) { + static std::pair cpuinfer_interface(MOE& moe, int qlen, int k, intptr_t expert_ids, intptr_t weights, intptr_t input, intptr_t output) { Args* args = new Args{nullptr, &moe, qlen, k, (const uint64_t*)expert_ids, (const float*)weights, (const void*)input, (void*)output}; return std::make_pair((intptr_t)&inner, (intptr_t)args); } @@ -154,8 +154,8 @@ PYBIND11_MODULE(cpuinfer_ext, m) { })); py::class_(linear_module, "Linear") .def(py::init()) - .def("warm_up", &LinearBindings::WarmUpBindinds::interface) - .def("forward", &LinearBindings::ForwardBindings::interface); + .def("warm_up", &LinearBindings::WarmUpBindinds::cpuinfer_interface) + .def("forward", &LinearBindings::ForwardBindings::cpuinfer_interface); auto mlp_module = m.def_submodule("mlp"); py::class_(mlp_module, "MLPConfig") @@ -164,8 +164,8 @@ PYBIND11_MODULE(cpuinfer_ext, m) { })); py::class_(mlp_module, "MLP") .def(py::init()) - .def("warm_up", &MLPBindings::WarmUpBindinds::interface) - .def("forward", &MLPBindings::ForwardBindings::interface); + .def("warm_up", &MLPBindings::WarmUpBindinds::cpuinfer_interface) + .def("forward", &MLPBindings::ForwardBindings::cpuinfer_interface); auto moe_module = m.def_submodule("moe"); py::class_(moe_module, "MOEConfig") @@ -174,6 +174,6 @@ PYBIND11_MODULE(cpuinfer_ext, m) { })); py::class_(moe_module, "MOE") .def(py::init()) - .def("warm_up", &MOEBindings::WarmUpBindinds::interface) - .def("forward", &MOEBindings::ForwardBindings::interface); + .def("warm_up", &MOEBindings::WarmUpBindinds::cpuinfer_interface) + .def("forward", &MOEBindings::ForwardBindings::cpuinfer_interface); } diff --git a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq.py b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq.py deleted file mode 100644 index cda3e7a..0000000 --- a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq.py +++ /dev/null @@ -1,206 +0,0 @@ -import math -import os -import time -from logging import getLogger - -import torch -import torch.nn as nn -import transformers - -from .quantizer import Quantizer - - -logger = getLogger(__name__) - -torch.backends.cuda.matmul.allow_tf32 = False -torch.backends.cudnn.allow_tf32 = False - - -class GPTQ: - def __init__(self, layer): - self.layer = layer - self.dev = self.layer.weight.device - W = layer.weight.data.clone() - if isinstance(self.layer, nn.Conv2d): - W = W.flatten(1) - if isinstance(self.layer, transformers.pytorch_utils.Conv1D): - W = W.t() - self.rows = W.shape[0] - self.columns = W.shape[1] - self.H = torch.zeros((self.columns, self.columns), device=self.dev) - self.nsamples = 0 - self.quantizer = Quantizer() - - def add_batch(self, inp, out): - if os.environ.get("DEBUG"): - self.inp1 = inp - self.out1 = out - if len(inp.shape) == 2: - inp = inp.unsqueeze(0) - tmp = inp.shape[0] - if isinstance(self.layer, nn.Linear) or isinstance(self.layer, transformers.Conv1D): - if len(inp.shape) == 3: - inp = inp.reshape((-1, inp.shape[-1])) - inp = inp.t() - if isinstance(self.layer, nn.Conv2d): - unfold = nn.Unfold( - self.layer.kernel_size, - dilation=self.layer.dilation, - padding=self.layer.padding, - stride=self.layer.stride, - ) - inp = unfold(inp) - inp = inp.permute([1, 0, 2]) - inp = inp.flatten(1) - self.H *= self.nsamples / (self.nsamples + tmp) - self.nsamples += tmp - # inp = inp.float() - inp = math.sqrt(2 / self.nsamples) * inp.float() - # self.H += 2 / self.nsamples * inp.matmul(inp.t()) - self.H += inp.matmul(inp.t()) - - def fasterquant( - self, - blocksize=128, - percdamp=0.01, - group_size=-1, - actorder=False, - static_groups=False, - ): - W = self.layer.weight.data.clone() - if isinstance(self.layer, nn.Conv2d): - W = W.flatten(1) - if isinstance(self.layer, transformers.Conv1D): - W = W.t() - W = W.float() - - tick = time.time() - - if not self.quantizer.ready(): - self.quantizer.find_params(W, weight=True) - - H = self.H - del self.H - dead = torch.diag(H) == 0 - H[dead, dead] = 1 - W[:, dead] = 0 - - g_idx = [] - scale = [] - zero = [] - now_idx = 1 - - if static_groups: - import copy - - groups = [] - for i in range(0, self.columns, group_size): - quantizer = copy.deepcopy(self.quantizer) - quantizer.find_params(W[:, i : (i + group_size)], weight=True) - scale.append(quantizer.scale) - zero.append(quantizer.zero) - groups.append(quantizer) - - if actorder: - perm = torch.argsort(torch.diag(H), descending=True) - W = W[:, perm] - H = H[perm][:, perm] - invperm = torch.argsort(perm) - - Losses = torch.zeros_like(W) - Q = torch.zeros_like(W) - - damp = percdamp * torch.mean(torch.diag(H)) - diag = torch.arange(self.columns, device=self.dev) - H[diag, diag] += damp - H = torch.linalg.cholesky(H) - H = torch.cholesky_inverse(H) - H = torch.linalg.cholesky(H, upper=True) - Hinv = H - - for i1 in range(0, self.columns, blocksize): - i2 = min(i1 + blocksize, self.columns) - count = i2 - i1 - - W1 = W[:, i1:i2].clone() - Q1 = torch.zeros_like(W1) - Err1 = torch.zeros_like(W1) - Losses1 = torch.zeros_like(W1) - Hinv1 = Hinv[i1:i2, i1:i2] - - for i in range(count): - w = W1[:, i] - d = Hinv1[i, i] - - if group_size != -1: - if not static_groups: - if (i1 + i) % group_size == 0: - self.quantizer.find_params(W[:, (i1 + i) : (i1 + i + group_size)], weight=True) - - if ((i1 + i) // group_size) - now_idx == -1: - scale.append(self.quantizer.scale) - zero.append(self.quantizer.zero) - now_idx += 1 - else: - idx = i1 + i - if actorder: - idx = perm[idx] - self.quantizer = groups[idx // group_size] - - q = self.quantizer.quantize(w.unsqueeze(1)).flatten() - Q1[:, i] = q - Losses1[:, i] = (w - q) ** 2 / d**2 - - err1 = (w - q) / d - W1[:, i:] -= err1.unsqueeze(1).matmul(Hinv1[i, i:].unsqueeze(0)) - Err1[:, i] = err1 - - Q[:, i1:i2] = Q1 - Losses[:, i1:i2] = Losses1 / 2 - - W[:, i2:] -= Err1.matmul(Hinv[i1:i2, i2:]) - - if os.environ.get("DEBUG"): - self.layer.weight.data[:, :i2] = Q[:, :i2] - self.layer.weight.data[:, i2:] = W[:, i2:] - logger.debug(torch.sum((self.layer(self.inp1) - self.out1) ** 2)) - logger.debug(torch.sum(Losses)) - - torch.cuda.synchronize() - logger.info(f"duration: {(time.time() - tick)}") - logger.info(f"avg loss: {torch.sum(Losses).item() / self.nsamples}") - - group_size = group_size if group_size != -1 else self.columns - if static_groups and actorder: - g_idx = [perm[i] // group_size for i in range(self.columns)] - else: - g_idx = [i // group_size for i in range(self.columns)] - g_idx = torch.tensor(g_idx, dtype=torch.int32, device=Q.device) - if actorder: - Q = Q[:, invperm] - g_idx = g_idx[invperm] - - if isinstance(self.layer, transformers.Conv1D): - Q = Q.t() - self.layer.weight.data = Q.reshape(self.layer.weight.shape).type_as(self.layer.weight.data) - if os.environ.get("DEBUG"): - logger.debug(torch.sum((self.layer(self.inp1) - self.out1) ** 2)) - - if scale == []: - scale.append(self.quantizer.scale) - zero.append(self.quantizer.zero) - scale = torch.cat(scale, dim=1) - zero = torch.cat(zero, dim=1) - return scale, zero, g_idx - - def free(self): - if os.environ.get("DEBUG"): - self.inp1 = None - self.out1 = None - self.H = None - self.Losses = None - self.Trace = None - torch.cuda.empty_cache() - - -__all__ = ["GPTQ"] diff --git a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq_marlin.py b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq_marlin.py deleted file mode 100644 index 599070f..0000000 --- a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/gptq_marlin.py +++ /dev/null @@ -1,458 +0,0 @@ -import enum -from enum import Enum -from typing import Any, Dict, List, Optional - -import torch -from torch.nn.parameter import Parameter - -from vllm import _custom_ops as ops -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, - set_weight_attrs) -from vllm.model_executor.layers.quantization.base_config import ( - QuantizationConfig) - -logger = init_logger(__name__) - -GPTQ_MARLIN_TILE = 16 -GPTQ_MARLIN_MIN_THREAD_N = 64 -GPTQ_MARLIN_MIN_THREAD_K = 128 -GPTQ_MARLIN_MAX_PARALLEL = 16 - -GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8] -GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] -GPTQ_MARLIN_SUPPORTED_SYM = [True] - - -# Permutations for Marlin scale shuffling -def get_scale_perms(num_bits: int): - scale_perm: List[int] = [] - for i in range(8): - scale_perm.extend([i + 8 * j for j in range(8)]) - scale_perm_single: List[int] = [] - for i in range(4): - scale_perm_single.extend( - [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]]) - return scale_perm, scale_perm_single - - -def get_pack_factor(num_bits: int): - assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS - ), f"Unsupported num_bits = {num_bits}" - return 32 // num_bits - - -def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int, - group_size: int, num_bits: int): - scale_perm, scale_perm_single = get_scale_perms(num_bits) - if group_size < size_k and group_size != -1: - s = s.reshape((-1, len(scale_perm)))[:, scale_perm] - else: - s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] - s = s.reshape((-1, size_n)).contiguous() - - return s - - -class GPTQMarlinConfig(QuantizationConfig): - """Config class for GPTQ Marlin""" - - def __init__(self, weight_bits: int, group_size: int, desc_act: bool, - is_sym: bool) -> None: - if desc_act and group_size == -1: - # In this case, act_order == True is the same as act_order == False - # (since we have only one group per output channel) - desc_act = False - - self.weight_bits = weight_bits - self.group_size = group_size - self.desc_act = desc_act - self.is_sym = is_sym - - # Verify - if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS: - raise ValueError( - f"Marlin does not support weight_bits = {self.weight_bits}. " - f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} " - "are supported.") - if self.group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES: - raise ValueError( - f"Marlin does not support group_size = {self.group_size}. " - f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} " - "are supported.") - if self.is_sym not in GPTQ_MARLIN_SUPPORTED_SYM: - raise ValueError( - f"Marlin does not support is_sym = {self.is_sym}. " - f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.") - - # Init - self.pack_factor = get_pack_factor(weight_bits) - self.tile_size = GPTQ_MARLIN_TILE - self.min_thread_n = GPTQ_MARLIN_MIN_THREAD_N - self.min_thread_k = GPTQ_MARLIN_MIN_THREAD_K - self.max_parallel = GPTQ_MARLIN_MAX_PARALLEL - - def __repr__(self) -> str: - return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, " - f"group_size={self.group_size}, " - f"desc_act={self.desc_act})") - - @classmethod - def get_name(cls) -> str: - return "gptq_marlin" - - @classmethod - def get_supported_act_dtypes(cls) -> List[torch.dtype]: - return [torch.half, torch.bfloat16] - - @classmethod - def get_min_capability(cls) -> int: - return 80 - - @classmethod - def get_config_filenames(cls) -> List[str]: - return ["quantize_config.json"] - - @classmethod - def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlinConfig": - weight_bits = cls.get_from_keys(config, ["bits"]) - group_size = cls.get_from_keys(config, ["group_size"]) - desc_act = cls.get_from_keys(config, ["desc_act"]) - is_sym = cls.get_from_keys(config, ["sym"]) - return cls(weight_bits, group_size, desc_act, is_sym) - - @classmethod - def override_quantization_method(cls, hf_quant_cfg, - user_quant) -> Optional[str]: - can_convert = cls.is_marlin_compatible(hf_quant_cfg) - - is_valid_user_quant = (user_quant is None or user_quant == "marlin") - - if can_convert and is_valid_user_quant: - msg = ("The model is convertible to {} during runtime." - " Using {} kernel.".format(cls.get_name(), cls.get_name())) - logger.info(msg) - return cls.get_name() - - if can_convert and user_quant == "gptq": - logger.info("Detected that the model can run with gptq_marlin" - ", however you specified quantization=gptq explicitly," - " so forcing gptq. Use quantization=gptq_marlin for" - " faster inference") - return None - - def get_quant_method( - self, - layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]: - if isinstance(layer, LinearBase): - return GPTQMarlinLinearMethod(self) - return None - - def get_scaled_act_names(self) -> List[str]: - return [] - - @classmethod - def is_marlin_compatible(cls, quant_config: Dict[str, Any]): - # Extract data from quant config. - num_bits = quant_config.get("bits", None) - group_size = quant_config.get("group_size", None) - sym = quant_config.get("sym", None) - desc_act = quant_config.get("desc_act", None) - - # If we cannot find the info needed in the config, cannot convert. - if (num_bits is None or group_size is None or sym is None - or desc_act is None): - return False - - # If the capability of the device is too low, cannot convert. - major, minor = torch.cuda.get_device_capability() - device_capability = major * 10 + minor - if device_capability < cls.get_min_capability(): - return False - - # Otherwise, can convert if model satisfies marlin constraints. - return (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS - and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES - and sym in GPTQ_MARLIN_SUPPORTED_SYM) - - -class GPTQMarlinState(Enum): - REPACK = enum.auto() - READY = enum.auto() - - -class GPTQMarlinLinearMethod(LinearMethodBase): - """Linear method for GPTQ Marlin. - - Args: - quant_config: The GPTQ Marlin quantization config. - """ - - def __init__(self, quant_config: GPTQMarlinConfig) -> None: - self.quant_config = quant_config - - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: List[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ) -> None: - del output_size - - # Normalize group_size - if self.quant_config.group_size != -1: - group_size = self.quant_config.group_size - else: - group_size = input_size - - # Validate dtype - if params_dtype not in [torch.float16, torch.bfloat16]: - raise ValueError(f"The params dtype must be float16 " - f"or bfloat16, but got {params_dtype}") - - # Validate output_size_per_partition - output_size_per_partition = sum(output_partition_sizes) - if output_size_per_partition % self.quant_config.min_thread_n != 0: - raise ValueError( - f"Weight output_size_per_partition = " - f"{output_size_per_partition} is not divisible by " - f" min_thread_n = {self.quant_config.min_thread_n}.") - - # Validate input_size_per_partition - if input_size_per_partition % self.quant_config.min_thread_k != 0: - raise ValueError( - f"Weight input_size_per_partition = " - f"{input_size_per_partition} is not divisible " - f"by min_thread_k = {self.quant_config.min_thread_k}.") - - if (group_size < input_size - and input_size_per_partition % group_size != 0): - raise ValueError( - f"Weight input_size_per_partition = {input_size_per_partition}" - f" is not divisible by group_size = {group_size}.") - - # Detect sharding of scales/zp - - # By default, no sharding over "input dim" - scales_and_zp_size = input_size // group_size - scales_and_zp_input_dim = None - - if self.quant_config.desc_act: - # Act-order case - assert self.quant_config.group_size != -1 - - is_k_full = input_size_per_partition == input_size - - else: - # No act-order case - - # K is always full due to full alignment with - # group-size and shard of scales/zp - is_k_full = True - - # If this is a row-parallel case, then shard scales/zp - if (input_size != input_size_per_partition - and self.quant_config.group_size != -1): - scales_and_zp_size = input_size_per_partition // group_size - scales_and_zp_input_dim = 0 - - # Init buffers - - # Quantized weights - qweight = Parameter( - torch.empty( - input_size_per_partition // self.quant_config.pack_factor, - output_size_per_partition, - dtype=torch.int32, - ), - requires_grad=False, - ) - set_weight_attrs( - qweight, - { - **extra_weight_attrs, - "input_dim": 0, - "output_dim": 1, - "packed_dim": 0, - "pack_factor": self.quant_config.pack_factor, - }, - ) - - # Activation order - g_idx = Parameter( - torch.empty( - input_size_per_partition, - dtype=torch.int32, - ), - requires_grad=False, - ) - # Ignore warning from fused linear layers such as QKVParallelLinear. - set_weight_attrs( - g_idx, - { - **extra_weight_attrs, "input_dim": 0, - "ignore_warning": True - }, - ) - - g_idx_sort_indices = torch.empty( - g_idx.shape, - dtype=torch.int32, - ) - - # Scales - scales = Parameter( - torch.empty( - scales_and_zp_size, - output_size_per_partition, - dtype=params_dtype, - ), - requires_grad=False, - ) - set_weight_attrs( - scales, - { - **extra_weight_attrs, - "input_dim": scales_and_zp_input_dim, - "output_dim": 1, - }, - ) - - # Quantized zero-points - qzeros = Parameter( - torch.empty( - scales_and_zp_size, - output_size_per_partition // self.quant_config.pack_factor, - dtype=torch.int32, - device="meta", - ), - requires_grad=False, - ) - set_weight_attrs( - qzeros, - { - **extra_weight_attrs, - "input_dim": scales_and_zp_input_dim, - "output_dim": 1, - "packed_dim": 1, - "pack_factor": self.quant_config.pack_factor, - }, - ) - - # Allocate marlin workspace - max_workspace_size = ( - output_size_per_partition // - self.quant_config.min_thread_n) * self.quant_config.max_parallel - workspace = torch.zeros(max_workspace_size, - dtype=torch.int, - requires_grad=False) - - layer.register_parameter("qweight", qweight) - layer.register_parameter("g_idx", g_idx) - layer.register_parameter("scales", scales) - layer.register_parameter("qzeros", qzeros) - layer.g_idx_sort_indices = g_idx_sort_indices - layer.workspace = workspace - layer.input_size_per_partition = input_size_per_partition - layer.output_size_per_partition = output_size_per_partition - layer.input_size = input_size - layer.is_k_full = is_k_full - layer.marlin_state = GPTQMarlinState.REPACK - - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - reshaped_x = x.reshape(-1, x.shape[-1]) - - size_m = reshaped_x.shape[0] - part_size_n = layer.output_size_per_partition - part_size_k = layer.input_size_per_partition - full_size_k = layer.input_size - - out_shape = x.shape[:-1] + (part_size_n, ) - - if layer.marlin_state == GPTQMarlinState.REPACK: - layer.marlin_state = GPTQMarlinState.READY - - # Newly generated tensors need to replace existing tensors that are - # already registered as parameters by vLLM (and won't be freed) - def replace_tensor(name, new_t): - # It is important to use resize_() here since it ensures - # the same buffer is reused - getattr(layer, name).resize_(new_t.shape) - getattr(layer, name).copy_(new_t) - del new_t - - cur_device = layer.qweight.device - - # Process act_order - if self.quant_config.desc_act: - # Get sorting based on g_idx - g_idx_sort_indices = torch.argsort(layer.g_idx).to(torch.int) - - sorted_g_idx = layer.g_idx[g_idx_sort_indices] - - replace_tensor("g_idx", sorted_g_idx) - replace_tensor("g_idx_sort_indices", g_idx_sort_indices) - - else: - # Reset g_idx related tensors - layer.g_idx = Parameter( - torch.empty(0, dtype=torch.int, device=cur_device), - requires_grad=False, - ) - layer.g_idx_sort_indices = Parameter( - torch.empty(0, dtype=torch.int, device=cur_device), - requires_grad=False, - ) - - # Repack weights - marlin_qweight = ops.gptq_marlin_repack( - layer.qweight, - layer.g_idx_sort_indices, - part_size_k, - part_size_n, - self.quant_config.weight_bits, - ) - replace_tensor("qweight", marlin_qweight) - - # Permute scales - scales_size_k = part_size_k - scales_size_n = part_size_n - if self.quant_config.desc_act: - scales_size_k = full_size_k - - marlin_scales = marlin_permute_scales( - layer.scales, - scales_size_k, - scales_size_n, - self.quant_config.group_size, - self.quant_config.weight_bits, - ) - replace_tensor("scales", marlin_scales) - - output = ops.gptq_marlin_gemm( - reshaped_x, - layer.qweight, - layer.scales, - layer.g_idx, - layer.g_idx_sort_indices, - layer.workspace, - self.quant_config.weight_bits, - size_m, - part_size_n, - part_size_k, - layer.is_k_full, - ) - - if bias is not None: - output.add_(bias) # In-place add - - return output.reshape(out_shape) diff --git a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/quantizer.py b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/quantizer.py deleted file mode 100644 index e945a70..0000000 --- a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/quantizer.py +++ /dev/null @@ -1,140 +0,0 @@ -from logging import getLogger - -import torch -import torch.nn as nn - - -logger = getLogger(__name__) - - -def quantize(x, scale, zero, maxq): - if maxq < 0: - return (x > scale / 2).float() * scale + (x < zero / 2).float() * zero - q = torch.clamp(torch.round(x / scale) + zero, 0, maxq) - return scale * (q - zero) - - -class Quantizer(nn.Module): - def __init__(self, shape=1): - super(Quantizer, self).__init__() - self.register_buffer("maxq", torch.tensor(0)) - self.register_buffer("scale", torch.zeros(shape)) - self.register_buffer("zero", torch.zeros(shape)) - - def configure( - self, - bits, - perchannel=False, - sym=True, - mse=False, - norm=2.4, - grid=100, - maxshrink=0.8, - trits=False, - ): - self.maxq = torch.tensor(2**bits - 1) - self.perchannel = perchannel - self.sym = sym - self.mse = mse - self.norm = norm - self.grid = grid - self.maxshrink = maxshrink - if trits: - self.maxq = torch.tensor(-1) - - def find_params(self, x, weight=False): - dev = x.device - self.maxq = self.maxq.to(dev) - - shape = x.shape - if self.perchannel: - if weight: - x = x.flatten(1) - else: - if len(shape) == 4: - x = x.permute([1, 0, 2, 3]) - x = x.flatten(1) - if len(shape) == 3: - x = x.reshape((-1, shape[-1])).t() - if len(shape) == 2: - x = x.t() - else: - x = x.flatten().unsqueeze(0) - - tmp = torch.zeros(x.shape[0], device=dev) - xmin = torch.minimum(x.min(1)[0], tmp) - xmax = torch.maximum(x.max(1)[0], tmp) - - if self.sym: - xmax = torch.maximum(torch.abs(xmin), xmax) - tmp = xmin < 0 - if torch.any(tmp): - xmin[tmp] = -xmax[tmp] - tmp = (xmin == 0) & (xmax == 0) - xmin[tmp] = -1 - xmax[tmp] = +1 - - if self.maxq < 0: - self.scale = xmax - self.zero = xmin - else: - self.scale = (xmax - xmin) / self.maxq - if self.sym: - self.zero = torch.full_like(self.scale, (self.maxq + 1) / 2) - else: - self.zero = torch.round(-xmin / self.scale) - - if self.mse: - best = torch.full([x.shape[0]], float("inf"), device=dev) - for i in range(int(self.maxshrink * self.grid)): - p = 1 - i / self.grid - xmin1 = p * xmin - xmax1 = p * xmax - scale1 = (xmax1 - xmin1) / self.maxq - zero1 = torch.round(-xmin1 / scale1) if not self.sym else self.zero - q = quantize(x, scale1.unsqueeze(1), zero1.unsqueeze(1), self.maxq) - q -= x - q.abs_() - q.pow_(self.norm) - err = torch.sum(q, 1) - tmp = err < best - if torch.any(tmp): - best[tmp] = err[tmp] - self.scale[tmp] = scale1[tmp] - self.zero[tmp] = zero1[tmp] - if not self.perchannel: - if weight: - tmp = shape[0] - else: - tmp = shape[1] if len(shape) != 3 else shape[2] - self.scale = self.scale.repeat(tmp) - self.zero = self.zero.repeat(tmp) - - if weight: - shape = [-1] + [1] * (len(shape) - 1) - self.scale = self.scale.reshape(shape) - self.zero = self.zero.reshape(shape) - return - if len(shape) == 4: - self.scale = self.scale.reshape((1, -1, 1, 1)) - self.zero = self.zero.reshape((1, -1, 1, 1)) - if len(shape) == 3: - self.scale = self.scale.reshape((1, 1, -1)) - self.zero = self.zero.reshape((1, 1, -1)) - if len(shape) == 2: - self.scale = self.scale.unsqueeze(0) - self.zero = self.zero.unsqueeze(0) - - def quantize(self, x): - if self.ready(): - return quantize(x, self.scale, self.zero, self.maxq) - return x - - def enabled(self): - return self.maxq > 0 - - def ready(self): - return torch.all(self.scale != 0) - - -__all__ = ["Quantizer"] diff --git a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/repack.py b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/repack.py deleted file mode 100644 index 987f05b..0000000 --- a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/repack.py +++ /dev/null @@ -1,99 +0,0 @@ -import torch -import enum -from enum import Enum -from typing import Any, Dict, List, Optional -from torch.nn.parameter import Parameter - -def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: Optional[torch.Tensor] = None, -) -> torch.Tensor: - reshaped_x = x.reshape(-1, x.shape[-1]) - - size_m = reshaped_x.shape[0] - part_size_n = layer.output_size_per_partition - part_size_k = layer.input_size_per_partition - full_size_k = layer.input_size - - out_shape = x.shape[:-1] + (part_size_n, ) - - if layer.marlin_state == GPTQMarlinState.REPACK: - layer.marlin_state = GPTQMarlinState.READY - - # Newly generated tensors need to replace existing tensors that are - # already registered as parameters by vLLM (and won't be freed) - def replace_tensor(name, new_t): - # It is important to use resize_() here since it ensures - # the same buffer is reused - getattr(layer, name).resize_(new_t.shape) - getattr(layer, name).copy_(new_t) - del new_t - - cur_device = layer.qweight.device - - # Process act_order - if self.quant_config.desc_act: - # Get sorting based on g_idx - g_idx_sort_indices = torch.argsort(layer.g_idx).to(torch.int) - - sorted_g_idx = layer.g_idx[g_idx_sort_indices] - - replace_tensor("g_idx", sorted_g_idx) - replace_tensor("g_idx_sort_indices", g_idx_sort_indices) - - else: - # Reset g_idx related tensors - layer.g_idx = Parameter( - torch.empty(0, dtype=torch.int, device=cur_device), - requires_grad=False, - ) - layer.g_idx_sort_indices = Parameter( - torch.empty(0, dtype=torch.int, device=cur_device), - requires_grad=False, - ) - - # Repack weights - marlin_qweight = ops.gptq_marlin_repack( - layer.qweight, - layer.g_idx_sort_indices, - part_size_k, - part_size_n, - self.quant_config.weight_bits, - ) - replace_tensor("qweight", marlin_qweight) - - # Permute scales - scales_size_k = part_size_k - scales_size_n = part_size_n - if self.quant_config.desc_act: - scales_size_k = full_size_k - - marlin_scales = marlin_permute_scales( - layer.scales, - scales_size_k, - scales_size_n, - self.quant_config.group_size, - self.quant_config.weight_bits, - ) - replace_tensor("scales", marlin_scales) - - output = ops.gptq_marlin_gemm( - reshaped_x, - layer.qweight, - layer.scales, - layer.g_idx, - layer.g_idx_sort_indices, - layer.workspace, - self.quant_config.weight_bits, - size_m, - part_size_n, - part_size_k, - layer.is_k_full, - ) - - if bias is not None: - output.add_(bias) # In-place add - - return output.reshape(out_shape) diff --git a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/utils/marlin_utils.py b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/utils/marlin_utils.py index 7b0398f..accbc00 100644 --- a/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/utils/marlin_utils.py +++ b/ktransformers/ktransformers_ext/operators/custom_marlin/quantize/utils/marlin_utils.py @@ -220,7 +220,7 @@ def compute_max_diff(output, output_ref): class MarlinWorkspace: - def __init__(self, out_features, min_thread_n, max_parallel): + def __init__(self, out_features, min_thread_n, max_parallel, device): assert (out_features % min_thread_n == 0), ( "out_features = {} is undivisible by min_thread_n = {}".format( out_features, min_thread_n)) @@ -229,4 +229,4 @@ def __init__(self, out_features, min_thread_n, max_parallel): self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, - device="cuda") + device=device) diff --git a/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp b/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp index 7dcba57..81e5006 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/linear.cpp @@ -47,13 +47,13 @@ void Linear::forward_many(int qlen, const void* input, void* output, Backend* ba int nth = config_.output_size / config_.stride; backend->do_work_stealing_job(nth, [&](int task_id) { int ith = task_id; - void* proj_ptr = proj_ + ith * config_.stride * config_.input_size * ggml_type_size(config_.proj_type) / ggml_blck_size(config_.proj_type); + void* proj_ptr = (uint8_t*)proj_ + ith * config_.stride * config_.input_size * ggml_type_size(config_.proj_type) / ggml_blck_size(config_.proj_type); float* proj_output_ptr = proj_output_ + ith * config_.stride; llamafile_sgemm(config_.stride, qlen, config_.input_size / ggml_blck_size(config_.proj_type), proj_ptr, config_.input_size / ggml_blck_size(config_.proj_type), proj_input_ptr, config_.input_size / ggml_blck_size(config_.proj_type), proj_output_ptr, config_.output_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.proj_type, ggml_internal_get_type_traits(config_.proj_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); if (config_.stride % ggml_blck_size(config_.hidden_type) == 0) { for (int i = 0; i < qlen; i++) { float* output_fp32_ptr = proj_output_ + i * config_.output_size + ith * config_.stride; - void* output_ptr = output + i * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + void* output_ptr = (uint8_t*)output + i * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); from_float(output_fp32_ptr, output_ptr, config_.stride, config_.hidden_type); } } @@ -69,5 +69,5 @@ void Linear::forward(int qlen, const void* input, void* output, Backend* backend } int forward_len = std::min(qlen, config_.group_max_len); forward_many(forward_len, input, output, backend); - forward(qlen - forward_len, input + forward_len * config_.input_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + forward_len * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); + forward(qlen - forward_len, (uint8_t*)input + forward_len * config_.input_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), (uint8_t*)output + forward_len * config_.output_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); } \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp b/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp index 8ef092f..abad01e 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/mlp.cpp @@ -74,10 +74,10 @@ void MLP::forward_many(int qlen, const void* input, void* output, Backend* backe int nth = config_.intermediate_size / config_.stride; backend->do_work_stealing_job(nth, [&](int task_id) { int ith = task_id; - void* gate_proj_ptr = gate_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); + void* gate_proj_ptr = (uint8_t*)gate_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); float* gate_output_ptr = gate_output_ + ith * config_.stride; llamafile_sgemm(config_.stride, qlen, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_proj_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_input_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.gate_type, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); - void* up_proj_ptr = up_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); + void* up_proj_ptr = (uint8_t*)up_proj_ + ith * config_.stride * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); float* up_output_ptr = up_output_ + ith * config_.stride; llamafile_sgemm(config_.stride, qlen, config_.hidden_size / ggml_blck_size(config_.up_type), up_proj_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_input_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.up_type, ggml_internal_get_type_traits(config_.up_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); for (int i = 0; i < qlen; i++) { @@ -86,7 +86,7 @@ void MLP::forward_many(int qlen, const void* input, void* output, Backend* backe } if (config_.stride % ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) == 0) { float* intermediate_fp32_ptr = intermediate_fp32_ + i * config_.intermediate_size + ith * config_.stride; - void* down_input_ptr = down_input_ + i * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) + ith * config_.stride * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); + void* down_input_ptr = (uint8_t*)down_input_ + i * config_.intermediate_size * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) + ith * config_.stride * ggml_type_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type) / ggml_blck_size(ggml_internal_get_type_traits(config_.down_type).vec_dot_type); from_float(intermediate_fp32_ptr, down_input_ptr, config_.stride, ggml_internal_get_type_traits(config_.down_type).vec_dot_type); } } @@ -97,13 +97,13 @@ void MLP::forward_many(int qlen, const void* input, void* output, Backend* backe nth = config_.hidden_size / config_.stride; backend->do_work_stealing_job(nth, [&](int task_id) { int ith = task_id; - void* down_proj_ptr = down_proj_ + ith * config_.stride * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); + void* down_proj_ptr = (uint8_t*)down_proj_ + ith * config_.stride * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); float* down_output_ptr = down_output_ + ith * config_.stride; llamafile_sgemm(config_.stride, qlen, config_.intermediate_size / ggml_blck_size(config_.down_type), down_proj_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), down_input_, config_.intermediate_size / ggml_blck_size(config_.down_type), down_output_ptr, config_.hidden_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.down_type, ggml_internal_get_type_traits(config_.down_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); if (config_.stride % ggml_blck_size(config_.hidden_type) == 0) { for (int i = 0; i < qlen; i++) { float* output_fp32_ptr = down_output_ + i * config_.hidden_size + ith * config_.stride; - void* output_ptr = output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + void* output_ptr = (uint8_t*)output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type) + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); from_float(output_fp32_ptr, output_ptr, config_.stride, config_.hidden_type); } } @@ -119,5 +119,5 @@ void MLP::forward(int qlen, const void* input, void* output, Backend* backend) { } int forward_len = std::min(qlen, config_.group_max_len); forward_many(forward_len, input, output, backend); - forward(qlen - forward_len, input + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); + forward(qlen - forward_len, (uint8_t*)input + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), (uint8_t*)output + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); } \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp b/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp index 8010f54..d75db65 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/moe.cpp @@ -9,7 +9,7 @@ **/ #include "moe.h" #include -#include "unistd.h" +#include MOE::MOE(MOEConfig config) { config_ = config; @@ -60,7 +60,7 @@ MOE::MOE(MOEConfig config) { m_local_pos_.resize(config_.group_max_len); for (int i = 0; i < config_.group_max_len; i++) { - m_local_pos_[i].reserve(config_.expert_num); + m_local_pos_[i].resize(config_.routed_expert_num); } m_local_num_.resize(config_.expert_num); m_local_gate_input_ptr_.resize(config_.expert_num); @@ -125,10 +125,10 @@ void MOE::forward_one(int k, const uint64_t* expert_ids, const float* weights, c int expert_idx = task_id / nth; uint64_t expert_id = expert_ids[expert_idx]; int ith = task_id % nth; - void* gate_proj_ptr = gate_proj_ + (expert_id * config_.intermediate_size + ith * config_.stride) * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); + void* gate_proj_ptr = (uint8_t*)gate_proj_ + (expert_id * config_.intermediate_size + ith * config_.stride) * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); float* gate_output_ptr = s_gate_output_[expert_idx] + ith * config_.stride; llamafile_sgemm(config_.stride, 1, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_proj_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_input_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.gate_type, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); - void* up_proj_ptr = up_proj_ + (expert_id * config_.intermediate_size + ith * config_.stride) * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); + void* up_proj_ptr = (uint8_t*)up_proj_ + (expert_id * config_.intermediate_size + ith * config_.stride) * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); float* up_output_ptr = s_up_output_[expert_idx] + ith * config_.stride; llamafile_sgemm(config_.stride, 1, config_.hidden_size / ggml_blck_size(config_.up_type), up_proj_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_input_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.up_type, ggml_internal_get_type_traits(config_.up_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); for (int i = ith * config_.stride; i < (ith + 1) * config_.stride; i++) { @@ -153,7 +153,7 @@ void MOE::forward_one(int k, const uint64_t* expert_ids, const float* weights, c } for (int expert_idx = 0; expert_idx < k; expert_idx++) { uint64_t expert_id = expert_ids[expert_idx]; - void* down_proj_ptr = down_proj_ + (expert_id * config_.hidden_size + ith * config_.stride) * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); + void* down_proj_ptr = (uint8_t*)down_proj_ + (expert_id * config_.hidden_size + ith * config_.stride) * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); float* down_output_ptr = s_down_output_[expert_idx] + ith * config_.stride; llamafile_sgemm(config_.stride, 1, config_.intermediate_size / ggml_blck_size(config_.down_type), down_proj_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), s_down_input_[expert_idx], config_.intermediate_size / ggml_blck_size(config_.down_type), down_output_ptr, config_.stride, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.down_type, ggml_internal_get_type_traits(config_.down_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); for (int i = ith * config_.stride; i < (ith + 1) * config_.stride; i++) { @@ -162,7 +162,7 @@ void MOE::forward_one(int k, const uint64_t* expert_ids, const float* weights, c } if (config_.stride % ggml_blck_size(config_.hidden_type) == 0) { float* output_fp32_ptr = s_output_fp32_ + ith * config_.stride; - void* output_ptr = output + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + void* output_ptr = (uint8_t*)output + ith * config_.stride * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); from_float(output_fp32_ptr, output_ptr, config_.stride, config_.hidden_type); } }); @@ -195,9 +195,9 @@ void MOE::forward_many(int qlen, int k, const uint64_t* expert_ids, const float* const void* gate_input_ptr; const void* up_input_ptr; if (config_.hidden_type == ggml_internal_get_type_traits(config_.gate_type).vec_dot_type && config_.hidden_type == ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { - gate_input_ptr = up_input_ptr = input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + gate_input_ptr = up_input_ptr = (uint8_t*)input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); } else { - to_float(input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), m_input_fp32_[i], config_.hidden_size, config_.hidden_type); + to_float((uint8_t*)input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), m_input_fp32_[i], config_.hidden_size, config_.hidden_type); if (ggml_internal_get_type_traits(config_.gate_type).vec_dot_type == ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { from_float(m_input_fp32_[i], m_gate_input_[i], config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); gate_input_ptr = up_input_ptr = m_gate_input_[i]; @@ -206,13 +206,13 @@ void MOE::forward_many(int qlen, int k, const uint64_t* expert_ids, const float* from_float(m_input_fp32_[i], m_gate_input_[i], config_.hidden_size, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type); gate_input_ptr = m_gate_input_[i]; } else { - gate_input_ptr = input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + gate_input_ptr = (uint8_t*)input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); } if (config_.hidden_type != ggml_internal_get_type_traits(config_.up_type).vec_dot_type) { from_float(m_input_fp32_[i], m_up_input_[i], config_.hidden_size, ggml_internal_get_type_traits(config_.up_type).vec_dot_type); up_input_ptr = m_up_input_[i]; } else { - up_input_ptr = input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); + up_input_ptr = (uint8_t*)input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type); } } } @@ -227,11 +227,11 @@ void MOE::forward_many(int qlen, int k, const uint64_t* expert_ids, const float* int expert_idx = task_id / nth; int ith = task_id % nth; void* gate_input_ptr = m_local_gate_input_ptr_[expert_idx]; - void* gate_proj_ptr = gate_proj_ + (expert_idx * config_.intermediate_size + ith * stride) * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); + void* gate_proj_ptr = (uint8_t*)gate_proj_ + (expert_idx * config_.intermediate_size + ith * stride) * config_.hidden_size * ggml_type_size(config_.gate_type) / ggml_blck_size(config_.gate_type); float* gate_output_ptr = m_local_gate_output_ptr_[expert_idx] + ith * stride; llamafile_sgemm(stride, m_local_num_[expert_idx], config_.hidden_size / ggml_blck_size(config_.gate_type), gate_proj_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_input_ptr, config_.hidden_size / ggml_blck_size(config_.gate_type), gate_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.gate_type, ggml_internal_get_type_traits(config_.gate_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); void* up_input_ptr = m_local_up_input_ptr_[expert_idx]; - void* up_proj_ptr = up_proj_ + (expert_idx * config_.intermediate_size + ith * stride) * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); + void* up_proj_ptr = (uint8_t*)up_proj_ + (expert_idx * config_.intermediate_size + ith * stride) * config_.hidden_size * ggml_type_size(config_.up_type) / ggml_blck_size(config_.up_type); float* up_output_ptr = m_local_up_output_ptr_[expert_idx] + ith * stride; llamafile_sgemm(stride, m_local_num_[expert_idx], config_.hidden_size / ggml_blck_size(config_.up_type), up_proj_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_input_ptr, config_.hidden_size / ggml_blck_size(config_.up_type), up_output_ptr, config_.intermediate_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.up_type, ggml_internal_get_type_traits(config_.up_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); for (int i = 0; i < m_local_num_[expert_idx]; i++) { @@ -249,7 +249,7 @@ void MOE::forward_many(int qlen, int k, const uint64_t* expert_ids, const float* int expert_idx = task_id / nth; int ith = task_id % nth; void* down_input_ptr = m_local_down_input_ptr_[expert_idx]; - void* down_proj_ptr = down_proj_ + (expert_idx * config_.hidden_size + ith * stride) * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); + void* down_proj_ptr = (uint8_t*)down_proj_ + (expert_idx * config_.hidden_size + ith * stride) * config_.intermediate_size * ggml_type_size(config_.down_type) / ggml_blck_size(config_.down_type); float* down_output_ptr = m_local_down_output_ptr_[expert_idx] + ith * stride; llamafile_sgemm(stride, m_local_num_[expert_idx], config_.intermediate_size / ggml_blck_size(config_.down_type), down_proj_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), down_input_ptr, config_.intermediate_size / ggml_blck_size(config_.down_type), down_output_ptr, config_.hidden_size, 0, 1, GGML_TASK_TYPE_COMPUTE, config_.down_type, ggml_internal_get_type_traits(config_.down_type).vec_dot_type, GGML_TYPE_F32, GGML_PREC_DEFAULT); }); @@ -262,18 +262,18 @@ void MOE::forward_many(int qlen, int k, const uint64_t* expert_ids, const float* m_output_fp32_[i][e] += m_local_down_output_ptr_[expert_ids[i * k + j]][m_local_pos_[i][j] * config_.hidden_size + e] * weights[i * k + j]; } } - from_float(m_output_fp32_[i], output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), config_.hidden_size, config_.hidden_type); + from_float(m_output_fp32_[i], (uint8_t*)output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), config_.hidden_size, config_.hidden_type); }); } void MOE::forward(int qlen, int k, const uint64_t* expert_ids, const float* weights, const void* input, void* output, Backend* backend) { if (qlen < config_.group_min_len) { for (int i = 0; i < qlen; i++) { - forward_one(k, expert_ids + i * k, weights + i * k, input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); + forward_one(k, expert_ids + i * k, weights + i * k, (uint8_t*)input + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), (uint8_t*)output + i * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); } return; } int forward_len = std::min(config_.group_max_len, qlen); forward_many(forward_len, k, expert_ids, weights, input, output, backend); - forward(qlen - forward_len, k, expert_ids + forward_len * k, weights + forward_len * k, input + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), output + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); + forward(qlen - forward_len, k, expert_ids + forward_len * k, weights + forward_len * k, (uint8_t*)input + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), (uint8_t*)output + forward_len * config_.hidden_size * ggml_type_size(config_.hidden_type) / ggml_blck_size(config_.hidden_type), backend); } \ No newline at end of file diff --git a/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp index b1599da..dc2d65d 100644 --- a/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp +++ b/ktransformers/ktransformers_ext/operators/llamafile/shared_mem_buffer.cpp @@ -49,7 +49,7 @@ void SharedMemBuffer::dealloc(void* object) { void SharedMemBuffer::arrange(std::vector> requests) { uint64_t offset = 0; for (auto& request : requests) { - *(request.first) = buffer_ + offset; + *(request.first) = (uint8_t*)buffer_ + offset; offset += request.second; } } diff --git a/ktransformers/local_chat.py b/ktransformers/local_chat.py old mode 100644 new mode 100755 index 59839be..b5782d1 --- a/ktransformers/local_chat.py +++ b/ktransformers/local_chat.py @@ -31,18 +31,21 @@ from ktransformers.optimize.optimize import optimize_and_load_gguf from ktransformers.models.modeling_deepseek import DeepseekV2ForCausalLM from ktransformers.models.modeling_qwen2_moe import Qwen2MoeForCausalLM +from ktransformers.models.modeling_mixtral import MixtralForCausalLM from ktransformers.util.utils import prefill_and_generate from ktransformers.server.config.config import Config custom_models = { "DeepseekV2ForCausalLM": DeepseekV2ForCausalLM, "Qwen2MoeForCausalLM": Qwen2MoeForCausalLM, + "MixtralForCausalLM": MixtralForCausalLM, } ktransformer_rules_dir = os.path.dirname(os.path.abspath(__file__)) + "/optimize/optimize_rules/" default_optimize_rules ={ "DeepseekV2ForCausalLM": ktransformer_rules_dir + "DeepSeek-V2-Chat.yaml", "Qwen2MoeForCausalLM": ktransformer_rules_dir + "Qwen2-57B-A14B-Instruct.yaml", + "MixtralForCausalLM": ktransformer_rules_dir + "Mixtral.yaml", } def local_chat( @@ -50,7 +53,8 @@ def local_chat( optimize_rule_path: str = None, gguf_path: str = None, max_new_tokens: int = 1000, - cpu_infer: int = Config().cpu_infer + cpu_infer: int = Config().cpu_infer, + use_cuda_graph: bool = True, ): torch.set_grad_enabled(False) @@ -64,6 +68,8 @@ def local_chat( print("using custom modeling_xxx.py.") if "Qwen2Moe" in config.architectures[0]: # Qwen2Moe must use flash_attention_2 to avoid overflow. config._attn_implementation = "flash_attention_2" + if "Mixtral" in config.architectures[0]: + config._attn_implementation = "flash_attention_2" model = custom_models[config.architectures[0]](config) else: model = AutoModelForCausalLM.from_config( @@ -100,7 +106,6 @@ def local_chat( while True: content = input("Chat: ") - # if content is num if content == "": content = "Please write a piece of quicksort code in C++." @@ -109,7 +114,7 @@ def local_chat( messages, add_generation_prompt=True, return_tensors="pt" ) torch.set_default_dtype(torch.bfloat16) # TODO: Remove this, replace dtype using config - generated = prefill_and_generate(model, tokenizer, input_tensor.cuda(), max_new_tokens) + generated = prefill_and_generate(model, tokenizer, input_tensor.cuda(), max_new_tokens, use_cuda_graph) if __name__ == "__main__": - fire.Fire(local_chat) + fire.Fire(local_chat) \ No newline at end of file diff --git a/ktransformers/models/custom_cache.py b/ktransformers/models/custom_cache.py index 385e6ec..dbaea57 100644 --- a/ktransformers/models/custom_cache.py +++ b/ktransformers/models/custom_cache.py @@ -22,13 +22,14 @@ class StaticCache(transformers.StaticCache): The maximum batch size with which the model will be used. max_cache_len (`int`): The maximum sequence length with which the model will be used. - device (`torch.device`): + device (`torch.device` or `dict`): The device on which the cache should be initialized. Should be the same as the layer. + If a `dict`, it should contain the `device` key with the device name as the value. dtype (*optional*, defaults to `torch.float32`): The default `dtype` to use when initializing the layer. """ - def __init__(self, config: PretrainedConfig, max_batch_size: int, max_cache_len: int, device, dtype=None) -> None: + def __init__(self, config: PretrainedConfig, max_batch_size: int, max_cache_len: int, device: torch.device| dict, dtype=None) -> None: Cache.__init__(self) self.max_batch_size = max_batch_size self.max_cache_len = config.max_position_embeddings if max_cache_len is None else max_cache_len @@ -46,6 +47,7 @@ def __init__(self, config: PretrainedConfig, max_batch_size: int, max_cache_len: self.value_cache: List[torch.Tensor] = [] cache_shape = (max_batch_size, self.num_key_value_heads, self.max_cache_len, self.head_dim) if config.architectures[0] == "DeepseekV2ForCausalLM": + # TODO: for deepseek, cache_shape is different whether using Absorbed MLA, check it automatically # key_shape = (max_batch_size, self.num_key_value_heads, self.max_cache_len, config.qk_rope_head_dim + config.qk_nope_head_dim) # value_shape = (max_batch_size, self.num_key_value_heads, self.max_cache_len, config.v_head_dim) key_shape = (max_batch_size, 1, self.max_cache_len, config.qk_rope_head_dim) @@ -56,11 +58,15 @@ def __init__(self, config: PretrainedConfig, max_batch_size: int, max_cache_len: self.past_tokens = [] self.num_hidden_layers = config.num_hidden_layers - for _ in range(self.num_hidden_layers): + for idx in range(self.num_hidden_layers): # Note: `mark_static_address` is used to tag the cache as an fixed data pointer, preventing cuda graph # breaks when updating the cache. - new_layer_key_cache = torch.zeros(key_shape, dtype=self.dtype, device=device) - new_layer_value_cache = torch.zeros(value_shape, dtype=self.dtype, device=device) + if isinstance(device, dict): + target_device = device[f"blk.{idx}.self_attn"]["generate_device"] + else: + target_device = device + new_layer_key_cache = torch.zeros(key_shape, dtype=self.dtype, device=target_device) + new_layer_value_cache = torch.zeros(value_shape, dtype=self.dtype, device=target_device) torch._dynamo.mark_static_address(new_layer_key_cache) torch._dynamo.mark_static_address(new_layer_value_cache) self.key_cache.append(new_layer_key_cache) diff --git a/ktransformers/models/modeling_deepseek.py b/ktransformers/models/modeling_deepseek.py index 81fee86..692020d 100644 --- a/ktransformers/models/modeling_deepseek.py +++ b/ktransformers/models/modeling_deepseek.py @@ -1048,7 +1048,7 @@ def _flash_attention_forward( """ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token first unpad the input, then computes the attention scores and pad the final attention scores. - Args: + # Args: query_states (`torch.Tensor`): Input query states to be passed to Flash Attention API key_states (`torch.Tensor`): @@ -1245,12 +1245,14 @@ def forward( cache_position=cache_position, **kwargs, ) + hidden_states = residual + hidden_states # Fully Connected residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states outputs = (hidden_states,) diff --git a/ktransformers/models/modeling_mixtral.py b/ktransformers/models/modeling_mixtral.py new file mode 100644 index 0000000..87d8cf1 --- /dev/null +++ b/ktransformers/models/modeling_mixtral.py @@ -0,0 +1,1735 @@ +# coding=utf-8 +''' +Description : +Author : kkk1nak0 +Date : 2024-07-29 02:58:57 +Version : 1.0.0 +LastEditors : kkk1nak0 +LastEditTime : 2024-08-02 06:08:34 +''' + +# Adapted from +# https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py +# Copyright 2023 Mistral AI and the HuggingFace Inc. team. All rights reserved. +# Copyright (c) 2024 by KVCache.AI, All Rights Reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Mixtral model.""" + +import inspect +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache, StaticCache +from transformers.modeling_attn_mask_utils import ( + AttentionMaskConverter, + _prepare_4d_causal_attention_mask, +) +from transformers.modeling_outputs import ( + MoeCausalLMOutputWithPast, + MoeModelOutputWithPast, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_utils import PreTrainedModel +from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_13 +from transformers.utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_2_available, + logging, + replace_return_docstrings, +) +from transformers.utils.import_utils import is_torch_fx_available +from transformers.models.mixtral.configuration_mixtral import MixtralConfig + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_varlen_func, flash_attn_func, flash_attn_with_kvcache + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) + +# This makes `_prepare_4d_causal_attention_mask` a leaf function in the FX graph. +# It means that the function will not be traced through and simply appear as a node in the graph. +if is_torch_fx_available(): + if not is_torch_greater_or_equal_than_1_13: + import torch.fx + + _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask) + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "MixtralConfig" + + +def load_balancing_loss_func( + gate_logits: torch.Tensor, num_experts: torch.Tensor = None, top_k=2, attention_mask: Optional[torch.Tensor] = None +) -> float: + r""" + Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. + + See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss + function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between + experts is too unbalanced. + + Args: + gate_logits (Union[`torch.Tensor`, Tuple[torch.Tensor]): + Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of + shape [batch_size X sequence_length, num_experts]. + attention_mask (`torch.Tensor`, None): + The attention_mask used in forward function + shape [batch_size X sequence_length] if not None. + num_experts (`int`, *optional*): + Number of experts + + Returns: + The auxiliary loss. + """ + if gate_logits is None or not isinstance(gate_logits, tuple): + return 0 + + if isinstance(gate_logits, tuple): + compute_device = gate_logits[0].device + concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0) + + routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) + + _, selected_experts = torch.topk(routing_weights, top_k, dim=-1) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) + + if attention_mask is None: + # Compute the percentage of tokens routed to each experts + tokens_per_expert = torch.mean(expert_mask.float(), dim=0) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.mean(routing_weights, dim=0) + else: + batch_size, sequence_length = attention_mask.shape + num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length) + + # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask + expert_attention_mask = ( + attention_mask[None, :, :, None, None] + .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts)) + .reshape(-1, top_k, num_experts) + .to(compute_device) + ) + + # Compute the percentage of tokens routed to each experts + tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum( + expert_attention_mask, dim=0 + ) + + # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert + router_per_expert_attention_mask = ( + attention_mask[None, :, :, None] + .expand((num_hidden_layers, batch_size, sequence_length, num_experts)) + .reshape(-1, num_experts) + .to(compute_device) + ) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum( + router_per_expert_attention_mask, dim=0 + ) + + overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0)) + return overall_loss * num_experts + + +# Copied from transformers.models.llama.modeling_llama._get_unpad_data +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Mixtral +class MixtralRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MixtralRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +# copied from transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding with Mistral->Mixtral +# TODO @longjie no longer copied from Mistral after static cache +class MixtralRotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + + @torch.no_grad() + def forward(self, x, position_ids): + # x: [bs, num_attention_heads, seq_len, head_size] + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# copied from transformers.models.mistral.modeling_mistral.apply_rotary_pos_emb +# TODO @longjie no longer copied from Mistral after static cache +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +# copied from transformers.models.mistral.modeling_mistral.MistralAttention with Mistral->Mixtral +# TODO @longjie no longer copied from Mistral after static cache +class MixtralAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: MixtralConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = MixtralRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +# copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2 with Mistral->Mixtral +# TODO @longjie no longer copied from Mistral after static cache +class MixtralFlashAttention2(MixtralAttention): + """ + Mixtral flash attention module. This module inherits from `MixtralAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + ): + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + cos, sin = self.rotary_emb(value_states, position_ids) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and self.config.use_sliding_window + ) + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention, for a more memory efficient implementation" + " make sure to upgrade flash-attn library." + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # we slice the states for static kv cache to be supported in FA2. Not sure it's a must as compile fails + # for bsz == 1, avoid using slice to capture cuda graph + if cache_position is not None and q_len > 1: + key_states = key_states[:, :, : cache_position[-1] + 1, :] + value_states = value_states[:, :, : cache_position[-1] + 1, :] + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + position_ids=position_ids, + dropout=dropout_rate, + sliding_window=getattr(self.config, "sliding_window", None), + is_causal=self.is_causal, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + q_len, + position_ids, + dropout, + sliding_window, + is_causal, + softmax_scale=None, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + + """ + + # Decide whether to use SWA or not by layer index. + # if use_sliding_windows and self.layer_idx >= self.config.max_window_layers: + # use_sliding_windows = False + use_sliding_windows = False + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, q_len + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=is_causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=is_causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, q_len) + else: + if not use_sliding_windows: + if q_len == 1: + position_ids = position_ids.to(dtype=torch.int32).squeeze(1) + attn_output = flash_attn_with_kvcache( + query_states, + key_states, + value_states, + cache_seqlens=position_ids, + softmax_scale=softmax_scale, + causal=is_causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=is_causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=is_causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + + + +# copied from transformers.models.mistral.modeling_mistral.MistralSdpaAttention with Mistral->Mixtral +# TODO @longjie no longer copied from Mistral after static cache +class MixtralSdpaAttention(MixtralAttention): + """ + Mixtral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `MixtralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from MixtralAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "MixtralModel is using MixtralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment + # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling. + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal = True if causal_mask is None and q_len > 1 else False + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=is_causal, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +MIXTRAL_ATTENTION_CLASSES = { + "eager": MixtralAttention, + "flash_attention_2": MixtralFlashAttention2, + "sdpa": MixtralSdpaAttention, +} + + +class MixtralBlockSparseTop2MLP(nn.Module): + def __init__(self, config: MixtralConfig): + super().__init__() + self.ffn_dim = config.intermediate_size + self.hidden_dim = config.hidden_size + + self.w1 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False) # gate + self.w2 = nn.Linear(self.ffn_dim, self.hidden_dim, bias=False) # down + self.w3 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False) # up + + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + current_hidden_states = self.act_fn(self.w1(hidden_states)) * self.w3(hidden_states) + current_hidden_states = self.w2(current_hidden_states) + return current_hidden_states + + +class MixtralSparseMoeBlock(nn.Module): + """ + This implementation is + strictly equivalent to standard MoE with full capacity (no + dropped tokens). It's faster since it formulates MoE operations + in terms of block-sparse operations to accomodate imbalanced + assignments of tokens to experts, whereas standard MoE either + (1) drop tokens at the cost of reduced performance or (2) set + capacity factor to number of experts and thus waste computation + and memory on padding. + """ + + def __init__(self, config): + super().__init__() + self.hidden_dim = config.hidden_size + self.ffn_dim = config.intermediate_size + self.num_experts = config.num_local_experts + self.top_k = config.num_experts_per_tok + + # gating + self.gate = nn.Linear(self.hidden_dim, self.num_experts, bias=False) + + self.experts = nn.ModuleList([MixtralBlockSparseTop2MLP(config) for _ in range(self.num_experts)]) + + # Jitter parameters + self.jitter_noise = config.router_jitter_noise + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """ """ + batch_size, sequence_length, hidden_dim = hidden_states.shape + if self.training and self.jitter_noise > 0: + hidden_states *= torch.empty_like(hidden_states).uniform_(1.0 - self.jitter_noise, 1.0 + self.jitter_noise) + hidden_states = hidden_states.view(-1, hidden_dim) + # router_logits: (batch * sequence_length, n_experts) + router_logits = self.gate(hidden_states) + + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + # we cast back to the input dtype + routing_weights = routing_weights.to(hidden_states.dtype) + + final_hidden_states = torch.zeros( + (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device + ) + + # One hot encode the selected experts to create an expert mask + # this will be used to easily index which expert is going to be sollicitated + expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0) + + # Loop over all available experts in the model and perform the computation on each expert + for expert_idx in range(self.num_experts): + expert_layer = self.experts[expert_idx] + idx, top_x = torch.where(expert_mask[expert_idx]) + + # Index the correct hidden states and compute the expert hidden state for + # the current expert. We need to make sure to multiply the output hidden + # states by `routing_weights` on the corresponding tokens (top-1 and top-2) + current_state = hidden_states[None, top_x].reshape(-1, hidden_dim) + current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None] + + # However `index_add_` only support torch tensors for indexing so we'll use + # the `top_x` tensor here. + final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype)) + final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim) + return final_hidden_states, router_logits + + +class MixtralDecoderLayer(nn.Module): + def __init__(self, config: MixtralConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = MIXTRAL_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx) + + self.block_sparse_moe = MixtralSparseMoeBlock(config) + self.input_layernorm = MixtralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = MixtralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + output_router_logits: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, sequence_length)` where padding elements are indicated by 0. + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_router_logits (`bool`, *optional*): + Whether or not to return the logits of all the routers. They are useful for computing the router loss, and + should not be returned during inference. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. + kwargs (`dict`, *optional*): + Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code + into the model + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states, router_logits = self.block_sparse_moe(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + if output_router_logits: + outputs += (router_logits,) + + return outputs + + +MIXTRAL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MixtralConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Mixtral Model outputting raw hidden-states without any specific head on top.", + MIXTRAL_START_DOCSTRING, +) +# Copied from transformers.models.qwen2.modeling_qwen2.Qwen2PreTrainedModel with Qwen2->Mixtral +class MixtralPreTrainedModel(PreTrainedModel): + config_class = MixtralConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["MixtralDecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +MIXTRAL_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + output_router_logits (`bool`, *optional*): + Whether or not to return the logits of all the routers. They are useful for computing the router loss, and + should not be returned during inference. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, + this tensor is not affected by padding. It is used to update the cache in the correct position and to infer + the complete sequence length. +""" + + +@add_start_docstrings( + "The bare Mixtral Model outputting raw hidden-states without any specific head on top.", + MIXTRAL_START_DOCSTRING, +) +# copied from transformers.models.mistral.modeling_mistral.MistralModel with MISTRAL->MIXTRAL,Mistral->Mixtral +# TODO @longjie no longer copied from Mistral after static cache +class MixtralModel(MixtralPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MixtralDecoderLayer`] + + Args: + config: MixtralConfig + """ + + def __init__(self, config: MixtralConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [MixtralDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.norm = MixtralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + # Ignore copy + @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, MoeModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one" + ) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + use_legacy_cache = False + if use_cache and not isinstance(past_key_values, Cache) and not self.training: + use_legacy_cache = True + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + logger.warning_once( + "We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. " + "Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)" + ) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_router_logits = () if output_router_logits else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + output_router_logits, + use_cache, + cache_position, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + output_router_logits=output_router_logits, + use_cache=use_cache, + cache_position=cache_position, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if output_router_logits: + all_router_logits += (layer_outputs[-1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache + + if not return_dict: + return tuple( + v + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits] + if v is not None + ) + return MoeModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + router_logits=all_router_logits, + ) + + # Copied from transformers.models.llama.modeling_llama.LlamaModel._update_causal_mask + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool, + ): + # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static + # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. + # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using + # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 + + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions: + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + if using_static_cache: + target_length = past_key_values.get_max_length() + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + if attention_mask is not None and attention_mask.dim() == 4: + # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing + if attention_mask.max() != 0: + raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`") + causal_mask = attention_mask + else: + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + if sequence_length != 1: + causal_mask = torch.triu(causal_mask, diagonal=1) + causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type == "cuda" + and not output_attentions + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + +class MixtralForCausalLM(MixtralPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = MixtralModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.router_aux_loss_coef = config.router_aux_loss_coef + self.num_experts = config.num_local_experts + self.num_experts_per_tok = config.num_experts_per_tok + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + # Ignore copy + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, MoeCausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, MixtralForCausalLM + + >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1") + >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + aux_loss = None + if output_router_logits: + aux_loss = load_balancing_loss_func( + outputs.router_logits if return_dict else outputs[-1], + self.num_experts, + self.num_experts_per_tok, + attention_mask, + ) + if labels is not None: + loss += self.router_aux_loss_coef * aux_loss.to(loss.device) # make sure to reside in the same device + + if not return_dict: + output = (logits,) + outputs[1:] + if output_router_logits: + output = (aux_loss,) + output + return (loss,) + output if loss is not None else output + + return MoeCausalLMOutputWithPast( + loss=loss, + aux_loss=aux_loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + router_logits=outputs.router_logits, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + output_router_logits=False, + position_ids=None, + use_cache=True, + **kwargs, + ): + # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens + # Exception 1: when passing input_embeds, input_ids may be missing entries + # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here + if past_key_values is not None: + if inputs_embeds is not None: # Exception 1 + input_ids = input_ids[:, -cache_position.shape[0] :] + elif input_ids.shape[1] != cache_position.shape[0]: # Default case (the "else", a no op, is Exception 2) + input_ids = input_ids[:, cache_position] + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and cache_position[0] == 0: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids.contiguous()} # `contiguous()` needed for compilation use cases + + model_inputs.update( + { + "position_ids": position_ids, + "cache_position": cache_position, + "past_key_values": past_key_values, + "use_cache": use_cache, + "attention_mask": attention_mask, + "output_router_logits": output_router_logits, + } + ) + return model_inputs + + +@add_start_docstrings( + """ + The Mixtral Model transformer with a sequence classification head on top (linear layer). + + [`MixtralForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + MIXTRAL_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Mixtral, LLAMA->MIXTRAL +class MixtralForSequenceClassification(MixtralPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = MixtralModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility + sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 + sequence_lengths = sequence_lengths % input_ids.shape[-1] + sequence_lengths = sequence_lengths.to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + +@add_start_docstrings( + """ + The Mixtral Model transformer with a token classification head on top (a linear layer on top of the hidden-states + output) e.g. for Named-Entity-Recognition (NER) tasks. + """, + MIXTRAL_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaForTokenClassification with Llama->Mixtral, LLAMA->MIXTRAL +class MixtralForTokenClassification(MixtralPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = MixtralModel(config) + if getattr(config, "classifier_dropout", None) is not None: + classifier_dropout = config.classifier_dropout + elif getattr(config, "hidden_dropout", None) is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.score = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + sequence_output = self.dropout(sequence_output) + logits = self.score(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) \ No newline at end of file diff --git a/ktransformers/operators/RoPE.py b/ktransformers/operators/RoPE.py index 5fcce4f..9dc233b 100644 --- a/ktransformers/operators/RoPE.py +++ b/ktransformers/operators/RoPE.py @@ -10,6 +10,7 @@ from ktransformers.util.custom_gguf import GGUFLoader from ktransformers.util.utils import InferenceState from transformers.configuration_utils import PretrainedConfig + # Copied from transformers.models.mixtral.modeling_mixtral.MixtralRotaryEmbedding with Mixtral->Qwen2Moe class RotaryEmbedding(BaseInjectedModule, DeepseekV2RotaryEmbedding): def __init__(self, @@ -17,12 +18,16 @@ def __init__(self, gguf_loader : GGUFLoader, config: PretrainedConfig, orig_module: nn.Module, - device: str = "cuda", + # device: str = "cuda", + generate_device: str = "cuda", + prefill_device: str = "cuda", **kwargs): - BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) + BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) self.orig_module.__init__(orig_module.dim, orig_module.max_position_embeddings, orig_module.base) + self.generate_device = generate_device + self.prefill_device = prefill_device def load(self): self.orig_module.__init__(self.orig_module.dim, @@ -36,9 +41,11 @@ def __init__(self, gguf_loader : GGUFLoader, config: PretrainedConfig, orig_module: nn.Module, - device: str = "cuda", + # device: str = "cuda", + generate_device: str = "cuda", + prefill_device: str = "cuda", **kwargs): - BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) + BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) self.orig_module.__init__(orig_module.dim, orig_module.max_position_embeddings, orig_module.base, @@ -49,13 +56,15 @@ def __init__(self, orig_module.beta_slow, orig_module.mscale, orig_module.mscale_all_dim) + self.generate_device = generate_device + self.prefill_device = prefill_device def load(self): self.orig_module.__init__(self.orig_module.dim, self.orig_module.max_position_embeddings, self.orig_module.base, - self.device, + self.generate_device, self.orig_module.scaling_factor, self.orig_module.original_max_position_embeddings, self.orig_module.beta_fast, diff --git a/ktransformers/operators/experts.py b/ktransformers/operators/experts.py index 0369f5f..7028c74 100644 --- a/ktransformers/operators/experts.py +++ b/ktransformers/operators/experts.py @@ -5,8 +5,8 @@ Author : Azure-Tang, Boxin Zhang, chenht2022 Date : 2024-07-25 11:25:24 Version : 0.1.0 -LastEditors : Azure -LastEditTime : 2024-07-26 09:27:41 +LastEditors : kkk1nak0 +LastEditTime : 2024-08-11 12:14:39 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -19,7 +19,9 @@ import sys, os from ktransformers.operators.base_operator import BaseInjectedModule -sys.path.append(os.path.dirname(__file__) + "/../ktransformers_ext/build") +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Release")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Debug")) import cpuinfer_ext from cpuinfer_ext.moe import MOEConfig, MOE import ctypes @@ -78,6 +80,25 @@ def load_weights(self, override_key: str | None = None, device: str = "cpu"): gate_type = self.gguf_loader.tensor_info[key + ".ffn_gate_exps.weight"]["ggml_type"] up_type = self.gguf_loader.tensor_info[key + ".ffn_up_exps.weight"]["ggml_type"] down_type = self.gguf_loader.tensor_info[key + ".ffn_down_exps.weight"]["ggml_type"] + elif key + ".ffn_down.0.weight" in self.gguf_loader.tensor_info: + # for supporting Mixtral-8x7B-Instuct + gate = [] + up = [] + down = [] + for i in range(8): + gatei, upi, downi = f".ffn_gate.{i}.weight", f".ffn_up.{i}.weight", f".ffn_down.{i}.weight" + targets = [gatei, upi, downi] + tensors = self.load_multi(key, targets, device=device) + gate_it, up_it, down_it = tensors[gatei], tensors[upi], tensors[downi] + gate.append(gate_it) + up.append(up_it) + down.append(down_it) + gate = torch.stack(gate) + up = torch.stack(up) + down = torch.stack(down) + gate_type = self.gguf_loader.tensor_info[key + ".ffn_gate.0.weight"]["ggml_type"] + up_type = self.gguf_loader.tensor_info[key + ".ffn_up.0.weight"]["ggml_type"] + down_type = self.gguf_loader.tensor_info[key + ".ffn_down.0.weight"]["ggml_type"] else: raise ValueError(f"Experts {key} not found in gguf_loader") res = {key:{"gate": gate, "up": up, "down": down, "gate_type": gate_type, "up_type": up_type, "down_type": down_type}} @@ -94,7 +115,8 @@ class MLPCPUExperts(MLPExpertsBase): expert_ids_cpu:Tensor = None weights_cpu:Tensor = None output_cpu:Tensor = None - output_gpu:Tensor = None + output_gpu_map:dict = {} # Manage output tensor buffer on different gpu + #stream_map:dict = {} # Manage cuda stream on different gpu CPU_INFER = cpuinfer_ext.CPUInfer(Config().cpu_infer) def __init__( self, @@ -113,81 +135,83 @@ def __init__( self.out_device = out_device def load(self, w: dict | nn.Parameter | tuple | None = None, device:str|None = None, warmup:bool = False): - if device: - assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU, Parameter \"device\" can be cpu or None." - if w is None: w = self.load_weights()[self.key] - self.gate = w["gate"] - self.up = w["up"] - self.down = w["down"] - self.gate_type = w["gate_type"] - self.up_type = w["up_type"] - self.down_type = w["down_type"] - gate_ptr = ctypes.addressof( - ctypes.cast(self.gate.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - up_ptr = ctypes.addressof( - ctypes.cast(self.up.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - down_ptr = ctypes.addressof( - ctypes.cast(self.down.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - # print(self.gate_qtype, self.up_qtype, self.down_qtype) - n_routed_experts = self.n_routed_experts - # n_routed_experts = len(self.orig_module) - moe_config = MOEConfig( - n_routed_experts, - self.config.num_experts_per_tok, - self.config.hidden_size, - self.config.moe_intermediate_size, - 64, - 10, - 1024, - gate_ptr, - up_ptr, - down_ptr, - self.gate_type, - self.up_type, - self.down_type, - 30, # TODO: get from model.dtype - ) - # print(n_routed_experts, hidden_size, moe_intermediate_size) - num_experts_per_tok = self.config.num_experts_per_tok - self.moe = MOE(moe_config) - self.cpu_infer = MLPCPUExperts.CPU_INFER - if warmup: - self.cpu_infer.submit(self.moe.warm_up()) - self.cpu_infer.sync() - if MLPCPUExperts.output_gpu == None: - MLPCPUExperts.input_tensor_cpu = torch.empty((self.config.hidden_size), device="cpu", pin_memory=True) - MLPCPUExperts.expert_ids_cpu = torch.empty((num_experts_per_tok), device="cpu", dtype=torch.long, pin_memory=True) - MLPCPUExperts.weights_cpu = torch.empty((num_experts_per_tok), device="cpu", dtype=torch.float32, pin_memory=True) - MLPCPUExperts.output_cpu = torch.empty((self.config.hidden_size), device="cpu", pin_memory=True) - MLPCPUExperts.output_gpu = torch.empty((self.config.hidden_size), device=self.out_device) - + with torch.device(self.out_device): + if device: + assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU, Parameter \"device\" can be cpu or None." + if w is None: w = self.load_weights()[self.key] + self.gate = w["gate"] + self.up = w["up"] + self.down = w["down"] + self.gate_type = w["gate_type"] + self.up_type = w["up_type"] + self.down_type = w["down_type"] + gate_ptr = ctypes.addressof( + ctypes.cast(self.gate.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + up_ptr = ctypes.addressof( + ctypes.cast(self.up.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + down_ptr = ctypes.addressof( + ctypes.cast(self.down.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + # print(self.gate_qtype, self.up_qtype, self.down_qtype) + n_routed_experts = self.n_routed_experts + # n_routed_experts = len(self.orig_module) + moe_config = MOEConfig( + n_routed_experts, + self.config.num_experts_per_tok, + self.config.hidden_size, + self.config.moe_intermediate_size, + 64, + 10, + 1024, + gate_ptr, + up_ptr, + down_ptr, + self.gate_type, + self.up_type, + self.down_type, + 30, # TODO: get from model.dtype + ) + # print(n_routed_experts, hidden_size, moe_intermediate_size) + num_experts_per_tok = self.config.num_experts_per_tok + self.moe = MOE(moe_config) + self.cpu_infer = MLPCPUExperts.CPU_INFER + if warmup: + self.cpu_infer.submit(self.moe.warm_up()) + self.cpu_infer.sync() + if self.out_device not in MLPCPUExperts.output_gpu_map: + MLPCPUExperts.output_gpu_map[self.out_device] = torch.zeros((self.config.hidden_size), device=self.out_device) + if MLPCPUExperts.input_tensor_cpu == None: + MLPCPUExperts.input_tensor_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True) + MLPCPUExperts.expert_ids_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.long, pin_memory=True) + MLPCPUExperts.weights_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.float32, pin_memory=True) + MLPCPUExperts.output_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True, dtype=torch.bfloat16) + def submit_for_one_decode(self, input_tensor, expert_ids, weights): MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) - self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(0), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) - + self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream(self.out_device).cuda_stream, self.moe.forward(1, expert_ids.size(0), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) + def sync_for_one_decode(self): - self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) - MLPCPUExperts.output_gpu.copy_(MLPCPUExperts.output_cpu, non_blocking=True) - #print("capturing experts finish") - return MLPCPUExperts.output_gpu + self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream(self.out_device).cuda_stream) + MLPCPUExperts.output_gpu_map[self.out_device].copy_(MLPCPUExperts.output_cpu, non_blocking=True) + return MLPCPUExperts.output_gpu_map[self.out_device] def forward(self, input_tensor, expert_ids, weights): # generate, capture and run cuda graph + # print(expert_ids) if input_tensor.size(0)==1: + # TODO: this branch is unreachable, but the shape of input_tensor([1,hidden_size]) and input_tensor_cpu([hidden_size]) is not compatible #print("capturing experts") MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(1), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) - MLPCPUExperts.output_gpu.copy_(MLPCPUExperts.output_cpu, non_blocking=True) - #print("capturing experts finish") - return MLPCPUExperts.output_gpu + MLPCPUExperts.output_gpu_map[self.out_device].copy_(MLPCPUExperts.output_cpu, non_blocking=True) + return MLPCPUExperts.output_gpu_map[self.out_device] else: input_tensor = input_tensor.contiguous().cpu() expert_ids = expert_ids.contiguous().cpu() @@ -195,7 +219,7 @@ def forward(self, input_tensor, expert_ids, weights): output = torch.empty_like(input_tensor).contiguous() self.cpu_infer.submit(self.moe.forward(expert_ids.size(0), expert_ids.size(1), expert_ids.data_ptr(), weights.data_ptr(), input_tensor.data_ptr(), output.data_ptr())) self.cpu_infer.sync() - return output.to(device=object.__getattribute__(self, "device")) + return output.to(device=object.__getattribute__(self, "out_device")) def unload(self): return @@ -222,6 +246,24 @@ def load_weights(self, override_key: str | None = None, device: str = "cpu"): gate_type = self.gguf_loader.tensor_info[key + ".ffn_gate_exps.weight"]["ggml_type"] up_type = self.gguf_loader.tensor_info[key + ".ffn_up_exps.weight"]["ggml_type"] down_type = self.gguf_loader.tensor_info[key + ".ffn_down_exps.weight"]["ggml_type"] + elif key + ".ffn_down.0.weight" in self.gguf_loader.tensor_info: + # for supporting Mixtral-8x7B-Instuct + gate = [] + up = [] + down = [] + for i in range(8): + gate_it = self.gguf_loader.get_mmap_tensor(f"{key}.ffn_gate.{i}.weight") + up_it = self.gguf_loader.get_mmap_tensor(f"{key}.ffn_up.{i}.weight") + down_it = self.gguf_loader.get_mmap_tensor(f"{key}.ffn_down.{i}.weight") + gate.append(gate_it) + up.append(up_it) + down.append(down_it) + gate = np.stack(gate) + up = np.stack(up) + down = np.stack(down) + gate_type = self.gguf_loader.tensor_info[key + ".ffn_gate.0.weight"]["ggml_type"] + up_type = self.gguf_loader.tensor_info[key + ".ffn_up.0.weight"]["ggml_type"] + down_type = self.gguf_loader.tensor_info[key + ".ffn_down.0.weight"]["ggml_type"] else: raise ValueError(f"Experts {key} not found in gguf_loader") res = {key:{"gate": gate, "up": up, "down": down, "gate_type": gate_type, "up_type": up_type, "down_type": down_type}} @@ -299,7 +341,7 @@ def load_weights(self, override_key: str | None = None): gate_type = self.gguf_loader.tensor_info[key + ".ffn_gate_exps.weight"]["ggml_type"] up_type = self.gguf_loader.tensor_info[key + ".ffn_up_exps.weight"]["ggml_type"] down_type = self.gguf_loader.tensor_info[key + ".ffn_down_exps.weight"]["ggml_type"] - # tensors = self.load_multi(key, [".ffn_gate_exps.weight", ".ffn_up_exps.weight", ".ffn_down_exps.weight"]) + # tensors = self.load_multi(key, [".ffn_gate_exps.weight", ".ffn_up_exps.weight", ".ffn_down_exps.weight"]) res = {key:{"gate": gate, "up": up, "down": down, "gate_type": gate_type, "up_type": up_type, "down_type": down_type}} return res @@ -359,6 +401,11 @@ def unload(self): self.down = None def forward(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch.Tensor, routing_weights_cpu: torch.Tensor) -> torch.Tensor: + + org_device = hidden_states_cpu.device + hidden_states_cpu = hidden_states_cpu.to(self.device) + selected_experts_cpu = selected_experts_cpu.to(self.device) + routing_weights_cpu = routing_weights_cpu.to(self.device) batch_sequence_length, hidden_dim = hidden_states_cpu.size() @@ -388,27 +435,29 @@ def forward(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch.T # the `top_x` tensor here. final_hidden_states.index_add_(0, top_x, current_hidden_states) - return final_hidden_states.to(org_dtype) + + return final_hidden_states.to(org_dtype, device=org_device) EXPERTS_MAP = { "MLPCPUExperts": MLPCPUExperts, "MLPExpertsTorch": MLPExpertsTorch, "MLPExpertsMarlin": MLPExpertsMarlin, } + class KTransformersMLPExpert(BaseInjectedModule, MLPExpertsBase): def __init__(self, key: str, gguf_loader: GGUFLoader, config: PretrainedConfig, orig_module: nn.Module, - device: str = "cuda", + # device: str = "cuda", prefill_device:str = "cuda", prefill_mlp_type: str | None = "MLPExpertsTorch", generate_device: str = "cpu", generate_mlp_type: str | None = "MLPCPUExperts", **kwargs): - BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) - MLPExpertsBase.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) + BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) + MLPExpertsBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) if generate_mlp_type is not None: self.generate_experts = EXPERTS_MAP[generate_mlp_type](key, gguf_loader, config, len(orig_module), device=generate_device, **kwargs) else: @@ -471,6 +520,7 @@ def set_inference_mode(self, mode: InferenceState): from ktransformers.models.modeling_deepseek import DeepseekV2MoE from ktransformers.models.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock +from ktransformers.models.modeling_mixtral import MixtralSparseMoeBlock class Qwen2MoeSparseMoeBlockInjected(BaseInjectedModule, Qwen2MoeSparseMoeBlock): @@ -578,7 +628,6 @@ def moe_infer(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch return final_hidden_states - class DeepseekV2MoEInjected(BaseInjectedModule, DeepseekV2MoE): def forward(self, hidden_states): identity = hidden_states @@ -587,7 +636,7 @@ def forward(self, hidden_states): topk_idx, topk_weight, aux_loss = self.gate(hidden_states) hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) - if sequence_length == 1: + if sequence_length == 1 and hasattr(self.experts.generate_experts, "submit_for_one_decode"): self.experts.generate_experts.submit_for_one_decode(hidden_states[0], topk_idx[0], topk_weight[0]) if self.config.n_shared_experts is not None: y_ = self.shared_experts(identity).squeeze(0) @@ -677,3 +726,102 @@ def moe_infer(self, x, topk_ids, topk_weight): .type(new_x.dtype) ) return final_out + +class MisrtalSparseMoEBlockInjected(BaseInjectedModule, MixtralSparseMoeBlock): + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """ """ + orig_shape = hidden_states.shape + batch_size, sequence_length, hidden_dim = hidden_states.shape + if self.training and self.jitter_noise > 0: + hidden_states *= torch.empty_like(hidden_states).uniform_(1.0 - self.jitter_noise, 1.0 + self.jitter_noise) + hidden_states = hidden_states.view(-1, hidden_dim) + # router_logits: (batch * sequence_length, n_experts) + router_logits = self.gate(hidden_states) + + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + # we cast back to the input dtype + routing_weights = routing_weights.to(hidden_states.dtype) + + if sequence_length == 1 and hasattr(self.experts.generate_experts, "submit_for_one_decode"): + self.experts.generate_experts.submit_for_one_decode(hidden_states[0], selected_experts[0], routing_weights[0]) + y = self.experts.generate_experts.sync_for_one_decode().unsqueeze(0) + y.resize_(*orig_shape) + return y, router_logits + + hidden_states_expert = hidden_states.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else hidden_states_expert.cpu() + selected_experts_expert = selected_experts.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else selected_experts_expert.cpu() + routing_weights_expert = routing_weights.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else routing_weights_expert.cpu() + + if isinstance(self.experts, MLPExpertsBase): + y = ( + self.moe_on_cpuinfer( + hidden_states_expert, selected_experts_expert, routing_weights_expert + ) + .view(*orig_shape) + .to(device=hidden_states.device) + ) + elif hidden_states_expert.size(0) > 10: + y = self.moe_infer( + hidden_states_expert, selected_experts_expert, routing_weights_expert, orig_shape + ).to(device=hidden_states.device) + else: + y = self.moe_infer_simple( + hidden_states_expert, selected_experts_expert, routing_weights_expert + ).to(device=hidden_states.device) + + y.resize_(*orig_shape) + return y, router_logits + + @torch.no_grad() + def moe_on_cpuinfer(self, x: torch.Tensor, topk_ids: torch.Tensor, topk_weight: torch.Tensor) -> torch.Tensor: + outs = torch.empty_like(x) + outs = self.experts(x, topk_ids, topk_weight) + return outs + + @torch.no_grad() + # TODO may bugs here + def moe_infer_simple(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch.Tensor, routing_weights_cpu: torch.Tensor) -> torch.Tensor: + ''' + hidden_states_cpu: [num_tokens, hidden_size] + topk_ids, topk_weight: [num_tokens, num_selected_experts] + ''' + outs = torch.zeros_like(hidden_states_cpu) + for token_idx in range(selected_experts_cpu.size(0)): + for expert_idx in range(selected_experts_cpu.size(1)): + expert = self.experts[selected_experts_cpu[token_idx, expert_idx]] + outs[token_idx] += expert.forward(hidden_states_cpu[token_idx]) * routing_weights_cpu[token_idx, expert_idx] + return outs + + @torch.no_grad() + # TODO may bugs here + def moe_infer(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch.Tensor, routing_weights_cpu: torch.Tensor, orig_shape: tuple) -> torch.Tensor: + + batch_size, sequence_length, hidden_dim = orig_shape + + final_hidden_states = torch.zeros( + (batch_size * sequence_length, hidden_dim), dtype=hidden_states_cpu.dtype, device=hidden_states_cpu.device + ) + + # One hot encode the selected experts to create an expert mask + # this will be used to easily index which expert is going to be sollicitated + expert_mask = torch.nn.functional.one_hot(selected_experts_cpu, num_classes=self.num_experts).permute(2, 1, 0) + + # Loop over all available experts in the model and perform the computation on each expert + for expert_idx in range(self.num_experts): + expert_layer = self.experts[expert_idx] + idx, top_x = torch.where(expert_mask[expert_idx]) + + # Index the correct hidden states and compute the expert hidden state for + # the current expert. We need to make sure to multiply the output hidden + # states by `routing_weights` on the corresponding tokens (top-1 and top-2) + current_state = hidden_states_cpu[None, top_x].reshape(-1, hidden_dim) + current_hidden_states = expert_layer.forward(current_state) * routing_weights_cpu[top_x, idx, None] + + # However `index_add_` only support torch tensors for indexing so we'll use + # the `top_x` tensor here. + final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states_cpu.dtype)) + + return final_hidden_states \ No newline at end of file diff --git a/ktransformers/operators/layer_wise_prefill.py b/ktransformers/operators/layer_wise_prefill.py index 61efed8..2a1d1fe 100644 --- a/ktransformers/operators/layer_wise_prefill.py +++ b/ktransformers/operators/layer_wise_prefill.py @@ -6,7 +6,7 @@ Date : 2024-07-25 11:25:24 Version : 1.0.0 LastEditors : Azure -LastEditTime : 2024-07-26 09:27:48 +LastEditTime : 2024-08-08 10:09:14 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -45,6 +45,8 @@ from transformers.models.qwen2_moe.configuration_qwen2_moe import Qwen2MoeConfig from ktransformers.operators.base_operator import BaseInjectedModule from ktransformers.util.utils import InferenceState +from ktransformers.util.custom_gguf import GGUFLoader +from transformers.configuration_utils import PretrainedConfig if is_flash_attn_2_available(): from flash_attn import flash_attn_func, flash_attn_varlen_func @@ -73,34 +75,6 @@ [`~PreTrainedModel.from_pretrained`] method to load the model weights. """ - -@add_start_docstrings( - "The bare Qwen2MoE Model outputting raw hidden-states without any specific head on top.", - QWEN2MOE_START_DOCSTRING, -) -class Qwen2MoePreTrainedModel(PreTrainedModel): - config_class = Qwen2MoeConfig - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["Qwen2MoeDecoderLayer"] - _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True - _supports_sdpa = True - _supports_cache_class = True - _supports_static_cache = True - - def _init_weights(self, module): - std = self.config.initializer_range - if isinstance(module, nn.Linear): - module.weight.data.normal_(mean=0.0, std=std) - if module.bias is not None: - module.bias.data.zero_() - elif isinstance(module, nn.Embedding): - module.weight.data.normal_(mean=0.0, std=std) - if module.padding_idx is not None: - module.weight.data[module.padding_idx].zero_() - - QWEN2MOE_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -177,13 +151,11 @@ def _init_weights(self, module): the complete sequence length. """ -from ktransformers.util.custom_gguf import GGUFLoader -from transformers.configuration_utils import PretrainedConfig @add_start_docstrings( "The bare Qwen2MoE Model outputting raw hidden-states without any specific head on top.", QWEN2MOE_START_DOCSTRING, ) -class Qwen2MoeModelPerLayerPrefill(BaseInjectedModule): +class Qwen2MoeModelKTransformers(BaseInjectedModule): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2MoeDecoderLayer`] @@ -198,10 +170,13 @@ def __init__( orig_module: nn.Module, device: str = "cuda", per_layer_prefill_intput_threshold: int = 30000, # if None, no per-layer prefill + transfer_map: dict = None, **kwargs, ): BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) self.per_layer_prefill_intput_threshold = per_layer_prefill_intput_threshold + self.transfer_map = transfer_map + self.stream_device_map = dict() @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) def forward( @@ -287,7 +262,20 @@ def forward( all_router_logits = () if output_router_logits else None next_decoder_cache = None - for decoder_layer in self.layers: + for i, decoder_layer in enumerate(self.layers): + if self.transfer_map is not None and i in self.transfer_map: + prev_stream = torch.cuda.current_stream() + cur_device = self.transfer_map[i] + if cur_device not in self.stream_device_map: + self.stream_device_map[cur_device] = torch.cuda.Stream(cur_device) + torch.cuda.set_device(cur_device) + self.stream_device_map[cur_device].wait_stream(prev_stream) + torch.cuda.set_stream(self.stream_device_map[cur_device]) + hidden_states = hidden_states.to(self.transfer_map[i], non_blocking = True) + causal_mask = causal_mask.to(self.transfer_map[i], non_blocking = True) if causal_mask is not None else None + position_ids = position_ids.to(self.transfer_map[i], non_blocking = True) if position_ids is not None else None + cache_position = cache_position.to(self.transfer_map[i], non_blocking = True) if cache_position is not None else None + if output_hidden_states: all_hidden_states += (hidden_states,) @@ -463,7 +451,7 @@ def load_layer_to(self, layer:Qwen2MoeDecoderLayer, target: InferenceState): """ -class DeepseekV2ModelPerLayerPrefill(BaseInjectedModule): +class DeepseekV2ModelKTransformers(BaseInjectedModule): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DeepseekV2DecoderLayer`] @@ -478,10 +466,13 @@ def __init__( orig_module: nn.Module, device: str = "cuda", per_layer_prefill_intput_threshold: int = 30000, # if None, no per-layer prefill + transfer_map: dict = None, **kwargs, ): BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) self.per_layer_prefill_intput_threshold = per_layer_prefill_intput_threshold + self.transfer_map = transfer_map + self.stream_device_map = dict() @add_start_docstrings_to_model_forward(DeepseekV2_INPUTS_DOCSTRING) def forward( @@ -584,7 +575,20 @@ def forward( t_cpu = 0 t_f = 0 - for decoder_layer in self.layers: + for i, decoder_layer in enumerate(self.layers): + if self.transfer_map is not None and i in self.transfer_map: + prev_stream = torch.cuda.current_stream() + cur_device = self.transfer_map[i] + if cur_device not in self.stream_device_map: + self.stream_device_map[cur_device] = torch.cuda.Stream(cur_device) + torch.cuda.set_device(cur_device) + self.stream_device_map[cur_device].wait_stream(prev_stream) + torch.cuda.set_stream(self.stream_device_map[cur_device]) + hidden_states = hidden_states.to(self.transfer_map[i], non_blocking = True) + causal_mask = causal_mask.to(self.transfer_map[i], non_blocking = True) if causal_mask is not None else None + position_ids = position_ids.to(self.transfer_map[i], non_blocking = True) if position_ids is not None else None + cache_position = cache_position.to(self.transfer_map[i], non_blocking = True) if cache_position is not None else None + if output_hidden_states: all_hidden_states += (hidden_states,) diff --git a/ktransformers/operators/linear.py b/ktransformers/operators/linear.py index e264323..90b5506 100644 --- a/ktransformers/operators/linear.py +++ b/ktransformers/operators/linear.py @@ -176,7 +176,7 @@ def __init__( self.act_order = act_order self.is_k_full = is_k_full - def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = "cuda"): + def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = None): if device is None: device = self.device assert device.lower() != "cpu", "Marlin quantized linear only supports GPU device" if w is None: w = self.load_weight(device=device) @@ -200,7 +200,7 @@ def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = weight, self.num_bits, self.group_size, self.act_order ) self.workspace = MarlinWorkspace( - self.out_features, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_MAX_PARALLEL + self.out_features, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_MAX_PARALLEL,self.device ) self.marlin_q_w = marlin_q_w self.marlin_s = marlin_s @@ -247,7 +247,6 @@ def unload(self): LINEAR_MAP = { "QuantizedLinearMarlin": QuantizedLinearMarlin, "QuantizedLinearTorch": QuantizedLinearTorch, - "QuantizedLinearTorch": QuantizedLinearTorch, } class KTransformerLinear(BaseInjectedModule, QuantizedLinearBase): @@ -257,15 +256,15 @@ def __init__( gguf_loader: GGUFLoader, config: PretrainedConfig, orig_module: nn.Module, - device: str = "cuda", + # device: str = "cuda", generate_device: str = "cuda", generate_op: str| None = "QuantizedLinearMarlin", prefill_device: str = "cuda", prefill_op: str| None = "QuantizedLinearTorch", **kwargs, ): - BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) - QuantizedLinearBase.__init__(self, key, gguf_loader, config, orig_module, device, **kwargs) + BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) + QuantizedLinearBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) # build all the linear operators if prefill_op is not None: assert prefill_op in LINEAR_MAP, f"linear_type {prefill_op} not supported" @@ -289,7 +288,6 @@ def __init__( self.generate_linear = LINEAR_MAP[generate_op](key, gguf_loader, config, orig_module, generate_device, **kwargs) else: self.generate_linear = None - self.device = device self.mode = InferenceState.UNLOAD def forward(self, x): diff --git a/ktransformers/optimize/optimize.py b/ktransformers/optimize/optimize.py index 7062166..36ab62d 100644 --- a/ktransformers/optimize/optimize.py +++ b/ktransformers/optimize/optimize.py @@ -1,6 +1,6 @@ ''' Description : -Author : Boxin Zhang +Author : Boxin Zhang, Azure-Tang Version : 0.1.0 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -15,6 +15,7 @@ from ktransformers.util.custom_gguf import GGUFLoader, translate_name_to_gguf from ktransformers.util.utils import set_module, load_weights import itertools +import copy def inject(module, local_optimization_dict, model_config:AutoConfig ,gguf_loader:GGUFLoader, prefix=''): for name, child in module._modules.items(): @@ -22,18 +23,20 @@ def inject(module, local_optimization_dict, model_config:AutoConfig ,gguf_loader child_prefix = prefix + name if child_prefix in local_optimization_dict: inject_module_meta=local_optimization_dict[child_prefix] - if isinstance(inject_module_meta, Mapping): + if inject_module_meta["class"] != "default": import_path = inject_module_meta["class"].split(".") import_module_name = ".".join(import_path[:-1]) + gguf_loader.tensor_device_map[inject_module_meta["key"]] = inject_module_meta["kwargs"] if "kwargs" in inject_module_meta else dict() import_class_name = import_path[-1] module_cls=getattr(__import__(import_module_name, fromlist=[""]), import_class_name) print(f"Injecting {child_prefix} as", import_module_name, ".", import_class_name) - inject_module=module_cls(key = inject_module_meta["key"], gguf_loader = gguf_loader, config = model_config, orig_module=child, device = inject_module_meta["device"], **inject_module_meta["kwargs"]) + inject_module=module_cls(key = inject_module_meta["key"], gguf_loader = gguf_loader, config = model_config, orig_module=child, **inject_module_meta["kwargs"]) set_module(module, name, inject_module) - elif isinstance(inject_module_meta, str): - assert inject_module_meta=="default", "for str inject_module_meta, only support \"default\"." + elif inject_module_meta["class"] == "default": + print(f"Injecting {child_prefix} as default") + gguf_loader.tensor_device_map[inject_module_meta["key"]] = inject_module_meta["kwargs"] if "kwargs" in inject_module_meta else dict() else: - raise Exception("inject_module_meta must be a dict or str") + raise Exception("inject_module_meta[\"class\"] must be \"default\" or a class path") child_prefix += "." child_optimization_dict = {k: v for k, v in local_optimization_dict.items() if k.startswith(child_prefix)} inject(child, child_optimization_dict, model_config, gguf_loader, child_prefix) @@ -57,6 +60,8 @@ def gen_optimize_config(module: nn.Module, out_data: Mapping, rule_list: List, p for rule in rule_list: #print(rule) match_meta = rule["match"] + if "class" not in match_meta and "name" not in match_meta: + raise Exception("match must have at least one of \"class\" and \"name\"") if "class" in match_meta: import_path = match_meta["class"].split(".") import_module_name = ".".join(import_path[:-1]) @@ -67,16 +72,29 @@ def gen_optimize_config(module: nn.Module, out_data: Mapping, rule_list: List, p if "name" in match_meta: if re.search(match_meta["name"], module_name) is None: continue - replace_meta = rule["replace"] - out_data[module_name]={"key": translated_name, - "class": replace_meta["class"], - "device": replace_meta["device"] if "device" in replace_meta else default_device, - "kwargs": replace_meta["kwargs"] if "kwargs" in replace_meta else dict()} + if "replace" not in rule: + raise Exception("replace must be in rule") + if "replace" in rule: + replace_meta = rule["replace"] + if module_name not in out_data: + out_data[module_name]={"key": translated_name, + "class": replace_meta["class"] if "class" in replace_meta else "default", + # "device": replace_meta["device"] if "device" in replace_meta else default_device, + "kwargs": copy.deepcopy(replace_meta["kwargs"]) if "kwargs" in replace_meta else dict()} + else: + if out_data[module_name]["class"] == "default": + out_data[module_name]["class"] = replace_meta["class"] if "class" in replace_meta else "default" + out_data[module_name]["kwargs"].update(copy.deepcopy(replace_meta["kwargs"]) if "kwargs" in replace_meta else dict()) if "recursive" in rule: recursive = bool(rule["recursive"]) if module_name not in out_data: - out_data[module_name]="default" + out_data[module_name]= { + "class": "default", + "key": translated_name, + "kwargs": {"generate_device": default_device, + "prefill_device": default_device} + } #print(out_data[module_name]) #input() @@ -88,6 +106,14 @@ def gen_optimize_config(module: nn.Module, out_data: Mapping, rule_list: List, p gen_optimize_config(child, out_data, rule_list, child_prefix) +def translate_model_config(model_config: PretrainedConfig): + # for supporting some special model + if model_config.model_type == "mixtral": + model_config.moe_intermediate_size = model_config.intermediate_size + + return model_config + + def optimize_and_load_gguf(module: nn.Module, rule_file: str, gguf_path: str, model_config: PretrainedConfig, default_device: str = "cuda:0"): with open(rule_file, 'r', encoding='utf-8') as f: rule_list = yaml.load(f.read(), Loader=yaml.FullLoader) @@ -95,8 +121,11 @@ def optimize_and_load_gguf(module: nn.Module, rule_file: str, gguf_path: str, mo optimize_config = dict() gen_optimize_config(module, optimize_config, rule_list, default_device = default_device) + model_config = translate_model_config(model_config) + gguf_loader=GGUFLoader(gguf_path) with torch.device("meta"): inject(module, optimize_config, model_config, gguf_loader) load_weights(module, gguf_loader) + model_config.gguf_loader = gguf_loader del_meta(module) diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml new file mode 100644 index 0000000..1d6b46f --- /dev/null +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml @@ -0,0 +1,228 @@ +- match: + name: "^model\\.layers\\.([0-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "(^model\\.layers\\.([1][0-9])\\.)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "(^model\\.layers\\.([2][0-9])\\.)" + replace: + class: "default" + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" +- match: + name: "(^model\\.layers\\.([345][0-9])\\.)|(^model.norm)|(^lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" + +- match: + name: "^model\\.layers\\.([0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([1][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model\\.layers\\.([2][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" +- match: + name: "^model\\.layers\\.([345][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" + +- match: + name: "^model\\.layers\\.([1][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\.([1][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\.([2][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\.([345][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" + +- match: + name: "^model\\.layers\\.([0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([1][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model\\.layers\\.([2][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" +- match: + name: "^model\\.layers\\.([345][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" + +- match: + name: "^model\\.layers\\.([0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:0" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:0" + recursive: False # don't recursively inject submodules of this module +- match: + name: "^model\\.layers\\.([1][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:1" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:1" + recursive: False # don't recursively inject submodules of this module +- match: + name: "^model\\.layers\\.([2][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:2" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:2" + recursive: False # don't recursively inject submodules of this module +- match: + name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:3" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:3" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.([0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([1][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model\\.layers\\.([2][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" +- match: + name: "^model\\.layers\\.([345][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" + +- match: + name: "^model$" + replace: + class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + kwargs: + per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill + transfer_map: + 10: "cuda:1" + 20: "cuda:2" + 30: "cuda:3" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml new file mode 100644 index 0000000..45af034 --- /dev/null +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml @@ -0,0 +1,126 @@ +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + +- match: + name: "(^model\\.layers\\.([345][0-9])\\.)|(model.norm)|(lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([345][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" + +- match: + name: "^model\\.layers\\.([345][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([345][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:0" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:0" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:1" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:1" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([345][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model$" + replace: + class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + kwargs: + per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill + transfer_map: + 30: "cuda:1" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml index 025bd2b..328c9d7 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml @@ -1,3 +1,10 @@ +- match: + name: "^model\\.layers\\..*\\.|^lm_head" + replace: + class: "default" + kwargs: + generate_device: "cuda" + prefill_device: "cuda" - match: class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding replace: @@ -21,12 +28,11 @@ name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism - device: "cpu" # which devices to load this module when initializing kwargs: prefill_device: "cuda" prefill_mlp_type: "MLPExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_mlp_type: "MLPCPUExperts" out_device: "cuda" recursive: False # don't recursively inject submodules of this module - match: @@ -36,6 +42,13 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelPerLayerPrefill" + class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml new file mode 100644 index 0000000..c9c1809 --- /dev/null +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml @@ -0,0 +1,126 @@ +- match: + name: "^model\\.layers\\.(0|[1-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + +- match: + name: "(^model\\.layers\\.([12][0-9])\\.)|(model.norm)|(lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" + +- match: + name: "^model\\.layers\\.(0|[1-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([12][0-9])\\." + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" + +- match: + name: "^model\\.layers\\.([12][0-9])\\.(?!self_attn).*$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" + +- match: + name: "^model\\.layers\\.(0|[1-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([12][0-9])\\.mlp$" + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:0" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:0" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.([12][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + kwargs: + prefill_device: "cuda:1" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:1" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.(0|[1-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([12][0-9])\\.self_attn$" + replace: + class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model$" + replace: + class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + kwargs: + per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill + transfer_map: + 10: "cuda:1" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/Mixtral.yaml b/ktransformers/optimize/optimize_rules/Mixtral.yaml new file mode 100644 index 0000000..5bd6705 --- /dev/null +++ b/ktransformers/optimize/optimize_rules/Mixtral.yaml @@ -0,0 +1,45 @@ +- match: + name: "^model\\.layers\\..*\\." + replace: + class: "default" + kwargs: + generate_device: "cuda" + prefill_device: "cuda" +- match: + class: ktransformers.models.modeling_mixtral.MixtralRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.RotaryEmbedding +- match: + name: "^model\\.layers\\..*$" + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda" + prefill_device: "cuda" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\..*\\.block_sparse_moe$" + class: ktransformers.models.modeling_mixtral.MixtralSparseMoeBlock + replace: + class: ktransformers.operators.experts.MisrtalSparseMoEBlockInjected +- match: + name: "^model\\.layers\\..*\\.block_sparse_moe\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert + kwargs: + prefill_device: "cuda" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml new file mode 100644 index 0000000..82415aa --- /dev/null +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml @@ -0,0 +1,111 @@ +- match: + name: "^model\\.layers\\.([012])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([012])\\." + class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.RotaryEmbedding + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "^model\\.layers\\.([012])$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\.([012])\\.mlp$" + class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock + replace: + class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function +- match: + name: "^model\\.layers\\.([012])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + # device: "cpu" # which devices to load this module when initializing + kwargs: + prefill_device: "cuda:0" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:0" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])\\." + class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.RotaryEmbedding + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])$" # regular expression + class: torch.nn.Linear # only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + generate_op: "QuantizedLinearMarlin" + prefill_op: "QuantizedLinearTorch" +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])\\.mlp$" + class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock + replace: + class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + # device: "cpu" # which devices to load this module when initializing + kwargs: + prefill_device: "cuda:1" + prefill_mlp_type: "MLPExpertsTorch" + generate_device: "cpu" + generate_mlp_type: "MLPCPUExperts" + out_device: "cuda:1" + recursive: False # don't recursively inject submodules of this module + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" + +- match: + name: "(^model.norm)|(^lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" + +- match: + name: "^model$" + replace: + class: "ktransformers.operators.layer_wise_prefill.Qwen2MoeModelKTransformers" + kwargs: + per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill + transfer_map: + 3: "cuda:1" + diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml index 2b4e312..3fd59cb 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml @@ -1,3 +1,10 @@ +- match: + name: "^model\\.layers\\..*\\." + replace: + class: "default" + kwargs: + generate_device: "cuda" + prefill_device: "cuda" - match: class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding replace: @@ -21,7 +28,7 @@ name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism - device: "cpu" # which devices to load this module when initializing + # device: "cpu" # which devices to load this module when initializing kwargs: prefill_device: "cuda" prefill_mlp_type: "MLPExpertsTorch" @@ -32,6 +39,13 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.Qwen2MoeModelPerLayerPrefill" + class: "ktransformers.operators.layer_wise_prefill.Qwen2MoeModelKTransformers" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" + prefill_device: "cpu" \ No newline at end of file diff --git a/ktransformers/tests/dequant_gpu.py b/ktransformers/tests/dequant_gpu.py index 4fbca1c..9c839c1 100644 --- a/ktransformers/tests/dequant_gpu.py +++ b/ktransformers/tests/dequant_gpu.py @@ -1,12 +1,9 @@ import os -os.environ["CUDA_VISIBLE_DEVICES"]="1" +# os.environ["CUDA_VISIBLE_DEVICES"]="1,2" # add path import sys current_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(current_path+"/../..") -import pycuda.autoinit -import pycuda.driver as cuda -from pycuda.compiler import SourceModule import numpy as np # from ktransformers.operators.linear import KTransformerLinear, QuantizedLinearMarlin # from ktransformers.operators.experts import KTransformersMLPExpert, MLPExpertsTorch @@ -18,40 +15,44 @@ from transformers import ( AutoConfig, ) +import os +# CUDA_LAUNCH_BLOCKING=1 +os.environ["CUDA_LAUNCH_BLOCKING"]="1" gguf_config = GGUFLoader("/data/Qwen2-57B-A14B-Instruct-GGUF/q4_k_m") model_name = "/data/Qwen2-57B-A14B-Instruct" -key = "blk.0." -target = "ffn_down_exps.weight" + +# Q4k +key = "blk.1." +target = "attn_q.weight" t1 = time.time() q_weight_cpu = gguf_config.load_gguf_tensor(key+target, "cpu") # q_weight_cpu = torch.from_numpy(q_weight_cpu) t2 = time.time() -q_weight_gpu = gguf_config.load_gguf_tensor(key+target, "cuda") +q_weight_gpu = gguf_config.load_gguf_tensor(key+target, "cuda:0") t3 = time.time() print() -allclose = torch.allclose(q_weight_cpu, q_weight_gpu.cpu().to(torch.float32), atol=1e-6) -print(f"Q6k {key+target}") +allclose = torch.allclose(q_weight_cpu, q_weight_gpu.cpu(), atol=1e-6) +print(f"Q4k {key+target}") print("load gguf tensor from cpu cost: ", t2-t1) print("load gguf tensor from gpu cost: ", t3-t2) print("allclose: ", allclose) -key = "blk.1." -target = "ffn_up_shexp.weight" +# Q6k +key = "blk.0." +target = "ffn_down_exps.weight" t1 = time.time() q_weight_cpu = gguf_config.load_gguf_tensor(key+target, "cpu") -# q_weight_cpu = torch.from_numpy(q_weight_cpu) - t2 = time.time() -q_weight_gpu = gguf_config.load_gguf_tensor(key+target, "cuda") +q_weight_gpu = gguf_config.load_gguf_tensor(key+target, "cuda:0") t3 = time.time() print() -allclose = torch.allclose(q_weight_cpu, q_weight_gpu.cpu(), atol=1e-6) -print(f"Q4k {key+target}") +allclose = torch.allclose(q_weight_cpu, q_weight_gpu.cpu().to(torch.float32), atol=1e-6) +print(f"Q6k {key+target}") print("load gguf tensor from cpu cost: ", t2-t1) print("load gguf tensor from gpu cost: ", t3-t2) print("allclose: ", allclose) diff --git a/ktransformers/tests/dequant_gpu_t.py b/ktransformers/tests/dequant_gpu_t.py index 3efcdf3..8abc89d 100644 --- a/ktransformers/tests/dequant_gpu_t.py +++ b/ktransformers/tests/dequant_gpu_t.py @@ -11,7 +11,7 @@ from ktransformers.operators.experts import KTransformersMLPExpert, MLPExpertsTorch from ktransformers.util.custom_gguf import GGUFLoader, dequantize_q4_k_gpu, dequantize_q4_k import torch -import CudaOps +import KTransformersOps torch.set_default_dtype(torch.bfloat16) import time from transformers import ( diff --git a/ktransformers/util/cuda_graph_runner.py b/ktransformers/util/cuda_graph_runner.py index 2ac7a17..c7a9c87 100644 --- a/ktransformers/util/cuda_graph_runner.py +++ b/ktransformers/util/cuda_graph_runner.py @@ -21,6 +21,7 @@ def capture( position_ids, cache_position, past_key_values, + main_device, **kwargs, ) -> None: assert self.graph is None @@ -29,15 +30,24 @@ def capture( self.graph = torch.cuda.CUDAGraph() #self.graph.enable_debug_mode() self.model = model - inputs_embeds = model.model.embed_tokens(cur_token.to("cpu")).to("cuda") - with torch.cuda.graph(self.graph): + inputs_embeds = model.model.embed_tokens(cur_token.to("cpu")).to(main_device) + # torch.cuda.set_device can't set "cuda", must have a index + if main_device == "cuda": + main_device = "cuda:0" + torch.cuda.set_device(main_device) + self.main_device = main_device + capture_stream = torch.cuda.Stream() + with torch.cuda.graph(self.graph, stream = capture_stream): logits=model(inputs_embeds=inputs_embeds, position_ids=position_ids, cache_position=cache_position, past_key_values=past_key_values, **kwargs)[0] + capture_stream.wait_stream(torch.cuda.current_stream()) + torch.cuda.set_device(main_device) + torch.cuda.set_stream(capture_stream) past_key_values.change_seq_length(-1) - torch.cuda.synchronize() + torch.cuda.synchronize(self.main_device) #self.graph.debug_dump("cuda_graph_hooked.dot") # Save the input and output buffers. @@ -65,7 +75,7 @@ def forward( #print("begin replay") #time.sleep(1) self.graph.replay() - torch.cuda.synchronize() + torch.cuda.synchronize(self.main_device) # Return the output tensor. return self.output_buffers["logits"] diff --git a/ktransformers/util/custom_gguf.py b/ktransformers/util/custom_gguf.py index 643713e..fe796a7 100644 --- a/ktransformers/util/custom_gguf.py +++ b/ktransformers/util/custom_gguf.py @@ -5,8 +5,11 @@ Author : Azure-Tang, Boxin Zhang, chenht2022 Date : 2024-07-26 08:48:54 Version : 1.0.0 -LastEditors : Azure -LastEditTime : 2024-07-26 09:28:25 +LastEditors : kkk1nak0 +LastEditTime : 2024-08-09 08:03:44 +Adapted from https://github.com/99991/pygguf/blob/main/gguf.py +Copyright (c) 2023-2024 The ggml authors +Copyright (c) 2024 Thomas Germer Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' # copied from llama.cpp/gguf-py/gguf/constants.py to satisfy dependence of gguf @@ -15,6 +18,7 @@ import struct import warnings import numpy as np +import re import numpy.typing as npt from typing import Sequence import os @@ -96,6 +100,8 @@ def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantization GGML_TYPES = { "F32": 0, "F16": 1, + "Q4_0": 2, + "Q5_0": 6, "Q8_0": 8, "Q2_K": 10, "Q3_K": 11, @@ -109,6 +115,8 @@ def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantization GGML_BLOCK_SIZES = { "F32": 4, "F16": 2, + "Q4_0": 2 + 16, + "Q5_0": 2 + 4 + 16, "Q8_0": 2 + 32, "Q2_K": 256 // 16 + 256 // 4 + 2 + 2, "Q3_K": 256 // 8 + 256 // 4 + 12 + 2, @@ -120,6 +128,8 @@ def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantization GGML_ELEMENTS_PER_BLOCK = { "F32": 1, "F16": 1, + "Q4_0": 32, + "Q5_0": 32, "Q8_0": 32, "Q2_K": 256, "Q3_K": 256, @@ -128,14 +138,6 @@ def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantization "Q6_K": 256, } -# DATA_TYPES = { -# "uint32": 4, -# "int32": 5, -# "float32": 6, -# "string": 8, -# "array": 9, -# "uint64": 10, -# } DATA_TYPES = { "uint8": 0, "int8": 1, @@ -167,6 +169,7 @@ def __init__(self, gguf_path: str): self.tensor_file_map = {} self.file_data_map = {} self.gguf_file_meta = {} + self.tensor_device_map = {} # Walk through all the .gguf files in the directory for root, dirs, files in os.walk(gguf_path): @@ -272,7 +275,7 @@ def get_mmap_tensor(self, name): def load_gguf_tensor(self, name: str, device:str = "cpu")->torch.Tensor: t = self.tensor_info[name] - + shape = t["shape"] ggml_type = t["ggml_type"] @@ -282,15 +285,28 @@ def load_gguf_tensor(self, name: str, device:str = "cpu")->torch.Tensor: ggml_name = GGML_NAMES[ggml_type] data = self.get_mmap_tensor(name) - if "cuda" in device.lower(): values = GGML_DEQUANTIZE_GPU[ggml_name](data, device) + #values = GGML_DEQUANTIZE[ggml_name](data) + #print("load_gguf_tensor") + #values = torch.from_numpy(values).to(device = device) else: values = GGML_DEQUANTIZE[ggml_name](data) values = torch.from_numpy(values) - - return values.view(shape[::-1]) + + values = values.view(shape[::-1]) + if "attn_q" in name and self.gguf_file_meta['general.architecture'] in ["llama"]: + n_head = self.gguf_file_meta['llama.attention.head_count'] + values = (values.reshape(n_head, values.shape[0] // n_head // 2, 2, *values.shape[1:]) + .swapaxes(1, 2) + .reshape(values.shape)) + elif "attn_k" in name and self.gguf_file_meta['general.architecture'] in ["llama"]: + n_head = self.gguf_file_meta['llama.attention.head_count_kv'] + values = (values.reshape(n_head, values.shape[0] // n_head // 2, 2, *values.shape[1:]) + .swapaxes(1, 2) + .reshape(values.shape)) + return values def read_value(f, data_type): if data_type == DATA_TYPES["string"]: @@ -375,7 +391,7 @@ def dequantize_q2_k(data): return d * (scales & 15) * (tmp & 3) - dmin * (scales >> 4) def dequantize_q2_k_gpu(data): - pass + raise NotImplementedError() def dequantize_q3_k(data): # C implementation @@ -420,7 +436,7 @@ def dequantize_q3_k(data): ], axis=1) def dequantize_q3_k_gpu(data): - pass + raise NotImplementedError() def dequantize_q4_k(data): # C implementation @@ -429,20 +445,16 @@ def dequantize_q4_k(data): # https://github.com/ggerganov/ggml/blob/fca1caafea7de9fbd7efc733b9818f9cf2da3050/src/ggml-quants.h#L116 block_size = GGML_BLOCK_SIZES["Q4_K"] num_blocks = len(data) // block_size - data_f16 = np.frombuffer(data, dtype=np.float16).reshape(num_blocks, block_size // 2) data_u8 = np.frombuffer(data, dtype=np.uint8).reshape(num_blocks, block_size) - # Casting to float32 because float16 is very slow on CPU scale_factors = data_f16[:, 0].reshape(num_blocks, 1, 1).astype(np.float32) scale_offsets = data_f16[:, 1].reshape(num_blocks, 1, 1).astype(np.float32) qs1 = data_u8[:, 4:16].reshape(num_blocks, 12, 1) qs2 = data_u8[:, 16:].reshape(num_blocks, 4, 32) - # Dequantize scales and offsets (6 bits and 4 + 2 bits) factors = scale_factors * np.concatenate([qs1[:, 0:4] & 0b111111, (qs1[:, 8:] & 15) | ((qs1[:, 0:4] >> 6) << 4)], axis=1) offsets = scale_offsets * np.concatenate([qs1[:, 4:8] & 0b111111, (qs1[:, 8:] >> 4) | ((qs1[:, 4:8] >> 6) << 4)], axis=1) - # Interleave low and high quantized bits qs2 = np.stack([qs2 & 0xf, qs2 >> 4], axis=2).reshape(num_blocks, 8, 32) # Dequantize final weights using scales and offsets @@ -512,9 +524,14 @@ def dequantize_q5_k(data): d8 * (qs_hi_4[:, 3] + (bits[:, :, 7] << 4)) - m8, ], axis=1) -def dequantize_q5_k_gpu(data): - pass - +def dequantize_q5_k_gpu(data, device:str ="cuda"): + block_size = GGML_BLOCK_SIZES["Q5_K"] + data = np.frombuffer(data, dtype=data.dtype) + device = torch.device(device) + # TODO: this and from_numpy in other functions will cause a warning saying that numpy is not writable, + # the best way to fix this is transfer ptr to KTransformersOps instead of Tensor. + data = torch.from_numpy(data) + return KTransformersOps.dequantize_q5_k(data, block_size, device) def dequantize_q6_k(data): # C implementation @@ -571,7 +588,49 @@ def dequantize_q6_k_gpu(data: np.ndarray, device:str = "cuda"): num_blocks = len(data) // block_size data = np.frombuffer(data, dtype=data.dtype) data = torch.from_numpy(data) - return KTransformersOps.dequantize_q6_k(data, 210, device) + return KTransformersOps.dequantize_q6_k(data, block_size, device) + +def dequantize_q4_0(data): + # C implementation + # https://github.com/ggerganov/ggml/blob/a3c0188a4b5d3dec052ff87c9f773baa53631d70/src/ggml-quants.c#L1515 + # C struct definition + # https://github.com/ggerganov/ggml/blob/a3c0188a4b5d3dec052ff87c9f773baa53631d70/src/ggml-common.h#L141 + num_blocks = len(data) // GGML_BLOCK_SIZES["Q4_0"] + + scales = np.frombuffer(data, dtype=np.float16).reshape(num_blocks, 1 + 8)[:, :1].astype(np.float32) + qs = np.frombuffer(data, dtype=np.uint8).reshape(num_blocks, 2 + 16)[:, 2:] + + return np.concatenate([ + scales * ((qs & 0xf).astype(np.int8) - 8), + scales * ((qs >> 4).astype(np.int8) - 8), + ], axis=1) + +def dequantize_q4_0_gpu(data): + raise NotImplementedError() + +def dequantize_q5_0(data): + # C implementation + # https://github.com/ggerganov/ggml/blob/a3c0188a4b5d3dec052ff87c9f773baa53631d70/src/ggml-quants.c#L1556 + # C struct definition + # https://github.com/ggerganov/ggml/blob/a3c0188a4b5d3dec052ff87c9f773baa53631d70/src/ggml-common.h#L161 + num_blocks = len(data) // GGML_BLOCK_SIZES["Q5_0"] + + scales = np.frombuffer(data, dtype=np.float16).reshape(num_blocks, 1 + 2 + 8)[:, :1].astype(np.float32) + qh = np.frombuffer(data, dtype=np.uint8).reshape(num_blocks, 2 + 4 + 16)[:, 2:2 + 4] + qs = np.frombuffer(data, dtype=np.uint8).reshape(num_blocks, 2 + 4 + 16)[:, 2 + 4:] + + bits = np.unpackbits(qh, axis=-1, bitorder="little") + + x0 = ((qs & 0xf).astype(np.int8) | (bits[:, :16] << 4)) - 16 + x1 = ((qs >> 4).astype(np.int8) | (bits[:, 16:] << 4)) - 16 + + return np.concatenate([ + scales * x0, + scales * x1, + ], axis=1) + +def dequantize_q5_0_gpu(data): + raise NotImplementedError() def dequantize_q8_0(data): # C struct definition @@ -615,6 +674,8 @@ def dequantize_f16_gpu(data, device): GGML_DEQUANTIZE = { "F32": dequantize_f32, "F16": dequantize_f16, + "Q4_0": dequantize_q4_0, + "Q5_0": dequantize_q5_0, "Q8_0": dequantize_q8_0, "Q2_K": dequantize_q2_k, "Q3_K": dequantize_q3_k, @@ -626,6 +687,8 @@ def dequantize_f16_gpu(data, device): GGML_DEQUANTIZE_GPU = { "F32": dequantize_f32_gpu, "F16": dequantize_f16_gpu, + "Q4_0": dequantize_q4_0_gpu, + "Q5_0": dequantize_q5_0_gpu, "Q8_0": dequantize_q8_0_gpu, "Q2_K": dequantize_q2_k_gpu, "Q3_K": dequantize_q3_k_gpu, @@ -634,7 +697,34 @@ def dequantize_f16_gpu(data, device): "Q6_K": dequantize_q6_k_gpu, } + +def translate_name_to_gguf_mixtral(name): + + replacement_template = { + "w1.weight": "ffn_gate", + "w2.weight": "ffn_down", + "w3.weight": "ffn_up" + } + + pattern = re.compile(r"model.layers\.(\d+)\.block_sparse_moe\.experts\.(\d+)\.(w\d\.weight)") + + def replace_match(match): + blk_id = match.group(1) + expert_id = match.group(2) + weight_type = match.group(3) + if weight_type in replacement_template: + return f"blk.{blk_id}.{replacement_template[weight_type]}.{expert_id}.weight" + else: + return match.group(0) + + new_name = re.sub(pattern, replace_match, name) + + return new_name + def translate_name_to_gguf(name): + + name = translate_name_to_gguf_mixtral(name) + name = name.replace("lm_head.", "output.") name = name.replace("model.embed_tokens.", "token_embd.") name = name.replace("model.norm.", "output_norm.") @@ -671,9 +761,14 @@ def translate_name_to_gguf(name): name = name.replace(".mlp.experts.ffn_gate_exps", ".ffn_gate_exps") name = name.replace(".mlp.experts.ffn_up_exps", ".ffn_up_exps") + + name = name.replace(".block_sparse_moe.gate.", ".ffn_gate_inp.") + name = name.replace(".block_sparse_moe.experts", "") + return name if __name__ == '__main__': gguf_path = '/mnt/data/model/DeepSeek-Coder-V2-GGUF-WJH' loader = GGUFLoader(gguf_path) loader.load_gguf_tensor('token_embd.weight') + diff --git a/ktransformers/util/utils.py b/ktransformers/util/utils.py index 7976e56..7993d62 100644 --- a/ktransformers/util/utils.py +++ b/ktransformers/util/utils.py @@ -39,6 +39,22 @@ def set_param(module: nn.Module, name: str, weights: torch.Tensor): param.unsqueeze_(0) setattr(module, name, param) +def get_device(gguf_module_key:str, device_map:dict): + if gguf_module_key in device_map: + return device_map[gguf_module_key]["generate_device"] + else: + return "cuda" + +def get_all_used_cuda_device(device_map:dict): + all_device_list = set() + for key in device_map: + all_device_list.add(device_map[key]["generate_device"]) if "generate_device" in device_map[key] else None + all_device_list.add(device_map[key]["prefill_device"]) if "prefill_device" in device_map[key] else None + if "cpu" in all_device_list: + all_device_list.remove("cpu") + all_device_list = list(all_device_list) + return all_device_list + def load_cur_state_dict(module: nn.Module, gguf_loader: GGUFLoader, prefix: str = ""): prefix = prefix.replace("orig_module.", "") persistent_buffers = {k: v for k, v in module._buffers.items() if k not in module._non_persistent_buffers_set} @@ -47,18 +63,19 @@ def load_cur_state_dict(module: nn.Module, gguf_loader: GGUFLoader, prefix: str for name, param in local_state.items(): key = prefix + name translated_key = translate_name_to_gguf(key) - print("default loading weights", key, translated_key) if translated_key in gguf_loader.tensor_file_map: target_dtype = torch.get_default_dtype() - device = "cpu" if "embd" in translated_key else "cuda" + device = get_device(translated_key[:translated_key.rfind(".")], gguf_loader.tensor_device_map) + print(f"loading {translated_key} to {device}") + # device = "cpu" if "embd" in translated_key else "cuda" weights = gguf_loader.load_gguf_tensor(translated_key, device = device).to(dtype = target_dtype) set_param(module, name, weights) del weights else: #print(load_config.tensor_file_map.keys()) - raise Exception(f"can't fand {translated_key} in GGUF file!") + raise Exception(f"can't find {translated_key} in GGUF file!") -def load_weights(module:nn.Module, gguf_loader:GGUFLoader, prefix='', return_when_injected:bool = False, only_load_injected:bool = False): +def load_weights(module:nn.Module, gguf_loader:GGUFLoader, prefix=''): # print(f"recursively loading weights {prefix},{return_when_injected=}, {only_load_injected=}") if not isinstance(module, base_operator.BaseInjectedModule): load_cur_state_dict(module, gguf_loader, prefix) @@ -66,27 +83,36 @@ def load_weights(module:nn.Module, gguf_loader:GGUFLoader, prefix='', return_whe load_weights(child, gguf_loader, prefix+name+".") else: module.load() - -def prefill_and_generate(model, tokenizer, inputs, max_new_tokens=10000): + +def prefill_and_generate(model, tokenizer, inputs, max_new_tokens=10000, use_cuda_graph: bool = True): import os os.environ["TOKENIZERS_PARALLELISM"] = "false" torch._dynamo.config.suppress_errors = True batch_size, seq_length = inputs.shape - torch_device = inputs.device + device_map = model.config.gguf_loader.tensor_device_map + torch_device = get_device('blk.0.self_attn', device_map) + torch_device = "cuda:0" if torch_device == "cuda" else torch_device + inputs = inputs.to(torch_device) + all_cuda_device = get_all_used_cuda_device(device_map) + tokens = [] - def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position, past_key_values): - logits = cuda_graph_runner(cur_token, position_ids, cache_position) + def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position, past_key_values, use_cuda_graph: bool = True): + if use_cuda_graph: + logits = cuda_graph_runner(cur_token, position_ids, cache_position) + else: + # custom_stream = torch.cuda.Stream() + torch.cuda.set_device(torch_device) + inputs_embeds = model.model.embed_tokens(cur_token.to("cpu")).to(torch_device) + # with torch.cuda.stream(custom_stream): + logits=model(inputs_embeds=inputs_embeds, + position_ids=position_ids, + cache_position=cache_position, + past_key_values=past_key_values, + return_dict=False, use_cache=True)[0] past_key_values.change_seq_length(1) - """ - with torch.cuda.stream(custom_stream): - logits=model(cur_token, - position_ids=position_ids, - cache_position=cache_position, - past_key_values=past_key_values, - return_dict=False, use_cache=True)[0] - #""" - torch.cuda.synchronize() + for device in all_cuda_device: + torch.cuda.synchronize(device) #print(logits) next_token_scores = logits_warper(inputs, logits[:, -1, :]) if generation_config.do_sample: @@ -95,11 +121,12 @@ def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position else: next_token = torch.argmax(next_token_scores, dim=-1) return next_token - + + torch.cuda.set_device(torch_device) with torch.no_grad(): stream = TextStreamer(tokenizer) past_key_values = StaticCache( - config = model.config, max_batch_size = 1, max_cache_len = seq_length + max_new_tokens, device = torch_device, dtype = model.dtype + config = model.config, max_batch_size = 1, max_cache_len = seq_length + max_new_tokens, device = device_map, dtype = model.dtype ) cache_position = torch.arange(seq_length, device=torch_device) generated_ids = torch.zeros( @@ -108,23 +135,22 @@ def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position generated_ids[:, cache_position] = inputs.to(torch_device).to(torch.int) past_key_values.cur_idx=cache_position start_time = time.time() - #custom_stream = torch.cuda.Stream() - inputs_embeds = model.model.embed_tokens(inputs.to("cpu")).to("cuda") + inputs_embeds = model.model.embed_tokens(inputs.to("cpu")).to(torch_device) logits = model( inputs_embeds = inputs_embeds, cache_position=cache_position, past_key_values=past_key_values, return_dict=False, use_cache=True - )[0][:,-1,:].unsqueeze(0).clone() + )[0][:,-1,:].unsqueeze(0).clone().to(torch_device) generation_config, model_kwargs = model._prepare_generation_config( None, max_length=max_new_tokens, do_sample=True, top_k=5, top_p=0.85, temperature=0.1 # change this to modify generate config ) try: # transformers==4.43 logits_warper = ( - model._get_logits_warper(generation_config,device=inputs.device) if generation_config.do_sample else None + model._get_logits_warper(generation_config,device=inputs.device) ) except: logits_warper = ( - model._get_logits_warper(generation_config) if generation_config.do_sample else None + model._get_logits_warper(generation_config) ) next_token_scores = logits_warper(inputs, logits[:, -1, :]) if generation_config.do_sample: @@ -136,7 +162,6 @@ def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position prefill_count = seq_length prefill_time = first_token_time - print(stream.put(next_token.item()), end="", flush=True) generated_ids[:, seq_length] = next_token tokens.append(next_token) @@ -144,12 +169,16 @@ def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position cache_position = torch.tensor([seq_length], device=torch_device) position_ids = cache_position.unsqueeze(0) seq_length += 1 - - cuda_graph_runner = CUDAGraphRunner() - cuda_graph_runner.capture(model, next_token.unsqueeze(0), position_ids, cache_position, past_key_values, return_dict=False, use_cache=True) + + if use_cuda_graph: + cuda_graph_runner = CUDAGraphRunner() + cuda_graph_runner.capture(model, next_token.unsqueeze(0), position_ids, cache_position, past_key_values, torch_device, return_dict=False, use_cache=True) + else: + cuda_graph_runner = None + start_time = time.time() for _ in range(1, max_new_tokens): - next_token = decode_one_tokens(cuda_graph_runner, next_token.unsqueeze(0), position_ids, cache_position, past_key_values) + next_token = decode_one_tokens(cuda_graph_runner, next_token.unsqueeze(0), position_ids, cache_position, past_key_values, use_cuda_graph).to(torch_device) inputs = torch.cat((inputs, next_token.unsqueeze(0)), dim=-1) generated_ids[:, cache_position] = next_token.int() tokens.append(next_token.int()) @@ -162,6 +191,7 @@ def decode_one_tokens(cuda_graph_runner, cur_token, position_ids, cache_position print(stream.put(next_token.item()), end="", flush=True) cache_position += 1 position_ids = cache_position.unsqueeze(0) + total_time = time.time() - start_time tokens_generated = len(tokens) diff --git a/pyproject.toml b/pyproject.toml index 8cfe290..863fcb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ requires = [ "setuptools", "torch >= 2.3.0", "ninja", - "packaging" + "packaging", + "cpufeature" ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 1b2d3cf..aeff5f6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ Date : 2024-07-27 16:15:27 Version : 1.0.0 LastEditors : chenxl -LastEditTime : 2024-07-31 09:44:46 +LastEditTime : 2024-08-08 02:45:15 Adapted from: https://github.com/Dao-AILab/flash-attention/blob/v2.6.3/setup.py Copyright (c) 2023, Tri Dao. @@ -19,6 +19,7 @@ import ast import subprocess import platform +import shutil import http.client import urllib.request import urllib.error @@ -27,6 +28,7 @@ import torch.version from wheel.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools import setup, Extension +from cpufeature.extension import CPUFeature from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME class CpuInstructInfo: @@ -67,6 +69,8 @@ def get_platform(self,): """ if sys.platform.startswith("linux"): return f'linux_{platform.uname().machine}' + elif sys.platform == "win32": + return "win_amd64" else: raise ValueError("Unsupported platform: {}".format(sys.platform)) @@ -97,6 +101,15 @@ def get_cpu_instruct(self,): return 'avx2' raise ValueError( "Unsupported cpu Instructions: {}".format(flags_line)) + elif sys.platform == "win32": + if CPUFeature.get("AVX512bw", False): + return 'fancy' + if CPUFeature.get("AVX512f", False): + return 'avx512' + if CPUFeature.get("AVX2", False): + return 'avx2' + raise ValueError( + "Unsupported cpu Instructions: {}".format(str(CPUFeature))) else: raise ValueError("Unsupported platform: {}".format(sys.platform)) @@ -154,7 +167,7 @@ def run(self): wheel_path = os.path.join(self.dist_dir, archive_basename + ".whl") print("Raw wheel path", wheel_path) - os.rename(wheel_filename, wheel_path) + shutil.move(wheel_filename, wheel_path) except (urllib.error.HTTPError, urllib.error.URLError, http.client.RemoteDisconnected): print("Precompiled wheel not found. Building from source...") # If the wheel could not be downloaded, build from source diff --git a/third_party/llamafile/iqk_mul_mat.inc b/third_party/llamafile/iqk_mul_mat.inc index 150a8f9..5e9d688 100644 --- a/third_party/llamafile/iqk_mul_mat.inc +++ b/third_party/llamafile/iqk_mul_mat.inc @@ -22,7 +22,7 @@ #include #include -#if defined __x86_64__ || defined __aarch64__ +#if defined __x86_64__ || defined __aarch64__ || defined(_M_X64) #include "llama.cpp/ggml-impl.h" #include "llama.cpp/ggml-quants.h" @@ -225,7 +225,7 @@ bool iqk_mul_mat_moe(long Nx, long Ny, long ne00, int ne11, int typeA, const voi return true; } -#if defined __x86_64__ +#if defined __x86_64__ || defined(_M_X64) #if defined HAVE_FANCY_SIMD #undef HAVE_FANCY_SIMD @@ -1412,7 +1412,8 @@ template void MulMat::set_functions(MulMat& m) { bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8, int) { - row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00); + if (ne00 % ggml_blck_size(GGML_TYPE_Q8_K) == 0) + row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00); switch (typeA) { case GGML_TYPE_Q2_K: diff --git a/third_party/llamafile/iqk_mul_mat_amd_avx2.cpp b/third_party/llamafile/iqk_mul_mat_amd_avx2.cpp index 9e3de18..bfd12da 100644 --- a/third_party/llamafile/iqk_mul_mat_amd_avx2.cpp +++ b/third_party/llamafile/iqk_mul_mat_amd_avx2.cpp @@ -3,6 +3,6 @@ // Copyrigth 2024 Iwan Kawrakow. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #include "iqk_mul_mat.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/iqk_mul_mat_amd_zen4.cpp b/third_party/llamafile/iqk_mul_mat_amd_zen4.cpp index 4d0a979..f0f439f 100644 --- a/third_party/llamafile/iqk_mul_mat_amd_zen4.cpp +++ b/third_party/llamafile/iqk_mul_mat_amd_zen4.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Iwan Kawrakow. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define iqk_mul_mat iqk_mul_mat_zen4 #define iqk_mul_mat_moe iqk_mul_mat_moe_zen4 #include "iqk_mul_mat.inc" diff --git a/third_party/llamafile/sgemm.cpp b/third_party/llamafile/sgemm.cpp index 7ec34ff..6a7cab4 100644 --- a/third_party/llamafile/sgemm.cpp +++ b/third_party/llamafile/sgemm.cpp @@ -22,19 +22,22 @@ #include "sgemm.h" // #include -#include +// #include // #include #include -#include +// #include #include // #include "llamafile.h" static const struct GemmFuncs { - typeof(llamafile_sgemm)* sgemm; - typeof(llamafile_mixmul)* mixmul; - typeof(llamafile_mixmul_iqk)* iqk_mixmul = iqk_mul_mat_moe_unsupported; + bool (*sgemm)(long, long, long, const void*, long, const void*, long, void*, long, int, int, int, int, int, int, int); + bool (*mixmul)(const struct ggml_compute_params*, const struct ggml_tensor*, const struct ggml_tensor*, const struct ggml_tensor*, struct ggml_tensor*); + bool (*iqk_mixmul)(long, long, long, int, int, const void*, const void*, float*, long, long, const void*, int, int); + // typeof(llamafile_sgemm)* sgemm; + // typeof(llamafile_mixmul)* mixmul; + // typeof(llamafile_mixmul_iqk)* iqk_mixmul = iqk_mul_mat_moe_unsupported; GemmFuncs() { -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) // if (X86_HAVE(AVX)) { // if (X86_HAVE(FMA)) { // if (X86_HAVE(AVX2)) { @@ -86,10 +89,12 @@ static const struct GemmFuncs { // sgemm = llamafile_sgemm_unsupported; // mixmul = llamafile_mixmul_unsupported; // } + #if defined(__AVX__) -#if defined(__FMA__) +#if defined(__FMA__) || (defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))) #if defined(__AVX2__) #if defined(__AVX512F__) + printf("__AVX512F__\n"); #if defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) && defined(__AVX512BF16__) // AMD Zen4+ (2023-) sgemm = llamafile_sgemm_amd_zen4; diff --git a/third_party/llamafile/tinyblas_cpu.h b/third_party/llamafile/tinyblas_cpu.h index f361c0c..962c47c 100644 --- a/third_party/llamafile/tinyblas_cpu.h +++ b/third_party/llamafile/tinyblas_cpu.h @@ -223,7 +223,7 @@ inline float32x4_t badder(float32x4_t a, float b, float32x4_t c, float32x4_t* e) } #endif -#if defined(__FMA__) +#if defined(__FMA__) || (defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) template <> inline __m256 madd(__m256 a, __m256 b, __m256 c) { diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx.cpp index 255f873..5cbf5df 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_avx #include "tinyblas_cpu_mixmul.inc" diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx2.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx2.cpp index 552d1aa..95d44bf 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx2.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx2.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_avx2 #include "tinyblas_cpu_mixmul.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx512f.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx512f.cpp index b5e5183..82ab637 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx512f.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avx512f.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_avx512f #include "tinyblas_cpu_mixmul.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.cpp index c2b2790..2726ac8 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_avxvnni #include "tinyblas_cpu_mixmul.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_fma.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_fma.cpp index 6fd25c9..4d4c4d8 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_fma.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_fma.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_fma #include "tinyblas_cpu_mixmul.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_mixmul_amd_zen4.cpp b/third_party/llamafile/tinyblas_cpu_mixmul_amd_zen4.cpp index aaac6e1..3d478c1 100644 --- a/third_party/llamafile/tinyblas_cpu_mixmul_amd_zen4.cpp +++ b/third_party/llamafile/tinyblas_cpu_mixmul_amd_zen4.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_mixmul llamafile_mixmul_amd_zen4 #include "tinyblas_cpu_mixmul.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm.inc b/third_party/llamafile/tinyblas_cpu_sgemm.inc index c9d1f47..634dc3e 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm.inc +++ b/third_party/llamafile/tinyblas_cpu_sgemm.inc @@ -321,8 +321,8 @@ bool llamafile_sgemm(long m, long n, long k, const void* A, long lda, const void assert(ith < nth); #if QK_K == 256 -#if defined(__x86_64__) -#if defined(__AVX2__) && defined(__FMA__) +#if defined(__x86_64__) || defined(_M_X64) +#if defined(__AVX2__) && (defined(__FMA__) || (defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__)))) // if (X86_CHECK(AVX2) && X86_CHECK(FMA)) { if (Btype == GGML_TYPE_Q8_K && Ctype == GGML_TYPE_F32) { if (iqk_mul_mat(m, n, k * QK_K, Atype, A, B, (float*)C, ldc, ith, nth)) { diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx.cpp index e57eda6..439e55d 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_avx #include "tinyblas_cpu_sgemm.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp index 0e1fe84..4b46f01 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_avx2 #include "tinyblas_cpu_sgemm.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx512f.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx512f.cpp index cafcaa2..16425e4 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx512f.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avx512f.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_avx512f #include "tinyblas_cpu_sgemm.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.cpp index 5d2ddce..a4ac488 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_avxvnni #include "tinyblas_cpu_sgemm.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_fma.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_fma.cpp index 275c9b4..e1559da 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_fma.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_fma.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_fma #include "tinyblas_cpu_sgemm.inc" #endif // __x86_64__ diff --git a/third_party/llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp b/third_party/llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp index 01924a7..f524ba1 100644 --- a/third_party/llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp +++ b/third_party/llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp @@ -3,7 +3,7 @@ // Copyrigth 2024 Mozilla Foundation. // Copyright(c) 2024 by KVCache.AI, All Rights Reserved. -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(_M_X64) #define llamafile_sgemm llamafile_sgemm_amd_zen4 #define iqk_mul_mat iqk_mul_mat_zen4 #include "tinyblas_cpu_sgemm.inc" From 7c4cb520bd935f944312604b4e6a79380da6a3ae Mon Sep 17 00:00:00 2001 From: BITcyman <815207911@qq.com> Date: Mon, 12 Aug 2024 12:53:12 +0000 Subject: [PATCH 3/8] [feature] support q2_k & q3_k dequantize on gpu --- .../ktransformers_ext/cuda/binding.cpp | 6 +- .../cuda/custom_gguf/binding.cpp | 5 + .../cuda/custom_gguf/dequant.cu | 132 +++++++++++++++++- .../ktransformers_ext/cuda/custom_gguf/ops.h | 6 +- ktransformers/util/custom_gguf.py | 22 ++- 5 files changed, 160 insertions(+), 11 deletions(-) diff --git a/ktransformers/ktransformers_ext/cuda/binding.cpp b/ktransformers/ktransformers_ext/cuda/binding.cpp index f17382d..06ec5f3 100644 --- a/ktransformers/ktransformers_ext/cuda/binding.cpp +++ b/ktransformers/ktransformers_ext/cuda/binding.cpp @@ -4,7 +4,7 @@ * @Date : 2024-07-25 13:38:30 * @Version : 1.0.0 * @LastEditors : kkk1nak0 - * @LastEditTime : 2024-08-09 01:45:02 + * @LastEditTime : 2024-08-12 03:05:04 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ @@ -27,6 +27,10 @@ PYBIND11_MODULE(KTransformersOps, m) { py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q4_k", &dequantize_q4_k, "Function to dequantize q4_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q3_k", &dequantize_q3_k, "Function to dequantize q3_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q2_k", &dequantize_q2_k, "Function to dequantize q2_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("gptq_marlin_gemm", &gptq_marlin_gemm, "Function to perform GEMM using Marlin quantization.", py::arg("a"), py::arg("b_q_weight"), py::arg("b_scales"), py::arg("g_idx"), py::arg("perm"), py::arg("workspace"), py::arg("num_bits"), py::arg("size_m"), diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp b/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp index 2cb46fc..70fc606 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/binding.cpp @@ -13,6 +13,7 @@ int test(){ torch::Tensor dequantize_q6_k(torch::Tensor data, int blk_size, torch::Device device); torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device device); +torch::Tensor dequantize_q2_k(torch::Tensor data, int blk_size, torch::Device device); PYBIND11_MODULE(cudaops, m) { m.def("dequantize_q8_0", &dequantize_q8_0, "Function to dequantize q8_0 data.", @@ -23,6 +24,10 @@ PYBIND11_MODULE(cudaops, m) { py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("dequantize_q4_k", &dequantize_q4_k, "Function to dequantize q4_k data.", py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q3_k", &dequantize_q3_k, "Function to dequantize q3_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); + m.def("dequantize_q2_k", &dequantize_q2_k, "Function to dequantize q2_k data.", + py::arg("data"), py::arg("blk_size"), py::arg("device")); m.def("test", &test, "Function to test."); } diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu b/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu index aaa6453..cc5552b 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu @@ -4,7 +4,7 @@ * @Date : 2024-07-25 13:38:30 * @Version : 1.0.0 * @LastEditors : kkk1nak0 - * @LastEditTime : 2024-08-09 07:57:06 + * @LastEditTime : 2024-08-12 04:18:04 * Adapted from https://github.com/ggerganov/ggml/blob/fca1caafea7de9fbd7efc733b9818f9cf2da3050/src/ggml-quants.c * Copyright (c) 2023-2024 The ggml authors * Copyright (c) 2024 by KVCache.AI, All Rights Reserved. @@ -36,6 +36,97 @@ __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t * __restrict_ } } +__global__ void dequantize_q2_k_kernel(int8_t* data, float* output, int blk_size, int num_blocks) { + int global_idx = blockIdx.x * blockDim.x + threadIdx.x; + for (auto block_id=global_idx; block_id(data + block_id * blk_size + 80))); + const float min = __half2float(*(reinterpret_cast(data + block_id * blk_size + 82))); + + const uint8_t * __restrict__ q = (uint8_t*)(data + block_id * blk_size + 16); + + int is = 0; + float dl, ml; + + for (int n = 0; n < 256; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + uint8_t* scales = (uint8_t*)(data + block_id * blk_size + (is++)); + uint8_t sc = *scales; + dl = d * (sc & 0xF); ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *output_blk++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml; + + scales = (uint8_t*)(data + block_id * blk_size + (is++)); + sc = *scales; + + dl = d * (sc & 0xF); ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *output_blk++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml; + + shift += 2; + } + q += 32; + } + } +} + +__global__ void dequantize_q3_k_kernel(int8_t* data, float* output, int blk_size, int num_blocks) { + + int global_idx = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + for (auto block_id=global_idx; block_id(data + block_id * blk_size + 108))); + + const uint8_t * __restrict__ q = (uint8_t*)(data + block_id * blk_size + 32); + const uint8_t * __restrict__ hm = (uint8_t*)(data + block_id * blk_size + 0); + uint8_t m = 1; + + + uint8_t* block_scales = (uint8_t*)(data + block_id * blk_size + 96); + + for (int i = 0; i < 3; i++) { + aux[i] = 0; + for (int j = 0; j < 4; j++) { + aux[i] |= ((uint32_t)block_scales[i * 4 + j]) << (j * 8); + } + } + + uint32_t tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + + int is = 0; + float dl; + for (int n = 0; n < 256; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + + dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) { + *output_blk++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4)); + } + + dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) { + *output_blk++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4)); + } + + shift += 2; + m <<= 1; + } + q += 32; + } + } +} + + __global__ void dequantize_q4_k_kernel(int8_t* data, float* output, int blk_size, int num_blocks) { int global_idx = blockIdx.x * blockDim.x + threadIdx.x; for (auto block_id=global_idx; block_id>>(data_gpu.data_ptr(), output.data_ptr(), blk_size, num_blocks); + + cudaDeviceSynchronize(); + return output; +} + torch::Tensor dequantize_q4_k(torch::Tensor data, int blk_size, torch::Device device) { // data.numel%blk_size should be 0, else raise err int num_blocks = data.numel() / blk_size; @@ -196,8 +305,25 @@ torch::Tensor dequantize_q4_k(torch::Tensor data, int blk_size, torch::Device de return output; } +torch::Tensor dequantize_q3_k(torch::Tensor data, int blk_size, torch::Device device) { + int num_blocks = data.numel() / blk_size; -torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device device) { + auto options = torch::TensorOptions().dtype(torch::kInt8).device(device).memory_format(torch::MemoryFormat::Contiguous); + auto data_gpu = torch::empty({data.numel()}, options); + + data_gpu.copy_(data, false); + + // Create output tensor + auto output = torch::zeros({num_blocks, 256}, torch::dtype(torch::kFloat32).device(device)); + + // Launch kernel + dequantize_q3_k_kernel<<< 512, 256 >>>(data_gpu.data_ptr(), output.data_ptr(), blk_size, num_blocks); + + cudaDeviceSynchronize(); + return output; +} + +torch::Tensor dequantize_q2_k(torch::Tensor data, int blk_size, torch::Device device) { int num_blocks = data.numel() / blk_size; auto options = torch::TensorOptions().dtype(torch::kInt8).device(device).memory_format(torch::MemoryFormat::Contiguous); @@ -209,7 +335,7 @@ torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device de auto output = torch::zeros({num_blocks, 256}, torch::dtype(torch::kFloat32).device(device)); // Launch kernel - dequantize_q5_k_kernel<<< 512, 256 >>>(data_gpu.data_ptr(), output.data_ptr(), blk_size, num_blocks); + dequantize_q2_k_kernel<<< 512, 256 >>>(data_gpu.data_ptr(), output.data_ptr(), blk_size, num_blocks); cudaDeviceSynchronize(); return output; diff --git a/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h b/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h index f5fde87..5196f88 100644 --- a/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h +++ b/ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h @@ -4,7 +4,7 @@ * @Date : 2024-07-22 09:27:55 * @Version : 1.0.0 * @LastEditors : kkk1nak0 - * @LastEditTime : 2024-08-09 01:44:21 + * @LastEditTime : 2024-08-12 03:48:46 * @Copyright (c) 2024 by KVCache.AI, All Rights Reserved. **/ #pragma once @@ -16,4 +16,6 @@ torch::Tensor dequantize_q8_0(torch::Tensor data, int blk_size, torch::Device device); torch::Tensor dequantize_q6_k(torch::Tensor data, int blk_size, torch::Device device); torch::Tensor dequantize_q5_k(torch::Tensor data, int blk_size, torch::Device device); -torch::Tensor dequantize_q4_k(torch::Tensor data, int blk_size, torch::Device device); \ No newline at end of file +torch::Tensor dequantize_q4_k(torch::Tensor data, int blk_size, torch::Device device); +torch::Tensor dequantize_q3_k(torch::Tensor data, int blk_size, torch::Device device); +torch::Tensor dequantize_q2_k(torch::Tensor data, int blk_size, torch::Device device); \ No newline at end of file diff --git a/ktransformers/util/custom_gguf.py b/ktransformers/util/custom_gguf.py index fe796a7..bd5c5b0 100644 --- a/ktransformers/util/custom_gguf.py +++ b/ktransformers/util/custom_gguf.py @@ -6,7 +6,7 @@ Date : 2024-07-26 08:48:54 Version : 1.0.0 LastEditors : kkk1nak0 -LastEditTime : 2024-08-09 08:03:44 +LastEditTime : 2024-08-12 07:21:55 Adapted from https://github.com/99991/pygguf/blob/main/gguf.py Copyright (c) 2023-2024 The ggml authors Copyright (c) 2024 Thomas Germer @@ -390,8 +390,14 @@ def dequantize_q2_k(data): return d * (scales & 15) * (tmp & 3) - dmin * (scales >> 4) -def dequantize_q2_k_gpu(data): - raise NotImplementedError() +def dequantize_q2_k_gpu(data, device:str ="cuda"): + block_size = GGML_BLOCK_SIZES["Q2_K"] + data = np.frombuffer(data, dtype=data.dtype) + device = torch.device(device) + # TODO: this and from_numpy in other functions will cause a warning saying that numpy is not writable, + # the best way to fix this is transfer ptr to KTransformersOps instead of Tensor. + data = torch.from_numpy(data) + return KTransformersOps.dequantize_q2_k(data, block_size, device) def dequantize_q3_k(data): # C implementation @@ -435,8 +441,14 @@ def dequantize_q3_k(data): (((qs[:, 48:64] >> 6) & 3) - bits[:, 16:, 7]) ], axis=1) -def dequantize_q3_k_gpu(data): - raise NotImplementedError() +def dequantize_q3_k_gpu(data, device:str ="cuda"): + block_size = GGML_BLOCK_SIZES["Q3_K"] + data = np.frombuffer(data, dtype=data.dtype) + device = torch.device(device) + # TODO: this and from_numpy in other functions will cause a warning saying that numpy is not writable, + # the best way to fix this is transfer ptr to KTransformersOps instead of Tensor. + data = torch.from_numpy(data) + return KTransformersOps.dequantize_q3_k(data, block_size, device) def dequantize_q4_k(data): # C implementation From 412055d450fd870d65caf6b17589f46df56b19af Mon Sep 17 00:00:00 2001 From: Atream Date: Wed, 14 Aug 2024 16:10:54 +0800 Subject: [PATCH 4/8] [feature] experts can be injected using CPUInfer [fix] fix ktransformers interface when use new CUDAGraphRunner [fix] fix YAML and optimize logic, the top rule has the highest priority --- ktransformers/operators/cpuinfer.py | 18 +++ ktransformers/operators/experts.py | 94 +++++++------- ktransformers/operators/linear.py | 119 +++++++++++++++++- ktransformers/optimize/optimize.py | 5 +- .../DeepSeek-V2-Chat-multi-gpu-4.yaml | 62 ++++----- .../DeepSeek-V2-Chat-multi-gpu.yaml | 34 ++--- .../optimize_rules/DeepSeek-V2-Chat.yaml | 31 +++-- .../DeepSeek-V2-Lite-Chat-multi-gpu.yaml | 34 ++--- .../optimize/optimize_rules/Mixtral.yaml | 16 ++- .../Qwen2-57B-A14B-Instruct-multi-gpu.yaml | 29 ++--- .../Qwen2-57B-A14B-Instruct.yaml | 20 +-- .../backend/interfaces/ktransformers.py | 6 +- ktransformers/util/utils.py | 2 +- 13 files changed, 315 insertions(+), 155 deletions(-) create mode 100644 ktransformers/operators/cpuinfer.py diff --git a/ktransformers/operators/cpuinfer.py b/ktransformers/operators/cpuinfer.py new file mode 100644 index 0000000..027cc8b --- /dev/null +++ b/ktransformers/operators/cpuinfer.py @@ -0,0 +1,18 @@ +import sys, os +from typing import Any +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Release")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Debug")) +import cpuinfer_ext +from ktransformers.server.config.config import Config +class CPUInfer: + cpu_infer = None + def __init__(self, cpu_infer:int = Config().cpu_infer): + if CPUInfer.cpu_infer is None: + CPUInfer.cpu_infer = cpuinfer_ext.CPUInfer(cpu_infer) + + def __getattribute__(self, __name: str) -> Any: + return CPUInfer.cpu_infer.__getattribute__(__name) + + def __setattr__(self, __name: str, __value: Any) -> None: + return CPUInfer.cpu_infer.__setattr__(__name, __value) \ No newline at end of file diff --git a/ktransformers/operators/experts.py b/ktransformers/operators/experts.py index 7028c74..75fb729 100644 --- a/ktransformers/operators/experts.py +++ b/ktransformers/operators/experts.py @@ -33,6 +33,7 @@ from abc import ABC, abstractmethod from ktransformers.operators.linear import QuantizedLinearMarlin, QuantizedLinearTorch, KTransformerLinear import time +from ktransformers.operators.cpuinfer import CPUInfer # class Base(BaseInjectedModule, ABC): @@ -117,7 +118,7 @@ class MLPCPUExperts(MLPExpertsBase): output_cpu:Tensor = None output_gpu_map:dict = {} # Manage output tensor buffer on different gpu #stream_map:dict = {} # Manage cuda stream on different gpu - CPU_INFER = cpuinfer_ext.CPUInfer(Config().cpu_infer) + CPU_INFER = CPUInfer(Config().cpu_infer) def __init__( self, key: str, @@ -126,7 +127,7 @@ def __init__( n_routed_experts: int, orig_module: nn.Module = None, device: str = "cpu", - out_device: str = "cuda", # this device mean which device the output should on + out_device: str = "cuda", # this device mean which device the output should on. TODO: support cpu. **kwargs ): super().__init__(key, gguf_loader, config, orig_module, device, **kwargs) @@ -135,51 +136,50 @@ def __init__( self.out_device = out_device def load(self, w: dict | nn.Parameter | tuple | None = None, device:str|None = None, warmup:bool = False): - with torch.device(self.out_device): - if device: - assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU, Parameter \"device\" can be cpu or None." - if w is None: w = self.load_weights()[self.key] - self.gate = w["gate"] - self.up = w["up"] - self.down = w["down"] - self.gate_type = w["gate_type"] - self.up_type = w["up_type"] - self.down_type = w["down_type"] - gate_ptr = ctypes.addressof( - ctypes.cast(self.gate.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - up_ptr = ctypes.addressof( - ctypes.cast(self.up.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - down_ptr = ctypes.addressof( - ctypes.cast(self.down.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents - ) - # print(self.gate_qtype, self.up_qtype, self.down_qtype) - n_routed_experts = self.n_routed_experts - # n_routed_experts = len(self.orig_module) - moe_config = MOEConfig( - n_routed_experts, - self.config.num_experts_per_tok, - self.config.hidden_size, - self.config.moe_intermediate_size, - 64, - 10, - 1024, - gate_ptr, - up_ptr, - down_ptr, - self.gate_type, - self.up_type, - self.down_type, - 30, # TODO: get from model.dtype - ) - # print(n_routed_experts, hidden_size, moe_intermediate_size) - num_experts_per_tok = self.config.num_experts_per_tok - self.moe = MOE(moe_config) - self.cpu_infer = MLPCPUExperts.CPU_INFER - if warmup: - self.cpu_infer.submit(self.moe.warm_up()) - self.cpu_infer.sync() + if device: + assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU, Parameter \"device\" can be cpu or None." + if w is None: w = self.load_weights()[self.key] + self.gate = w["gate"] + self.up = w["up"] + self.down = w["down"] + self.gate_type = w["gate_type"] + self.up_type = w["up_type"] + self.down_type = w["down_type"] + gate_ptr = ctypes.addressof( + ctypes.cast(self.gate.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + up_ptr = ctypes.addressof( + ctypes.cast(self.up.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + down_ptr = ctypes.addressof( + ctypes.cast(self.down.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + # print(self.gate_qtype, self.up_qtype, self.down_qtype) + n_routed_experts = self.n_routed_experts + # n_routed_experts = len(self.orig_module) + moe_config = MOEConfig( + n_routed_experts, + self.config.num_experts_per_tok, + self.config.hidden_size, + self.config.moe_intermediate_size, + 64, + 10, + 1024, + gate_ptr, + up_ptr, + down_ptr, + self.gate_type, + self.up_type, + self.down_type, + 30, # TODO: get from model.dtype + ) + # print(n_routed_experts, hidden_size, moe_intermediate_size) + num_experts_per_tok = self.config.num_experts_per_tok + self.moe = MOE(moe_config) + self.cpu_infer = MLPCPUExperts.CPU_INFER + if warmup: + self.cpu_infer.submit(self.moe.warm_up()) + self.cpu_infer.sync() if self.out_device not in MLPCPUExperts.output_gpu_map: MLPCPUExperts.output_gpu_map[self.out_device] = torch.zeros((self.config.hidden_size), device=self.out_device) if MLPCPUExperts.input_tensor_cpu == None: diff --git a/ktransformers/operators/linear.py b/ktransformers/operators/linear.py index 90b5506..e984a90 100644 --- a/ktransformers/operators/linear.py +++ b/ktransformers/operators/linear.py @@ -11,8 +11,9 @@ ''' +import ctypes import torch -from torch import nn +from torch import Tensor, nn import KTransformersOps from ktransformers.util.custom_gguf import GGUFLoader from ktransformers.util.utils import InferenceState @@ -25,7 +26,13 @@ from ktransformers.operators.base_operator import BaseInjectedModule from transformers.configuration_utils import PretrainedConfig from abc import ABC, abstractmethod - +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Release")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "ktransformers_ext", "build", "Debug")) +import cpuinfer_ext +from ktransformers.operators.cpuinfer import CPUInfer +from ktransformers.server.config.config import Config #class QuantizedLinearBase(BaseInjectedModule, ABC): class QuantizedLinearBase(ABC): @@ -118,6 +125,7 @@ def __init__( def forward(self, x: torch.Tensor) -> torch.Tensor: dtype = x.dtype out_device = x.device + # TODO: support CUDA Graph when using cpu, but CPUInfer is recommended. x = x.to(device=self.device, dtype=self.dtype) x = x @ self.w if self.has_bias: @@ -128,7 +136,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = None): if device is None: device = self.device if w is None: w = self.load_weight(device=device) - + if isinstance(w, nn.Parameter): self.w = w.to(dtype=self.dtype).view(self.out_features, self.in_features).T self.has_bias = False @@ -243,10 +251,113 @@ def unload(self): self.g_idx = None self.sort_indices = None self.workspace = None - + +class QuantizedLinearCPUInfer(QuantizedLinearBase): + CPU_INFER = CPUInfer(Config().cpu_infer) + def __init__( + self, + key: str, + gguf_loader: GGUFLoader, + config: PretrainedConfig, + orig_module: nn.Module = None, + device: str = "cpu", + out_device: str = "cuda", # this device mean which device the output should on. TODO: support cpu. + stride = 16, + group_max_len = 1024, + **kwargs, + ): + super().__init__(key, gguf_loader, config, orig_module, device, **kwargs) + self.has_bias = False + self.dtype = torch.get_default_dtype() + self.w = None + self.has_bias = False + self.stride = stride + self.group_max_len = group_max_len + self.out_device = out_device + + def forward(self, x: torch.Tensor) -> torch.Tensor: + origin_shape = x.shape # [batch_size, q_len, hidden_size] + if origin_shape[1] == 1: + out_device = x.device + self.input_tensor_cpu.copy_(x, non_blocking=True) + qlen = origin_shape[1] + QuantizedLinearCPUInfer.CPU_INFER.submit_with_cuda_stream( + torch.cuda.current_stream().cuda_stream, + self.linear.forward( + qlen, + self.input_tensor_cpu.data_ptr(), + self.output_cpu.data_ptr() + ) + ) + QuantizedLinearCPUInfer.CPU_INFER.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) + self.output_gpu.copy_(self.output_cpu, non_blocking=True) + if self.has_bias: + self.output_gpu += self.bias + return self.output_gpu + else: + dtype = x.dtype + out_device = x.device + x = x.to(device=self.device) + qlen = origin_shape[1] + output_shape = (*origin_shape[:-1], self.out_features) + output = torch.empty(output_shape, device=x.device, dtype=x.dtype) + QuantizedLinearCPUInfer.CPU_INFER.submit( + self.linear.forward( + qlen, + x.data_ptr(), + output.data_ptr() + ) + ) + QuantizedLinearCPUInfer.CPU_INFER.sync() + if self.has_bias: + output = output + self.bias + output = output.to(dtype=dtype, device=out_device) + return output + + def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = None, warmup:bool = True): + print(f"loading {self.key} to {self.device} using CPUInfer") + if device is None: device = self.device + self.load_weights(w=w, device=device) + if self.bias is not None: + self.has_bias = True + self.bias = self.bias.to(device) + + weight_ptr = ctypes.addressof( + ctypes.cast(self.weight.ctypes.data, ctypes.POINTER(ctypes.c_uint64)).contents + ) + config = cpuinfer_ext.linear.LinearConfig(self.in_features, self.out_features, self.stride, self.group_max_len, weight_ptr, self.weight_type, 30) + self.linear = cpuinfer_ext.linear.Linear(config) + + if warmup: + QuantizedLinearCPUInfer.CPU_INFER.submit(self.linear.warm_up()) + QuantizedLinearCPUInfer.CPU_INFER.sync() + self.input_tensor_cpu = torch.zeros((1, 1, self.in_features), device="cpu", pin_memory=True) + self.output_cpu = torch.zeros((1, 1, self.out_features), device="cpu", pin_memory=True, dtype=torch.bfloat16) + self.output_gpu = torch.zeros((1, 1, self.out_features), device=self.out_device) + + def load_weights(self, w: dict | nn.Parameter | tuple | None = None, device: str = "cpu"): + if self.key + ".weight" in self.gguf_loader.tensor_info: + if self.key + ".bias" in self.gguf_loader.tensor_file_map: + self.weight = self.gguf_loader.get_mmap_tensor(self.key + ".weight") + self.weight_type = self.gguf_loader.tensor_info[self.key + ".weight"]["ggml_type"] + self.bias = self.gguf_loader.load_gguf_tensor(self.key + ".bias", device=device) + else: + self.weight = self.gguf_loader.get_mmap_tensor(self.key + ".weight") + self.weight_type = self.gguf_loader.tensor_info[self.key + ".weight"]["ggml_type"] + self.bias = None + else: + raise ValueError(f"Linear {self.key} not found in gguf_loader") + + def unload(self): + if self.w is not None: + self.w = None + if self.has_bias: + self.bias = None + LINEAR_MAP = { "QuantizedLinearMarlin": QuantizedLinearMarlin, "QuantizedLinearTorch": QuantizedLinearTorch, + "QuantizedLinearCPUInfer": QuantizedLinearCPUInfer } class KTransformerLinear(BaseInjectedModule, QuantizedLinearBase): diff --git a/ktransformers/optimize/optimize.py b/ktransformers/optimize/optimize.py index 36ab62d..32eab01 100644 --- a/ktransformers/optimize/optimize.py +++ b/ktransformers/optimize/optimize.py @@ -58,7 +58,6 @@ def gen_optimize_config(module: nn.Module, out_data: Mapping, rule_list: List, p #print("gen_optimize_config", prefix, module_name, translated_name) recursive = True for rule in rule_list: - #print(rule) match_meta = rule["match"] if "class" not in match_meta and "name" not in match_meta: raise Exception("match must have at least one of \"class\" and \"name\"") @@ -87,6 +86,7 @@ def gen_optimize_config(module: nn.Module, out_data: Mapping, rule_list: List, p out_data[module_name]["kwargs"].update(copy.deepcopy(replace_meta["kwargs"]) if "kwargs" in replace_meta else dict()) if "recursive" in rule: recursive = bool(rule["recursive"]) + break if module_name not in out_data: out_data[module_name]= { @@ -127,5 +127,6 @@ def optimize_and_load_gguf(module: nn.Module, rule_file: str, gguf_path: str, mo with torch.device("meta"): inject(module, optimize_config, model_config, gguf_loader) load_weights(module, gguf_loader) - model_config.gguf_loader = gguf_loader + module.gguf_loader = gguf_loader del_meta(module) + torch.cuda.empty_cache() diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml index 1d6b46f..31c5c87 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml @@ -1,32 +1,3 @@ -- match: - name: "^model\\.layers\\.([0-9])\\." - replace: - class: "default" - kwargs: - generate_device: "cuda:0" - prefill_device: "cuda:0" -- match: - name: "(^model\\.layers\\.([1][0-9])\\.)" - replace: - class: "default" - kwargs: - generate_device: "cuda:1" - prefill_device: "cuda:1" -- match: - name: "(^model\\.layers\\.([2][0-9])\\.)" - replace: - class: "default" - kwargs: - generate_device: "cuda:2" - prefill_device: "cuda:2" -- match: - name: "(^model\\.layers\\.([345][0-9])\\.)|(^model.norm)|(^lm_head)" - replace: - class: "default" - kwargs: - generate_device: "cuda:3" - prefill_device: "cuda:3" - - match: name: "^model.embed_tokens" replace: @@ -69,7 +40,7 @@ prefill_device: "cuda:3" - match: - name: "^model\\.layers\\.([1][0-9])\\.(?!self_attn).*$" # regular expression + name: "^model\\.layers\\.([0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types @@ -225,4 +196,33 @@ transfer_map: 10: "cuda:1" 20: "cuda:2" - 30: "cuda:3" \ No newline at end of file + 30: "cuda:3" + +- match: + name: "^model\\.layers\\.([0-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" +- match: + name: "(^model\\.layers\\.([1][0-9])\\.)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" +- match: + name: "(^model\\.layers\\.([2][0-9])\\.)" + replace: + class: "default" + kwargs: + generate_device: "cuda:2" + prefill_device: "cuda:2" +- match: + name: "(^model\\.layers\\.([345][0-9])\\.)|(^model.norm)|(^lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:3" + prefill_device: "cuda:3" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml index 45af034..15e8e10 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml @@ -1,19 +1,3 @@ -- match: - name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\." - replace: - class: "default" - kwargs: - generate_device: "cuda:0" - prefill_device: "cuda:0" - -- match: - name: "(^model\\.layers\\.([345][0-9])\\.)|(model.norm)|(lm_head)" - replace: - class: "default" - kwargs: - generate_device: "cuda:1" - prefill_device: "cuda:1" - - match: name: "^model.embed_tokens" replace: @@ -123,4 +107,20 @@ kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: - 30: "cuda:1" \ No newline at end of file + 30: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + +- match: + name: "(^model\\.layers\\.([345][0-9])\\.)|(model.norm)|(lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml index 328c9d7..47fe084 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml @@ -1,14 +1,21 @@ - match: - name: "^model\\.layers\\..*\\.|^lm_head" + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding replace: - class: "default" + class: ktransformers.operators.RoPE.YarnRotaryEmbedding kwargs: generate_device: "cuda" prefill_device: "cuda" -- match: - class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding - replace: - class: ktransformers.operators.RoPE.YarnRotaryEmbedding +#- match: +# name: "^model\\.layers\\.([1-5][0-9])\\.mlp\\.shared_experts.*$" # regular expression +# class: torch.nn.Linear # only match modules matching name and class simultaneously +# replace: +# class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types +# kwargs: +# generate_device: "cpu" +# prefill_device: "cuda" +# generate_op: "QuantizedLinearCPUInfer" +# prefill_op: "QuantizedLinearTorch" +# out_device: "cuda" - match: name: "^model\\.layers\\.(?!.*self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously @@ -24,6 +31,9 @@ class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda" + prefill_device: "cuda" - match: name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: @@ -39,16 +49,21 @@ name: "^model\\.layers\\..*\\.self_attn$" replace: class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + kwargs: + generate_device: "cuda" + prefill_device: "cuda" - match: name: "^model$" replace: class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" kwargs: + generate_device: "cuda" + prefill_device: "cuda" per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill - match: name: "^model.embed_tokens" replace: class: "default" kwargs: - generate_device: "cpu" - prefill_device: "cpu" \ No newline at end of file + generate_device: "cpu" + prefill_device: "cpu" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml index c9c1809..e79e4fd 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml @@ -1,19 +1,3 @@ -- match: - name: "^model\\.layers\\.(0|[1-9])\\." - replace: - class: "default" - kwargs: - generate_device: "cuda:0" - prefill_device: "cuda:0" - -- match: - name: "(^model\\.layers\\.([12][0-9])\\.)|(model.norm)|(lm_head)" - replace: - class: "default" - kwargs: - generate_device: "cuda:1" - prefill_device: "cuda:1" - - match: name: "^model.embed_tokens" replace: @@ -123,4 +107,20 @@ kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: - 10: "cuda:1" \ No newline at end of file + 10: "cuda:1" + +- match: + name: "^model\\.layers\\.(0|[1-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + +- match: + name: "(^model\\.layers\\.([12][0-9])\\.)|(model.norm)|(lm_head)" + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/Mixtral.yaml b/ktransformers/optimize/optimize_rules/Mixtral.yaml index 5bd6705..21fdb72 100644 --- a/ktransformers/optimize/optimize_rules/Mixtral.yaml +++ b/ktransformers/optimize/optimize_rules/Mixtral.yaml @@ -1,14 +1,10 @@ - match: - name: "^model\\.layers\\..*\\." + class: ktransformers.models.modeling_mixtral.MixtralRotaryEmbedding replace: - class: "default" + class: ktransformers.operators.RoPE.RotaryEmbedding kwargs: generate_device: "cuda" prefill_device: "cuda" -- match: - class: ktransformers.models.modeling_mixtral.MixtralRotaryEmbedding - replace: - class: ktransformers.operators.RoPE.RotaryEmbedding - match: name: "^model\\.layers\\..*$" class: torch.nn.Linear # only match modules matching name and class simultaneously @@ -43,3 +39,11 @@ kwargs: generate_device: "cpu" prefill_device: "cpu" + +- match: + name: "^model\\.layers\\..*\\." + replace: + class: "default" + kwargs: + generate_device: "cuda" + prefill_device: "cuda" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml index 82415aa..d48ebeb 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml @@ -1,10 +1,3 @@ -- match: - name: "^model\\.layers\\.([012])\\." - replace: - class: "default" - kwargs: - generate_device: "cuda:0" - prefill_device: "cuda:0" - match: name: "^model\\.layers\\.([012])\\." class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding @@ -41,13 +34,6 @@ out_device: "cuda:0" recursive: False # don't recursively inject submodules of this module -- match: - name: "^model\\.layers\\.([12][0-9]|[3-9])\\." - replace: - class: "default" - kwargs: - generate_device: "cuda:1" - prefill_device: "cuda:1" - match: name: "^model\\.layers\\.([12][0-9]|[3-9])\\." class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding @@ -109,3 +95,18 @@ transfer_map: 3: "cuda:1" +- match: + name: "^model\\.layers\\.([012])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + prefill_device: "cuda:0" + +- match: + name: "^model\\.layers\\.([12][0-9]|[3-9])\\." + replace: + class: "default" + kwargs: + generate_device: "cuda:1" + prefill_device: "cuda:1" \ No newline at end of file diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml index 3fd59cb..a48b15a 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml @@ -1,14 +1,10 @@ - match: - name: "^model\\.layers\\..*\\." + class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding replace: - class: "default" + class: ktransformers.operators.RoPE.RotaryEmbedding kwargs: generate_device: "cuda" prefill_device: "cuda" -- match: - class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding - replace: - class: ktransformers.operators.RoPE.RotaryEmbedding - match: name: "^model\\.layers\\..*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously @@ -24,6 +20,9 @@ class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock replace: class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function + kwargs: + generate_device: "cuda" + prefill_device: "cuda" - match: name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: @@ -48,4 +47,11 @@ class: "default" kwargs: generate_device: "cpu" - prefill_device: "cpu" \ No newline at end of file + prefill_device: "cpu" +- match: + name: "^model\\.layers\\..*\\." + replace: + class: "default" + kwargs: + generate_device: "cuda" + prefill_device: "cuda" \ No newline at end of file diff --git a/ktransformers/server/backend/interfaces/ktransformers.py b/ktransformers/server/backend/interfaces/ktransformers.py index 77b0cda..8d121d5 100644 --- a/ktransformers/server/backend/interfaces/ktransformers.py +++ b/ktransformers/server/backend/interfaces/ktransformers.py @@ -6,6 +6,7 @@ from ktransformers.models.custom_cache import StaticCache from ktransformers.util.cuda_graph_runner import CUDAGraphRunner from ktransformers.local_chat import custom_models, default_optimize_rules +from ktransformers.util.utils import get_device class KTransformersThreadContext(TransformersThreadContext): @@ -48,8 +49,11 @@ def __init__(self,args:ConfigArgs= default_args): def decode_one_tokens(self): if not hasattr(self, "cuda_graph_runner"): + device_map = self.model.gguf_loader.tensor_device_map + torch_device = get_device('blk.0.self_attn', device_map) + torch_device = "cuda:0" if torch_device == "cuda" else torch_device self.cuda_graph_runner = CUDAGraphRunner() - self.cuda_graph_runner.capture(self.model, self.current_ids, self.active_cache_position.unsqueeze(0), self.active_cache_position, self.cache, return_dict=False, use_cache=True) + self.cuda_graph_runner.capture(self.model, self.current_ids, self.active_cache_position.unsqueeze(0), self.active_cache_position, self.cache, main_device=torch_device, return_dict=False, use_cache=True) if hasattr(self, "cuda_graph_runner"): logits = self.cuda_graph_runner(self.current_ids, self.active_cache_position.unsqueeze(0), self.active_cache_position) diff --git a/ktransformers/util/utils.py b/ktransformers/util/utils.py index 7993d62..8c91d47 100644 --- a/ktransformers/util/utils.py +++ b/ktransformers/util/utils.py @@ -89,7 +89,7 @@ def prefill_and_generate(model, tokenizer, inputs, max_new_tokens=10000, use_cud os.environ["TOKENIZERS_PARALLELISM"] = "false" torch._dynamo.config.suppress_errors = True batch_size, seq_length = inputs.shape - device_map = model.config.gguf_loader.tensor_device_map + device_map = model.gguf_loader.tensor_device_map torch_device = get_device('blk.0.self_attn', device_map) torch_device = "cuda:0" if torch_device == "cuda" else torch_device inputs = inputs.to(torch_device) From 1db4a67dca09d0a1d5a8c668502b029f47602192 Mon Sep 17 00:00:00 2001 From: chenxl Date: Wed, 14 Aug 2024 16:54:50 +0000 Subject: [PATCH 5/8] [feature] add github action for pre compile --- .github/workflows/package_wheel_release.yml | 252 ++++++++++++++++++++ .github/workflows/package_wheel_test.yml | 132 ++++++++++ ktransformers/__init__.py | 2 +- setup.py | 13 +- third_party/llamafile/sgemm.cpp | 1 - 5 files changed, 396 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/package_wheel_release.yml create mode 100644 .github/workflows/package_wheel_test.yml diff --git a/.github/workflows/package_wheel_release.yml b/.github/workflows/package_wheel_release.yml new file mode 100644 index 0000000..93e5f38 --- /dev/null +++ b/.github/workflows/package_wheel_release.yml @@ -0,0 +1,252 @@ +name: Build Wheels +on: + workflow_dispatch: + inputs: + release: + description: 'Release? 1 = yes, 0 = no' + default: '0' + required: true + type: string +jobs: + build_wheels: + name: ${{ matrix.os }} Python=${{ matrix.pyver }} CUDA=${{ matrix.cuda }} CPU_INSTRUCT=${{ matrix.instruct }} Torch=${{ matrix.torch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Ubuntu + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.11', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: ubuntu-20.04, pyver: '3.10', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + + # Windows + - { os: windows-2022, pyver: '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.11', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.4.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.2.2', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.4.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + - { os: windows-2022, pyver: '3.10', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'} + - { os: windows-2022, pyver: '3.10', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'} + + defaults: + run: + shell: pwsh + + steps: + - uses: actions/checkout@v3 + + - name: Free Disk Space + uses: jlumbroso/free-disk-space@v1.3.1 + if: runner.os == 'Linux' + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: check_space + run: | + if($IsLinux) {df -h} + if($IsWindows) {Get-PSDrive -PSProvider 'FileSystem'} + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Setup Mamba + if: matrix.cuda != '' + uses: conda-incubator/setup-miniconda@v2.3.0 + with: + activate-environment: "ktransformers" + python-version: ${{ matrix.pyver }} + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + add-pip-as-python-dependency: true + auto-activate-base: false + + + + - name: build web + run: | + cd ktransformers/website/ + npm install + npm run build + cd ../../ + + - name: build for cuda + if: matrix.cuda != '' + run: | + git submodule init + git submodule update + if($IsWindows){ + $originalPath = Get-Location + Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll' + Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -DevCmdArguments '-arch=x64 -host_arch=x64' + $env:DISTUTILS_USE_SDK=1 + Set-Location $originalPath + } + $cudaVersion = '${{ matrix.cuda }}' + $env:MAMBA_NO_LOW_SPEED_LIMIT = 1 + mamba install -y -c nvidia/label/cuda-$cudaVersion cuda-toolkit cuda-runtime + $env:CUDA_PATH = $env:CONDA_PREFIX + $env:CUDA_HOME = $env:CONDA_PREFIX + if ($IsLinux) { + $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH + $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib/python${{ matrix.pyver }}/site-packages/nvidia/nvjitlink/lib:' + $env:LD_LIBRARY_PATH + if (!(Test-Path $env:CUDA_HOME/lib64)) { + New-Item -ItemType SymbolicLink -Path $env:CUDA_HOME/lib64 -Target $env:CUDA_HOME/lib + } + } + if ($IsWindows) { + $env:CUDA_PATH = "$env:CUDA_PATH/Library" + $env:CUDA_HOME = $env:CUDA_PATH + $env:PATH = "$env:CUDA_PATH/bin;" + $env:PATH + cp $env:CUDA_PATH/lib/*.lib $env:CUDA_PATH/lib/x64/ + $env:INCLUDE =$env:CUDA_PATH + "/include/targets/x64;" + $env:INCLUDE + + } + python -m pip install torch==${{ matrix.torch }} torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${{ matrix.torch_cu }} + python -m pip install cpufeature build wheel ninja packaging setuptools + $env:KTRANSFORMERS_FORCE_BUILD = "TRUE" + $env:CPU_INSTRUCT = '${{ matrix.instruct }}' + $env:TORCH_CUDA_ARCH_LIST = '${{ matrix.cudaarch }}' + python -m build --no-isolation --verbose + + + - name: create Rlease dir + run: | + if ($IsWindows) { + $env:date = $(Get-Date -Format "yyyy-MM-dd") + New-Item -ItemType Directory -Force -Path "$Env:USERPROFILE\.ssh" + $Env:SSH_PATH = "$Env:USERPROFILE\.ssh\id_rsa" + Set-Content -Path $Env:SSH_PATH -Value "${{ secrets.SSH_PRIVATE_KEY }}" + (Get-Content -Path $Env:SSH_PATH).Replace("`r`n","`n") | Set-Content -Path $Env:SSH_PATH + chmod 600 $Env:SSH_PATH + } + if ($IsLinux) { + $env:date = $(date +%Y-%m-%d) + mkdir -p ~/.ssh/ + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + } + + ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no root@${{ secrets.SSH_SERVER }} "mkdir -p /mnt/data/release-$env:date" + scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no dist/*.whl root@${{ secrets.SSH_SERVER }}:/mnt/data/release-$env:date/ \ No newline at end of file diff --git a/.github/workflows/package_wheel_test.yml b/.github/workflows/package_wheel_test.yml new file mode 100644 index 0000000..9fe82f8 --- /dev/null +++ b/.github/workflows/package_wheel_test.yml @@ -0,0 +1,132 @@ +name: Build Wheels +on: + workflow_dispatch: + inputs: + release: + description: 'Release? 1 = yes, 0 = no' + default: '0' + required: true + type: string +jobs: + build_wheels: + name: ${{ matrix.os }} Python=${{ matrix.pyver }} CUDA=${{ matrix.cuda }} CPU_INSTRUCT=${{ matrix.instruct }} Torch=${{ matrix.torch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Ubuntu + - { os: ubuntu-20.04, pyver: '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'} + - { os: windows-2022, pyver: '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'} + + defaults: + run: + shell: pwsh + + steps: + - uses: actions/checkout@v3 + + - name: Free Disk Space + uses: jlumbroso/free-disk-space@v1.3.1 + if: runner.os == 'Linux' + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: check_space + run: | + if($IsLinux) {df -h} + if($IsWindows) {Get-PSDrive -PSProvider 'FileSystem'} + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Setup Mamba + if: matrix.cuda != '' + uses: conda-incubator/setup-miniconda@v2.3.0 + with: + activate-environment: "ktransformers" + python-version: ${{ matrix.pyver }} + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + add-pip-as-python-dependency: true + auto-activate-base: false + + + + - name: build web + run: | + cd ktransformers/website/ + npm install + npm run build + cd ../../ + + - name: build for cuda + if: matrix.cuda != '' + run: | + git submodule init + git submodule update + if($IsWindows){ + $originalPath = Get-Location + Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll' + Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -DevCmdArguments '-arch=x64 -host_arch=x64' + $env:DISTUTILS_USE_SDK=1 + Set-Location $originalPath + } + $cudaVersion = '${{ matrix.cuda }}' + $env:MAMBA_NO_LOW_SPEED_LIMIT = 1 + mamba install -y -c nvidia/label/cuda-$cudaVersion cuda-toolkit cuda-runtime + $env:CUDA_PATH = $env:CONDA_PREFIX + $env:CUDA_HOME = $env:CONDA_PREFIX + if ($IsLinux) { + $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH + $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib/python${{ matrix.pyver }}/site-packages/nvidia/nvjitlink/lib:' + $env:LD_LIBRARY_PATH + if (!(Test-Path $env:CUDA_HOME/lib64)) { + New-Item -ItemType SymbolicLink -Path $env:CUDA_HOME/lib64 -Target $env:CUDA_HOME/lib + } + } + if ($IsWindows) { + $env:CUDA_PATH = "$env:CUDA_PATH/Library" + $env:CUDA_HOME = $env:CUDA_PATH + $env:PATH = "$env:CUDA_PATH/bin;" + $env:PATH + cp $env:CUDA_PATH/lib/*.lib $env:CUDA_PATH/lib/x64/ + $env:INCLUDE =$env:CUDA_PATH + "/include/targets/x64;" + $env:INCLUDE + + } + python -m pip install torch==${{ matrix.torch }} torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${{ matrix.torch_cu }} + python -m pip install cpufeature build wheel ninja packaging setuptools + $env:KTRANSFORMERS_FORCE_BUILD = "TRUE" + $env:CPU_INSTRUCT = '${{ matrix.instruct }}' + $env:TORCH_CUDA_ARCH_LIST = '${{ matrix.cudaarch }}' + python -m build --no-isolation --verbose + + + - name: create Rlease dir + run: | + if ($IsWindows) { + $env:date = $(Get-Date -Format "yyyy-MM-dd") + New-Item -ItemType Directory -Force -Path "$Env:USERPROFILE\.ssh" + $Env:SSH_PATH = "$Env:USERPROFILE\.ssh\id_rsa" + Set-Content -Path $Env:SSH_PATH -Value "${{ secrets.SSH_PRIVATE_KEY }}" + (Get-Content -Path $Env:SSH_PATH).Replace("`r`n","`n") | Set-Content -Path $Env:SSH_PATH + chmod 600 $Env:SSH_PATH + } + if ($IsLinux) { + $env:date = $(date +%Y-%m-%d) + mkdir -p ~/.ssh/ + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + } + + ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no root@${{ secrets.SSH_SERVER }} "mkdir -p /mnt/data/release-$env:date" + scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no dist/*.whl root@${{ secrets.SSH_SERVER }}:/mnt/data/release-$env:date/ \ No newline at end of file diff --git a/ktransformers/__init__.py b/ktransformers/__init__.py index d1f2e39..48fef32 100644 --- a/ktransformers/__init__.py +++ b/ktransformers/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" \ No newline at end of file +__version__ = "0.1.2" \ No newline at end of file diff --git a/setup.py b/setup.py index aeff5f6..2a09b48 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ Date : 2024-07-27 16:15:27 Version : 1.0.0 LastEditors : chenxl -LastEditTime : 2024-08-08 02:45:15 +LastEditTime : 2024-08-14 16:36:19 Adapted from: https://github.com/Dao-AILab/flash-attention/blob/v2.6.3/setup.py Copyright (c) 2023, Tri Dao. @@ -299,6 +299,15 @@ def build_extension(self, ext) -> None: 'ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu', 'ktransformers/ktransformers_ext/cuda/binding.cpp', 'ktransformers/ktransformers_ext/cuda/gptq_marlin/gptq_marlin.cu' - ]) + ], + extra_compile_args={ + 'cxx': ['-O3'], + 'nvcc': [ + '-O3', + '--use_fast_math', + '-Xcompiler', '-fPIC', + ] + } + ) ] ) diff --git a/third_party/llamafile/sgemm.cpp b/third_party/llamafile/sgemm.cpp index 6a7cab4..38f6d18 100644 --- a/third_party/llamafile/sgemm.cpp +++ b/third_party/llamafile/sgemm.cpp @@ -94,7 +94,6 @@ static const struct GemmFuncs { #if defined(__FMA__) || (defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))) #if defined(__AVX2__) #if defined(__AVX512F__) - printf("__AVX512F__\n"); #if defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) && defined(__AVX512BF16__) // AMD Zen4+ (2023-) sgemm = llamafile_sgemm_amd_zen4; From 67043b4b5c274e84dca0ee3ace21a7f857721711 Mon Sep 17 00:00:00 2001 From: TangJingqi Date: Thu, 15 Aug 2024 10:44:59 +0800 Subject: [PATCH 6/8] [fix] format classes and files name --- README.md | 4 +- doc/en/deepseek-v2-injection.md | 20 +-- ktransformers/operators/attention.py | 2 +- ktransformers/operators/experts.py | 114 +++++++++--------- ktransformers/operators/linear.py | 52 ++++---- .../{layer_wise_prefill.py => models.py} | 6 +- .../DeepSeek-V2-Chat-multi-gpu-4.yaml | 66 +++++----- .../DeepSeek-V2-Chat-multi-gpu.yaml | 34 +++--- .../optimize_rules/DeepSeek-V2-Chat.yaml | 24 ++-- .../DeepSeek-V2-Lite-Chat-multi-gpu.yaml | 34 +++--- .../optimize/optimize_rules/Mixtral.yaml | 14 +-- .../Qwen2-57B-A14B-Instruct-multi-gpu.yaml | 30 ++--- .../Qwen2-57B-A14B-Instruct.yaml | 16 +-- ktransformers/tests/dequant_gpu.py | 4 +- ktransformers/tests/dequant_gpu_t.py | 4 +- 15 files changed, 212 insertions(+), 212 deletions(-) rename ktransformers/operators/{layer_wise_prefill.py => models.py} (99%) diff --git a/README.md b/README.md index a80fe67..9505c1b 100644 --- a/README.md +++ b/README.md @@ -276,11 +276,11 @@ Below is an example of a YAML template for replacing all original Linear modules name: "^model\\.layers\\..*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types device: "cpu" # which devices to load this module when initializing kwargs: generate_device: "cuda" - generate_linear_type: "QuantizedLinearMarlin" + generate_linear_type: "KLinearMarlin" ``` Each rule in the YAML file has two parts: `match` and `replace`. The `match` part specifies which module should be replaced, and the `replace` part specifies the module to be injected into the model along with the initialization keywords. diff --git a/doc/en/deepseek-v2-injection.md b/doc/en/deepseek-v2-injection.md index c1ccd39..e5dc1c2 100644 --- a/doc/en/deepseek-v2-injection.md +++ b/doc/en/deepseek-v2-injection.md @@ -90,7 +90,7 @@ The YAML rule is listed below. - match: name: "^model\\.layers\\..*\\.self_attn$" # regular expression replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation ``` As we can see, each rule in the YAML file has two parts: `match` and `replace`. @@ -98,9 +98,9 @@ The match part specifies which module should be replaced, and the replace part s

Routed Experts

-For routed experts, the module we inject is a wrapper of CPUInfer, KTransformersMLPExpert. There are several implementations within a wrapper, and we need to specify keywords to tell the wrapper which implementation we want to use and how we intend to use it. +For routed experts, the module we inject is a wrapper of CPUInfer, KTransformersExperts. There are several implementations within a wrapper, and we need to specify keywords to tell the wrapper which implementation we want to use and how we intend to use it. -In KTransformers, some models exhibit different behaviors during prefilling and generation for better performance. KTransformersMLPExpert is one of them. All these special modules have a `device` keyword describing which device the module should be initialized on. Other keywords specify the behaviors during prefilling and generation and may be differ when using different injection modules. Here, we specify which implementation on which device we want to use during prefilling and generation, and which device the output should be on. +In KTransformers, some models exhibit different behaviors during prefilling and generation for better performance. KTransformersExperts is one of them. All these special modules have a `device` keyword describing which device the module should be initialized on. Other keywords specify the behaviors during prefilling and generation and may be differ when using different injection modules. Here, we specify which implementation on which device we want to use during prefilling and generation, and which device the output should be on. Note that we only use these parameters when layer-wise prefilling is enabled; otherwise, prefilling is conducted with the same configuration as generation. In the original implementation of Transformers, MoE is implemented using `nn.ModuleList`. We don't want KTransformers to iterate through all the sub-modules in the list, so we set `recursive: False` in this rule to prevent recursive injection into submodules of the current module. Here is the YAML rule: @@ -109,13 +109,13 @@ In the original implementation of Transformers, MoE is implemented using `nn.Mod - match: name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert parallelism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert parallelism device: "cpu" # device to load this module on initialization kwargs: prefill_device: "cuda" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda" recursive: False # don't recursively inject submodules of this module ``` @@ -126,7 +126,7 @@ If we inject the expert list as a custom module, we can't use the interface in ` - match: class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # MLP module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # MLP module with custom forward function ```

Other Linear Modules

@@ -140,12 +140,12 @@ We also need to transfer some keywords similar to the injection of experts. Here name: "^model\\.layers\\.(?!.*self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda" prefill_device: "cuda" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" ```

Pre-compute Buffers

diff --git a/ktransformers/operators/attention.py b/ktransformers/operators/attention.py index 0648f51..3cfb9fd 100644 --- a/ktransformers/operators/attention.py +++ b/ktransformers/operators/attention.py @@ -15,7 +15,7 @@ from transformers.configuration_utils import PretrainedConfig from transformers.cache_utils import Cache -class DeepseekV2AttentionInjected(BaseInjectedModule, DeepseekV2Attention): +class KDeepseekV2Attention(BaseInjectedModule, DeepseekV2Attention): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__(self, diff --git a/ktransformers/operators/experts.py b/ktransformers/operators/experts.py index 75fb729..864c4b7 100644 --- a/ktransformers/operators/experts.py +++ b/ktransformers/operators/experts.py @@ -5,8 +5,8 @@ Author : Azure-Tang, Boxin Zhang, chenht2022 Date : 2024-07-25 11:25:24 Version : 0.1.0 -LastEditors : kkk1nak0 -LastEditTime : 2024-08-11 12:14:39 +LastEditors : Azure +LastEditTime : 2024-08-15 02:36:29 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -31,13 +31,13 @@ from transformers.activations import ACT2FN from transformers.configuration_utils import PretrainedConfig from abc import ABC, abstractmethod -from ktransformers.operators.linear import QuantizedLinearMarlin, QuantizedLinearTorch, KTransformerLinear +from ktransformers.operators.linear import KLinearMarlin, KLinearTorch, KTransformersLinear import time from ktransformers.operators.cpuinfer import CPUInfer # class Base(BaseInjectedModule, ABC): -class MLPExpertsBase(ABC): +class KExpertsBase(ABC): def __init__(self, key: str, gguf_loader: GGUFLoader, config: PretrainedConfig, orig_module: nn.Module, device: str = "cuda", **kwargs): # super().__init__(key, gguf_loader, config, orig_module, device, **kwargs) self.key = key @@ -111,7 +111,7 @@ def load_multi(self, key: str, keys: list[str], device: str = "cpu"): tensors[k] = self.gguf_loader.load_gguf_tensor(key + k, device=device) return tensors -class MLPCPUExperts(MLPExpertsBase): +class KExpertsCPU(KExpertsBase): input_tensor_cpu:Tensor = None expert_ids_cpu:Tensor = None weights_cpu:Tensor = None @@ -131,13 +131,13 @@ def __init__( **kwargs ): super().__init__(key, gguf_loader, config, orig_module, device, **kwargs) - assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU" + assert device.lower() == "cpu", "KExpertsCPU can only be loaded on CPU" self.n_routed_experts = n_routed_experts self.out_device = out_device def load(self, w: dict | nn.Parameter | tuple | None = None, device:str|None = None, warmup:bool = False): if device: - assert device.lower() == "cpu", "MLPCPUExperts can only be loaded on CPU, Parameter \"device\" can be cpu or None." + assert device.lower() == "cpu", "KExpertsCPU can only be loaded on CPU, Parameter \"device\" can be cpu or None." if w is None: w = self.load_weights()[self.key] self.gate = w["gate"] self.up = w["up"] @@ -176,28 +176,28 @@ def load(self, w: dict | nn.Parameter | tuple | None = None, device:str|None = N # print(n_routed_experts, hidden_size, moe_intermediate_size) num_experts_per_tok = self.config.num_experts_per_tok self.moe = MOE(moe_config) - self.cpu_infer = MLPCPUExperts.CPU_INFER + self.cpu_infer = KExpertsCPU.CPU_INFER if warmup: self.cpu_infer.submit(self.moe.warm_up()) self.cpu_infer.sync() - if self.out_device not in MLPCPUExperts.output_gpu_map: - MLPCPUExperts.output_gpu_map[self.out_device] = torch.zeros((self.config.hidden_size), device=self.out_device) - if MLPCPUExperts.input_tensor_cpu == None: - MLPCPUExperts.input_tensor_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True) - MLPCPUExperts.expert_ids_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.long, pin_memory=True) - MLPCPUExperts.weights_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.float32, pin_memory=True) - MLPCPUExperts.output_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True, dtype=torch.bfloat16) + if self.out_device not in KExpertsCPU.output_gpu_map: + KExpertsCPU.output_gpu_map[self.out_device] = torch.zeros((self.config.hidden_size), device=self.out_device) + if KExpertsCPU.input_tensor_cpu == None: + KExpertsCPU.input_tensor_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True) + KExpertsCPU.expert_ids_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.long, pin_memory=True) + KExpertsCPU.weights_cpu = torch.zeros((num_experts_per_tok), device="cpu", dtype=torch.float32, pin_memory=True) + KExpertsCPU.output_cpu = torch.zeros((self.config.hidden_size), device="cpu", pin_memory=True, dtype=torch.bfloat16) def submit_for_one_decode(self, input_tensor, expert_ids, weights): - MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) - MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) - MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) - self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream(self.out_device).cuda_stream, self.moe.forward(1, expert_ids.size(0), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) + KExpertsCPU.input_tensor_cpu.copy_(input_tensor, non_blocking=True) + KExpertsCPU.expert_ids_cpu.copy_(expert_ids, non_blocking=True) + KExpertsCPU.weights_cpu.copy_(weights, non_blocking=True) + self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream(self.out_device).cuda_stream, self.moe.forward(1, expert_ids.size(0), KExpertsCPU.expert_ids_cpu.data_ptr(), KExpertsCPU.weights_cpu.data_ptr(), KExpertsCPU.input_tensor_cpu.data_ptr(), KExpertsCPU.output_cpu.data_ptr())) def sync_for_one_decode(self): self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream(self.out_device).cuda_stream) - MLPCPUExperts.output_gpu_map[self.out_device].copy_(MLPCPUExperts.output_cpu, non_blocking=True) - return MLPCPUExperts.output_gpu_map[self.out_device] + KExpertsCPU.output_gpu_map[self.out_device].copy_(KExpertsCPU.output_cpu, non_blocking=True) + return KExpertsCPU.output_gpu_map[self.out_device] def forward(self, input_tensor, expert_ids, weights): # generate, capture and run cuda graph @@ -205,13 +205,13 @@ def forward(self, input_tensor, expert_ids, weights): if input_tensor.size(0)==1: # TODO: this branch is unreachable, but the shape of input_tensor([1,hidden_size]) and input_tensor_cpu([hidden_size]) is not compatible #print("capturing experts") - MLPCPUExperts.input_tensor_cpu.copy_(input_tensor, non_blocking=True) - MLPCPUExperts.expert_ids_cpu.copy_(expert_ids, non_blocking=True) - MLPCPUExperts.weights_cpu.copy_(weights, non_blocking=True) - self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(1), MLPCPUExperts.expert_ids_cpu.data_ptr(), MLPCPUExperts.weights_cpu.data_ptr(), MLPCPUExperts.input_tensor_cpu.data_ptr(), MLPCPUExperts.output_cpu.data_ptr())) + KExpertsCPU.input_tensor_cpu.copy_(input_tensor, non_blocking=True) + KExpertsCPU.expert_ids_cpu.copy_(expert_ids, non_blocking=True) + KExpertsCPU.weights_cpu.copy_(weights, non_blocking=True) + self.cpu_infer.submit_with_cuda_stream(torch.cuda.current_stream().cuda_stream, self.moe.forward(1, expert_ids.size(1), KExpertsCPU.expert_ids_cpu.data_ptr(), KExpertsCPU.weights_cpu.data_ptr(), KExpertsCPU.input_tensor_cpu.data_ptr(), KExpertsCPU.output_cpu.data_ptr())) self.cpu_infer.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) - MLPCPUExperts.output_gpu_map[self.out_device].copy_(MLPCPUExperts.output_cpu, non_blocking=True) - return MLPCPUExperts.output_gpu_map[self.out_device] + KExpertsCPU.output_gpu_map[self.out_device].copy_(KExpertsCPU.output_cpu, non_blocking=True) + return KExpertsCPU.output_gpu_map[self.out_device] else: input_tensor = input_tensor.contiguous().cpu() expert_ids = expert_ids.contiguous().cpu() @@ -269,7 +269,7 @@ def load_weights(self, override_key: str | None = None, device: str = "cpu"): res = {key:{"gate": gate, "up": up, "down": down, "gate_type": gate_type, "up_type": up_type, "down_type": down_type}} return res -class MLPExpertsMarlin(MLPExpertsBase): +class KExpertsMarlin(KExpertsBase): expert_num: int loaded_experts_idx: list[int] def __init__( @@ -290,11 +290,11 @@ def __init__( self.device = device # create empty marlin experts according to the number of experts per token # up - self.up_projs = [QuantizedLinearMarlin(key+ "." + "ffn_up_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] + self.up_projs = [KLinearMarlin(key+ "." + "ffn_up_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] # gate - self.gate_projs = [QuantizedLinearMarlin(key+ "." + "ffn_gate_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] + self.gate_projs = [KLinearMarlin(key+ "." + "ffn_gate_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] # down - self.down_projs = [QuantizedLinearMarlin(key+ "." + "ffn_down_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] + self.down_projs = [KLinearMarlin(key+ "." + "ffn_down_exps", gguf_loader, config, device=device) for i in range(self.expert_num)] def load(self, w: dict | nn.Parameter | tuple | None = None, device: str | None = None, warmup: bool = False): if device is None: device = self.device @@ -359,7 +359,7 @@ def forward(self, input_tensor:torch.Tensor, expert_ids, weights): outs = outs.to(device) return outs -class MLPExpertsTorch(MLPExpertsBase): +class KExpertsTorch(KExpertsBase): expert_num: int loaded_experts_idx: list[int] gate: torch.Tensor @@ -439,12 +439,12 @@ def forward(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch.T return final_hidden_states.to(org_dtype, device=org_device) EXPERTS_MAP = { - "MLPCPUExperts": MLPCPUExperts, - "MLPExpertsTorch": MLPExpertsTorch, - "MLPExpertsMarlin": MLPExpertsMarlin, + "KExpertsCPU": KExpertsCPU, + "KExpertsTorch": KExpertsTorch, + "KExpertsMarlin": KExpertsMarlin, } -class KTransformersMLPExpert(BaseInjectedModule, MLPExpertsBase): +class KTransformersExperts(BaseInjectedModule, KExpertsBase): def __init__(self, key: str, gguf_loader: GGUFLoader, @@ -452,22 +452,22 @@ def __init__(self, orig_module: nn.Module, # device: str = "cuda", prefill_device:str = "cuda", - prefill_mlp_type: str | None = "MLPExpertsTorch", + prefill_op: str | None = "KExpertsTorch", generate_device: str = "cpu", - generate_mlp_type: str | None = "MLPCPUExperts", + generate_op: str | None = "KExpertsCPU", **kwargs): BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) - MLPExpertsBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) - if generate_mlp_type is not None: - self.generate_experts = EXPERTS_MAP[generate_mlp_type](key, gguf_loader, config, len(orig_module), device=generate_device, **kwargs) + KExpertsBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) + if generate_op is not None: + self.generate_experts = EXPERTS_MAP[generate_op](key, gguf_loader, config, len(orig_module), device=generate_device, **kwargs) else: self.generate_experts = None - if prefill_mlp_type is not None: - self.prefill_experts = EXPERTS_MAP[prefill_mlp_type](key, gguf_loader, config, len(orig_module), device=prefill_device, **kwargs) + if prefill_op is not None: + self.prefill_experts = EXPERTS_MAP[prefill_op](key, gguf_loader, config, len(orig_module), device=prefill_device, **kwargs) else: self.prefill_experts = None - self.gpu_mlp_type = prefill_mlp_type - self.cpu_mlp_type = generate_mlp_type + self.gpu_mlp_type = prefill_op + self.cpu_mlp_type = generate_op self.mode = InferenceState.UNLOAD def load(self, w: dict = None, mode: InferenceState = None, warmup: bool = True): @@ -523,7 +523,7 @@ def set_inference_mode(self, mode: InferenceState): from ktransformers.models.modeling_mixtral import MixtralSparseMoeBlock -class Qwen2MoeSparseMoeBlockInjected(BaseInjectedModule, Qwen2MoeSparseMoeBlock): +class KQwen2MoeSparseMoeBlock(BaseInjectedModule, Qwen2MoeSparseMoeBlock): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: """ """ orig_shape = hidden_states.shape @@ -548,16 +548,16 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: y.resize_(*orig_shape) return y, router_logits - hidden_states_expert = hidden_states.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else hidden_states_expert.cpu() - selected_experts_expert = selected_experts.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else selected_experts_expert.cpu() - routing_weights_expert = routing_weights.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else routing_weights_expert.cpu() + hidden_states_expert = hidden_states.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else hidden_states_expert.cpu() + selected_experts_expert = selected_experts.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else selected_experts_expert.cpu() + routing_weights_expert = routing_weights.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else routing_weights_expert.cpu() shared_expert_output = self.shared_expert(hidden_states) shared_expert_output = ( F.sigmoid(self.shared_expert_gate(hidden_states)) * shared_expert_output ) - if isinstance(self.experts, MLPExpertsBase): + if isinstance(self.experts, KExpertsBase): y = ( self.moe_on_cpuinfer( hidden_states_expert, selected_experts_expert, routing_weights_expert @@ -628,7 +628,7 @@ def moe_infer(self, hidden_states_cpu: torch.Tensor, selected_experts_cpu: torch return final_hidden_states -class DeepseekV2MoEInjected(BaseInjectedModule, DeepseekV2MoE): +class KDeepseekV2MoE(BaseInjectedModule, DeepseekV2MoE): def forward(self, hidden_states): identity = hidden_states orig_shape = hidden_states.shape @@ -648,7 +648,7 @@ def forward(self, hidden_states): if self.config.n_shared_experts is not None: y_ = self.shared_experts(identity).squeeze(0) - if isinstance(self.experts, MLPExpertsBase): + if isinstance(self.experts, KExpertsBase): y = self.moe_on_cpuinfer(hidden_states, topk_idx, topk_weight).view(*orig_shape).to(device=hidden_states.device) elif hidden_states.size(0) > 10: # TODO may bugs here @@ -727,7 +727,7 @@ def moe_infer(self, x, topk_ids, topk_weight): ) return final_out -class MisrtalSparseMoEBlockInjected(BaseInjectedModule, MixtralSparseMoeBlock): +class KMisrtalSparseMoEBlock(BaseInjectedModule, MixtralSparseMoeBlock): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: """ """ @@ -751,11 +751,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: y.resize_(*orig_shape) return y, router_logits - hidden_states_expert = hidden_states.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else hidden_states_expert.cpu() - selected_experts_expert = selected_experts.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else selected_experts_expert.cpu() - routing_weights_expert = routing_weights.to(self.experts.device) if isinstance(self.experts, MLPExpertsBase) else routing_weights_expert.cpu() + hidden_states_expert = hidden_states.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else hidden_states_expert.cpu() + selected_experts_expert = selected_experts.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else selected_experts_expert.cpu() + routing_weights_expert = routing_weights.to(self.experts.device) if isinstance(self.experts, KExpertsBase) else routing_weights_expert.cpu() - if isinstance(self.experts, MLPExpertsBase): + if isinstance(self.experts, KExpertsBase): y = ( self.moe_on_cpuinfer( hidden_states_expert, selected_experts_expert, routing_weights_expert diff --git a/ktransformers/operators/linear.py b/ktransformers/operators/linear.py index e984a90..146fb85 100644 --- a/ktransformers/operators/linear.py +++ b/ktransformers/operators/linear.py @@ -6,7 +6,7 @@ Date : 2024-07-25 11:25:24 Version : 0.1.0 LastEditors : Azure -LastEditTime : 2024-07-26 09:27:53 +LastEditTime : 2024-08-14 14:57:04 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -34,8 +34,8 @@ from ktransformers.operators.cpuinfer import CPUInfer from ktransformers.server.config.config import Config -#class QuantizedLinearBase(BaseInjectedModule, ABC): -class QuantizedLinearBase(ABC): +#class KLinearBase(BaseInjectedModule, ABC): +class KLinearBase(ABC): def __init__( self, key: str, @@ -106,7 +106,7 @@ def unload(self): pass -class QuantizedLinearTorch(QuantizedLinearBase): +class KLinearTorch(KLinearBase): def __init__( self, key: str, @@ -158,7 +158,7 @@ def unload(self): self.bias = None -class QuantizedLinearMarlin(QuantizedLinearBase): +class KLinearMarlin(KLinearBase): marlin_q_w: torch.Tensor marlin_s: torch.Tensor g_idx: torch.Tensor @@ -252,7 +252,7 @@ def unload(self): self.sort_indices = None self.workspace = None -class QuantizedLinearCPUInfer(QuantizedLinearBase): +class KLinearCPUInfer(KLinearBase): CPU_INFER = CPUInfer(Config().cpu_infer) def __init__( self, @@ -281,7 +281,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: out_device = x.device self.input_tensor_cpu.copy_(x, non_blocking=True) qlen = origin_shape[1] - QuantizedLinearCPUInfer.CPU_INFER.submit_with_cuda_stream( + KLinearCPUInfer.CPU_INFER.submit_with_cuda_stream( torch.cuda.current_stream().cuda_stream, self.linear.forward( qlen, @@ -289,7 +289,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: self.output_cpu.data_ptr() ) ) - QuantizedLinearCPUInfer.CPU_INFER.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) + KLinearCPUInfer.CPU_INFER.sync_with_cuda_stream(torch.cuda.current_stream().cuda_stream) self.output_gpu.copy_(self.output_cpu, non_blocking=True) if self.has_bias: self.output_gpu += self.bias @@ -301,14 +301,14 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: qlen = origin_shape[1] output_shape = (*origin_shape[:-1], self.out_features) output = torch.empty(output_shape, device=x.device, dtype=x.dtype) - QuantizedLinearCPUInfer.CPU_INFER.submit( + KLinearCPUInfer.CPU_INFER.submit( self.linear.forward( qlen, x.data_ptr(), output.data_ptr() ) ) - QuantizedLinearCPUInfer.CPU_INFER.sync() + KLinearCPUInfer.CPU_INFER.sync() if self.has_bias: output = output + self.bias output = output.to(dtype=dtype, device=out_device) @@ -329,8 +329,8 @@ def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = self.linear = cpuinfer_ext.linear.Linear(config) if warmup: - QuantizedLinearCPUInfer.CPU_INFER.submit(self.linear.warm_up()) - QuantizedLinearCPUInfer.CPU_INFER.sync() + KLinearCPUInfer.CPU_INFER.submit(self.linear.warm_up()) + KLinearCPUInfer.CPU_INFER.sync() self.input_tensor_cpu = torch.zeros((1, 1, self.in_features), device="cpu", pin_memory=True) self.output_cpu = torch.zeros((1, 1, self.out_features), device="cpu", pin_memory=True, dtype=torch.bfloat16) self.output_gpu = torch.zeros((1, 1, self.out_features), device=self.out_device) @@ -355,12 +355,12 @@ def unload(self): self.bias = None LINEAR_MAP = { - "QuantizedLinearMarlin": QuantizedLinearMarlin, - "QuantizedLinearTorch": QuantizedLinearTorch, - "QuantizedLinearCPUInfer": QuantizedLinearCPUInfer + "KLinearMarlin": KLinearMarlin, + "KLinearTorch": KLinearTorch, + "KLinearCPUInfer": KLinearCPUInfer } -class KTransformerLinear(BaseInjectedModule, QuantizedLinearBase): +class KTransformersLinear(BaseInjectedModule, KLinearBase): def __init__( self, key: str, @@ -369,20 +369,20 @@ def __init__( orig_module: nn.Module, # device: str = "cuda", generate_device: str = "cuda", - generate_op: str| None = "QuantizedLinearMarlin", + generate_op: str| None = "KLinearMarlin", prefill_device: str = "cuda", - prefill_op: str| None = "QuantizedLinearTorch", + prefill_op: str| None = "KLinearTorch", **kwargs, ): BaseInjectedModule.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) - QuantizedLinearBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) + KLinearBase.__init__(self, key, gguf_loader, config, orig_module, generate_device, **kwargs) # build all the linear operators if prefill_op is not None: assert prefill_op in LINEAR_MAP, f"linear_type {prefill_op} not supported" - if prefill_op == "QuantizedLinearMarlin" and (orig_module.in_features%GPTQ_MARLIN_MIN_THREAD_N!=0 or orig_module.out_features%GPTQ_MARLIN_MIN_THREAD_N!=0): - print(f"This linear module's in_features or out_features is not divisible by GPTQ_MARLIN_MIN_THREAD_N({GPTQ_MARLIN_MIN_THREAD_N}), using QuantizedLinearTorch instead.") + if prefill_op == "KLinearMarlin" and (orig_module.in_features%GPTQ_MARLIN_MIN_THREAD_N!=0 or orig_module.out_features%GPTQ_MARLIN_MIN_THREAD_N!=0): + print(f"This linear module's in_features or out_features is not divisible by GPTQ_MARLIN_MIN_THREAD_N({GPTQ_MARLIN_MIN_THREAD_N}), using KLinearTorch instead.") print(f"module info: key:{key} orig_module:{orig_module}") - self.prefill_linear = QuantizedLinearTorch(key, gguf_loader, config, orig_module, prefill_device, **kwargs) + self.prefill_linear = KLinearTorch(key, gguf_loader, config, orig_module, prefill_device, **kwargs) else: self.prefill_linear = LINEAR_MAP[prefill_op](key, gguf_loader, config, orig_module, prefill_device, **kwargs) else: @@ -390,11 +390,11 @@ def __init__( if generate_op is not None: assert generate_op in LINEAR_MAP, f"linear_type {generate_op} not supported" - if generate_op == "QuantizedLinearMarlin" and (orig_module.in_features%GPTQ_MARLIN_MIN_THREAD_N!=0 or orig_module.out_features%GPTQ_MARLIN_MIN_THREAD_N!=0): - print(f"This linear module's in_features or out_features is not divisible by GPTQ_MARLIN_MIN_THREAD_N({GPTQ_MARLIN_MIN_THREAD_N}), using QuantizedLinearTorch instead.") + if generate_op == "KLinearMarlin" and (orig_module.in_features%GPTQ_MARLIN_MIN_THREAD_N!=0 or orig_module.out_features%GPTQ_MARLIN_MIN_THREAD_N!=0): + print(f"This linear module's in_features or out_features is not divisible by GPTQ_MARLIN_MIN_THREAD_N({GPTQ_MARLIN_MIN_THREAD_N}), using KLinearTorch instead.") print(f"module info: key:{key} orig_module:{orig_module}") - self.generate_op = "QuantizedLinearTorch" - self.generate_linear = QuantizedLinearTorch(key, gguf_loader, config, orig_module, generate_device, **kwargs) + self.generate_op = "KLinearTorch" + self.generate_linear = KLinearTorch(key, gguf_loader, config, orig_module, generate_device, **kwargs) else: self.generate_linear = LINEAR_MAP[generate_op](key, gguf_loader, config, orig_module, generate_device, **kwargs) else: diff --git a/ktransformers/operators/layer_wise_prefill.py b/ktransformers/operators/models.py similarity index 99% rename from ktransformers/operators/layer_wise_prefill.py rename to ktransformers/operators/models.py index 2a1d1fe..c95e1ee 100644 --- a/ktransformers/operators/layer_wise_prefill.py +++ b/ktransformers/operators/models.py @@ -6,7 +6,7 @@ Date : 2024-07-25 11:25:24 Version : 1.0.0 LastEditors : Azure -LastEditTime : 2024-08-08 10:09:14 +LastEditTime : 2024-08-14 14:53:05 Copyright (c) 2024 by KVCache.AI, All Rights Reserved. ''' @@ -155,7 +155,7 @@ "The bare Qwen2MoE Model outputting raw hidden-states without any specific head on top.", QWEN2MOE_START_DOCSTRING, ) -class Qwen2MoeModelKTransformers(BaseInjectedModule): +class KQwen2MoeModel(BaseInjectedModule): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2MoeDecoderLayer`] @@ -451,7 +451,7 @@ def load_layer_to(self, layer:Qwen2MoeDecoderLayer, target: InferenceState): """ -class DeepseekV2ModelKTransformers(BaseInjectedModule): +class KDeepseekV2Model(BaseInjectedModule): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DeepseekV2DecoderLayer`] diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml index 31c5c87..5f3b780 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml @@ -43,48 +43,48 @@ name: "^model\\.layers\\.([0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([1][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([2][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:2" prefill_device: "cuda:2" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([345][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:3" prefill_device: "cuda:3" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" @@ -92,7 +92,7 @@ name: "^model\\.layers\\.([1][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" @@ -100,7 +100,7 @@ name: "^model\\.layers\\.([2][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:2" prefill_device: "cuda:2" @@ -108,7 +108,7 @@ name: "^model\\.layers\\.([345][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:3" prefill_device: "cuda:3" @@ -116,73 +116,73 @@ - match: name: "^model\\.layers\\.([0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:0" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:0" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([1][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:1" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:1" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([2][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:2" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:2" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:3" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:3" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - match: name: "^model\\.layers\\.([1][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - match: name: "^model\\.layers\\.([2][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:2" prefill_device: "cuda:2" - match: name: "^model\\.layers\\.([345][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:3" prefill_device: "cuda:3" @@ -190,7 +190,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml index 15e8e10..882c75f 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml @@ -27,29 +27,29 @@ name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([345][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" @@ -57,7 +57,7 @@ name: "^model\\.layers\\.([345][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" @@ -65,45 +65,45 @@ - match: name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:0" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:0" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:1" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:1" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - match: name: "^model\\.layers\\.([345][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml index 47fe084..85d2e82 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml @@ -9,53 +9,53 @@ # name: "^model\\.layers\\.([1-5][0-9])\\.mlp\\.shared_experts.*$" # regular expression # class: torch.nn.Linear # only match modules matching name and class simultaneously # replace: -# class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types +# class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types # kwargs: # generate_device: "cpu" # prefill_device: "cuda" -# generate_op: "QuantizedLinearCPUInfer" -# prefill_op: "QuantizedLinearTorch" +# generate_op: "KLinearCPUInfer" +# prefill_op: "KLinearTorch" # out_device: "cuda" - match: name: "^model\\.layers\\.(?!.*self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda" prefill_device: "cuda" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\..*\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda" prefill_device: "cuda" - match: name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\..*\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda" prefill_device: "cuda" - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" kwargs: generate_device: "cuda" prefill_device: "cuda" diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml index e79e4fd..4115592 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml @@ -27,29 +27,29 @@ name: "^model\\.layers\\.(0|[1-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([12][0-9])\\.(?!self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.(0|[1-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" @@ -57,7 +57,7 @@ name: "^model\\.layers\\.([12][0-9])\\.mlp$" class: ktransformers.models.modeling_deepseek.DeepseekV2MoE replace: - class: ktransformers.operators.experts.DeepseekV2MoEInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KDeepseekV2MoE # mlp module with custom forward function kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" @@ -65,45 +65,45 @@ - match: name: "^model\\.layers\\.(0|[1-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:0" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:0" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.([12][0-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism kwargs: prefill_device: "cuda:1" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:1" recursive: False # don't recursively inject submodules of this module - match: name: "^model\\.layers\\.(0|[1-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - match: name: "^model\\.layers\\.([12][0-9])\\.self_attn$" replace: - class: ktransformers.operators.attention.DeepseekV2AttentionInjected # optimized MLA implementation + class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.DeepseekV2ModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/Mixtral.yaml b/ktransformers/optimize/optimize_rules/Mixtral.yaml index 21fdb72..ad7d293 100644 --- a/ktransformers/optimize/optimize_rules/Mixtral.yaml +++ b/ktransformers/optimize/optimize_rules/Mixtral.yaml @@ -9,26 +9,26 @@ name: "^model\\.layers\\..*$" class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda" prefill_device: "cuda" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\..*\\.block_sparse_moe$" class: ktransformers.models.modeling_mixtral.MixtralSparseMoeBlock replace: - class: ktransformers.operators.experts.MisrtalSparseMoEBlockInjected + class: ktransformers.operators.experts.KMisrtalSparseMoEBlock - match: name: "^model\\.layers\\..*\\.block_sparse_moe\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert + class: ktransformers.operators.experts.KTransformersExperts kwargs: prefill_device: "cuda" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda" recursive: False # don't recursively inject submodules of this module diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml index d48ebeb..37c8a36 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml @@ -10,27 +10,27 @@ name: "^model\\.layers\\.([012])$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:0" prefill_device: "cuda:0" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([012])\\.mlp$" class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock replace: - class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KQwen2MoeSparseMoeBlock # mlp module with custom forward function - match: name: "^model\\.layers\\.([012])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism # device: "cpu" # which devices to load this module when initializing kwargs: prefill_device: "cuda:0" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:0" recursive: False # don't recursively inject submodules of this module @@ -46,27 +46,27 @@ name: "^model\\.layers\\.([12][0-9]|[3-9])$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda:1" prefill_device: "cuda:1" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\.([12][0-9]|[3-9])\\.mlp$" class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock replace: - class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KQwen2MoeSparseMoeBlock # mlp module with custom forward function - match: name: "^model\\.layers\\.([12][0-9]|[3-9])\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism # device: "cpu" # which devices to load this module when initializing kwargs: prefill_device: "cuda:1" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda:1" recursive: False # don't recursively inject submodules of this module @@ -89,7 +89,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.Qwen2MoeModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KQwen2MoeModel" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml index a48b15a..a44c750 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml @@ -9,36 +9,36 @@ name: "^model\\.layers\\..*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types kwargs: generate_device: "cuda" prefill_device: "cuda" - generate_op: "QuantizedLinearMarlin" - prefill_op: "QuantizedLinearTorch" + generate_op: "KLinearMarlin" + prefill_op: "KLinearTorch" - match: name: "^model\\.layers\\..*\\.mlp$" class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeSparseMoeBlock replace: - class: ktransformers.operators.experts.Qwen2MoeSparseMoeBlockInjected # mlp module with custom forward function + class: ktransformers.operators.experts.KQwen2MoeSparseMoeBlock # mlp module with custom forward function kwargs: generate_device: "cuda" prefill_device: "cuda" - match: name: "^model\\.layers\\..*\\.mlp\\.experts$" replace: - class: ktransformers.operators.experts.KTransformersMLPExpert # custom MoE Kernel with expert paralleism + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism # device: "cpu" # which devices to load this module when initializing kwargs: prefill_device: "cuda" - prefill_mlp_type: "MLPExpertsTorch" + prefill_op: "KExpertsTorch" generate_device: "cpu" - generate_mlp_type: "MLPCPUExperts" + generate_op: "KExpertsCPU" out_device: "cuda" recursive: False # don't recursively inject submodules of this module - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.Qwen2MoeModelKTransformers" + class: "ktransformers.operators.layer_wise_prefill.KQwen2MoeModel" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill - match: diff --git a/ktransformers/tests/dequant_gpu.py b/ktransformers/tests/dequant_gpu.py index 9c839c1..0dd5272 100644 --- a/ktransformers/tests/dequant_gpu.py +++ b/ktransformers/tests/dequant_gpu.py @@ -5,8 +5,8 @@ current_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(current_path+"/../..") import numpy as np -# from ktransformers.operators.linear import KTransformerLinear, QuantizedLinearMarlin -# from ktransformers.operators.experts import KTransformersMLPExpert, MLPExpertsTorch +# from ktransformers.operators.linear import KTransformersLinear, KLinearMarlin +# from ktransformers.operators.experts import KTransformersExperts, KExpertsTorch from ktransformers.util.custom_gguf import GGUFLoader import torch import KTransformersOps diff --git a/ktransformers/tests/dequant_gpu_t.py b/ktransformers/tests/dequant_gpu_t.py index 8abc89d..4b2556d 100644 --- a/ktransformers/tests/dequant_gpu_t.py +++ b/ktransformers/tests/dequant_gpu_t.py @@ -7,8 +7,8 @@ import pycuda.driver as cuda from pycuda.compiler import SourceModule import numpy as np -from ktransformers.operators.linear import KTransformerLinear, QuantizedLinearMarlin -from ktransformers.operators.experts import KTransformersMLPExpert, MLPExpertsTorch +from ktransformers.operators.linear import KTransformersLinear, KLinearMarlin +from ktransformers.operators.experts import KTransformersExperts, KExpertsTorch from ktransformers.util.custom_gguf import GGUFLoader, dequantize_q4_k_gpu, dequantize_q4_k import torch import KTransformersOps From c47205dce9b7a64a47fc4cd3cd55f2fd6926f98b Mon Sep 17 00:00:00 2001 From: TangJingqi Date: Thu, 15 Aug 2024 11:25:12 +0800 Subject: [PATCH 7/8] fix name --- .../optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml | 2 +- .../optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml | 2 +- ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml | 2 +- .../optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml | 2 +- .../optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml | 2 +- .../optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml index 5f3b780..d7adfa2 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu-4.yaml @@ -190,7 +190,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" + class: "ktransformers.operators.models.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml index 882c75f..a21b22d 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml @@ -103,7 +103,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" + class: "ktransformers.operators.models.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml index 85d2e82..9d029a9 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml @@ -55,7 +55,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" + class: "ktransformers.operators.models.KDeepseekV2Model" kwargs: generate_device: "cuda" prefill_device: "cuda" diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml index 4115592..cfd77dc 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat-multi-gpu.yaml @@ -103,7 +103,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KDeepseekV2Model" + class: "ktransformers.operators.models.KDeepseekV2Model" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml index 37c8a36..bfa60b7 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct-multi-gpu.yaml @@ -89,7 +89,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KQwen2MoeModel" + class: "ktransformers.operators.models.KQwen2MoeModel" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill transfer_map: diff --git a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml index a44c750..073332c 100644 --- a/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml +++ b/ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml @@ -38,7 +38,7 @@ - match: name: "^model$" replace: - class: "ktransformers.operators.layer_wise_prefill.KQwen2MoeModel" + class: "ktransformers.operators.models.KQwen2MoeModel" kwargs: per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill - match: From de3faaf55d2125b02660ce83e625ec0851c304f1 Mon Sep 17 00:00:00 2001 From: TangJingqi Date: Thu, 15 Aug 2024 20:42:54 +0800 Subject: [PATCH 8/8] Update readme; add pipeline tutorial; add detailed inject tutorial --- README.md | 11 +- doc/assets/deepseekv2_structure.png | Bin 0 -> 106714 bytes doc/assets/model_structure_guild.png | Bin 0 -> 281048 bytes doc/assets/multi_gpu.png | Bin 0 -> 104155 bytes doc/en/injection_tutorial.md | 328 ++++++++++++++++++ .../optimize_rules/DeepSeek-V2-Chat.yaml | 19 - 6 files changed, 335 insertions(+), 23 deletions(-) create mode 100644 doc/assets/deepseekv2_structure.png create mode 100644 doc/assets/model_structure_guild.png create mode 100644 doc/assets/multi_gpu.png create mode 100644 doc/en/injection_tutorial.md diff --git a/README.md b/README.md index 9505c1b..8c5f505 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,10 @@ In this example, the AutoModel is first initialized on the meta device to avoid After injection, the original `generate` interface is available, but we also provide a compatible `prefill_and_generate` method, which enables further optimizations like CUDAGraph to improve generation speed. -

YAML Template

+

How to custom your model

+ +A detailed tutorial of the injection and multi-GPU using DeepSeek-V2 as an example is given [here](doc/en/injection_tutorial.md). + Below is an example of a YAML template for replacing all original Linear modules with Marlin, an advanced 4-bit quantization kernel. ```yaml @@ -276,18 +279,18 @@ Below is an example of a YAML template for replacing all original Linear modules name: "^model\\.layers\\..*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously replace: - class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types + class: ktransformers.operators.linear.KTransformerLinear # optimized Kernel on quantized data types device: "cpu" # which devices to load this module when initializing kwargs: generate_device: "cuda" - generate_linear_type: "KLinearMarlin" + generate_linear_type: "QuantizedLinearMarlin" ``` Each rule in the YAML file has two parts: `match` and `replace`. The `match` part specifies which module should be replaced, and the `replace` part specifies the module to be injected into the model along with the initialization keywords. You can find example rule templates for optimizing DeepSeek-V2 and Qwen2-57B-A14, two SOTA MoE models, in the [ktransformers/optimize/optimize_rules](ktransformers/optimize/optimize_rules) directory. These templates are used to power the `local_chat.py` demo. -A detailed description of the injection using DeepSeek-V2 as an example is given [here](doc/en/deepseek-v2-injection.md). +If you are interested in our design principles and the implementation of the injection framework, please refer to the [design document](doc/en/deepseek-v2-injection.md).

Acknowledgment and Contributors

diff --git a/doc/assets/deepseekv2_structure.png b/doc/assets/deepseekv2_structure.png new file mode 100644 index 0000000000000000000000000000000000000000..b9ced326ad19c5e280e1210992dd5020da9d8f57 GIT binary patch literal 106714 zcmeFZWk6J2+Xf25fP@H$2qGdXjfB!Yg3=}3p>%iGJPN3!gmj6-&=S%iAp!y-4Bg#G z3>|0BAjtE+-*?X6^W*!W?Ad#-weEHIz1Fq%TLn3Bd|Yx|G&D4PNr|V5XlPg&XlUSv z*jT`wEe+;a;19ZkqPPfJeka8u@Z+hGx}>qJEE*$ljg5vLV1@=pz5@Is2mYa z^TM<;)q{{x|9Elxo~M_U9;?CO{yXe|<|{3n{UM~^eUDdMM-Fl72nePQ{2IGnmmLyf z>D?X#JwylL1XG6*UO($ZtQeumDqmVunO#*>Sy}ZVw{86G!eX{|=O#aA)Zl)X!(;GU zG;|Ocn~)0P``;H30Lf3`~bR?`0XhHPbSt#!u- zc)@NQ%3dGR{a#~Y(I{%2GV0g$mQ=5{Y?ZrtZr`d53fa_VQHMjpf4u>e3-ecs;#Hya zf}U|JV*_@SiT-qelBHx5DB@@$s_||}pqz)l97NxIp;GzPg*(}#B77z1b2P7)rTPqw zkwEa{H6)ckC7}nta5|e>ScFD<;MqWA_UKU9y>XJ-)vG;}SVi>@DL}d}nmme& zc2JTxPqM7B(tZWBW!61Vu5R?S%hs!NP25GvAmt9!m|L}ps2;&!-D6pGV5NG z`X|l2lw(wn^%Hp5GWgzi&C$=*7eWf=yAza6z}VbxHUaZ?BaOQ!jT)|y7V0V>gQ1pq zO&v!u4`#6p^UmpCJ zpYtgw^;9~}Q~7yOuSl4PNMZz2p_@<%ihV zGR*3(&(4QcK!cWoKTR#rf6W4UN^rbKzEpX?@WbK>IIHp(&0`uA2fT;~W0vr$L@qex z&P=#I5JK8CweGCMr;@C%T=nA1*{G{Hxm*%75O`8S-kIc+FvYgCd7UnFlB!S@9UWGR zJCuwf$WfQdOYCKwLzU8T`Y`jDa7T;HnT}AMg2ZHI>q2``nuzR6KMKT@_@xUQrXn(Y zW?Q+>^*JA81tn=ds2oMn21y23a{w+-KkEa~*fpG7$H+)u<=q33&NJ@OfI!Mj-9^kO zy&;nUT#+Vbg z8Ke3{nb*xXX`fywWG7Nn!9&8W&L)3~sd_!(T`5V*l20=1OjFg77S4Nb`GS}BtcBr$ zsx0cWwfrh)bA3o;o2w`C;_Q7gb$~+XDjx~mIfZXPt8$|zk&#_^XH)QmI%6nXKfQHS z+Xq5vn=a5|a#f4@oaa|dAyrDdI1>1X7RWS`uoPd=DpuzGU^EqUVX8QkIaH7m#)nd0_w~vQ9)L!-syqjCYu1L^NN2aYw2sG;NH=&Y39zEpK%fSdJ0|%zti}t z&+$H?xU}RMTk36bJvZm#v+)mq%-8-bf-W979)7s3C3&WOjym;kVEM#M=eh_KFTAEJ=Z_3FcQU>Ha(X4`&#(h$lWp-9EP=ar;ObG@^ldoAHOwmSrlq)H0T&$ z=7?_^3>VsIg^Mj!ieq#Zcp&;3=wD$*Z!P}}$U?;icd*Aq@)yM>LmX@=qgHjfvtPNY z5L@$7`<*jw8YQAv3GYqJot644HZLeoRHuDc2l!m^qX$~K4bj0bjnD< zKT*;bZKz0WQ|Y6A$DfI1;RA$%m&C^2ENV*j@kaI2Kb)wf0|*n;;w3&OtmGaQJsc}s z_RlOsu(9QwUO%TjvujxpNXcTVnEhO|^_rMd4(XaSJ-em9KA+1U2I2n-o9&cW)y!Bu zvp^^&U@e$<1!sK%jolv>jyUrEqbRsW_$TW9gZ+>+!QAAj%abgQ2o%MR+7SKh=}(W( z0t7I=sW{cx+mWSP*UCJPG6-48FQjCR-Lw_tTKhf|7ZttaPERiWP>Vsb?SM#}d3!O^wnA&+-i*-fL4~WlE)oxfTCl@2XSKSvQWI^F~<& z8WfHl$tbDW5wdgd+(S?mlA}L?!LtSf{_r=W-hQ4ggLEyUwi-EPiYdUWXUlf0XYL)0 zy{)0Y@4LjRL8Z7|81)64=KSzhYey4m09kbX1Jb(|Hw zWy8Cbsq77CgC|L-g-)nP{C=1?bZ;=I#Xr|VdBaU9PfT_8v+U9t?VI<1mJul}-u2eF zvR91;y8<;JmAJbWcVfV^vbySc=o#&#sq76_rJJa-bh54)XdnGnqoDfD9Fw9&jmbn9 zlt}W1o1rOw{z&$hhFXP_SPDFAoh)XvVl}8(8_CKPb&wa1OmmeK_|rSYlb)d?yr#fP zF@?O`WY6flAMJf$_j?mnxuk7OOlk0DinMf9W3%N>T%Hh@J|$}~-@S?QpE%Fl95tzQ z>)y2l88cv|2gRAwzC3e$rpyY~HI>B{(_aO269^-ha&tzo9Yv_dy4OxkJi=?+ha8GX zpx)SfEXx7z&aoN4+rNCy@BxKP9~Y}VhR%Y8ws?Ia9-0_>tz*9U&9F?{=_+I9RNHen z#E8l}u&??^5lsWz%Uu##O=E0KS-gLGzq2XKGAqUMD zVi4RKNQqm5`{A28PsnZ|`;BM%?|rBD?i1QLFcXSzG*2iO&m%AXsP3fQ`I6p$=9{R1Xc*4v^J)=2|7Ka+h(7wmHGlYD2 z*Fy3Z!iqUHzu|LOFz6Ej5Lm<=`Ai=M5U5Bjy~#v)%Eb~T(&Lcm+Ig#=x4+fq6oyjN z8S#{yu9&t;dZPHd;!?3itipG}oDBI8iqB-PnlxbWH!Znd{#I@|>qV{sr3{Duc`ca~Y0aNaiJV{M?WG{=UAt9uU4TcaPmj=4*%$n4)E?o4~v~ z2_lf(iYrvgM_|tCqIFgxy})+lWQ%el`o8Veyk~1RJAtj7yzv1X7@w~CB-VJCV7e%G z{c+F9)t5FFJydv2RCY%`gAeyvEHZxW8)=M3fBwXyDRuuR0W>%8HDEXk_(s>i-?XLZ z`tq692b5?1#mq@4vhG6`5lRaeLLvPiB@G|6Z^rH!%07HrFt8?0$$^i?3|?hkZM%R{ zHk_`F4|Ggqia0I$K5kUA6r8*MgDleA2|ZH{p(M8f*3_Y}M0h)&vfcq^ zCT{+Sl~7#f7UyiwSwvA~{aISCtQs|C%J3Eq>WOP8TQIovQBFwf*RiHiaNB}+s}$g( zd7-(BoL=J8GrC_hoaoK3exn|0#tC z!Id{-%sHbEsp_*HK=0#v?p21~_&mrwyi%1*6B-4!XPEfM!KX4EZVEf;m4JccgcFj0 z)OR1r7MIhk(SLlOE8*P)^cx@+8inh~%ofG@qZaOHz?4^w4Dvs%ptGvvgE6<{`muG|vn)=z;u0s;~1K^^J&GyPHcFWa% zN)Xszl*$5^YTRgac^vRWnmUDxy1_H31o{=@1dmXbpxO`jGIGw@k>Z$SoIGUl0Nnvf z5~O4N6Eb}v#OP`1aVhG5BH0Zws%6>)vY6`$j6Yf+qDbS`_XEN+6S(-y)gT?&Kfc|H z=Tk%nH4~e1nz!K#G#?6>K0BF-u=Ek!Kx4S$?HfW2BBw}2$?-J_xZG4Q{;0c(XstFRAIn$oiIj|C^{HPA}Srw97V)C zJ@H2s+ep>}5#S9h_qkHznDgpc#>)g0znX79SY0#`=`&^M+1s~0?;;Mbr1Ep-KINZu zY!avy$j55okI$Iz3u(dBlofSYogE}NUUPkBc(%}kP53<((zjsOvp-u{tc+SgKA;7p z$nB1KK6fnniB1n^j=U0>MO=~kpJq2?QsBJL``nbjzTt>Q%@Yh|xQnX$Q2PG`vq((F zElFhfHP;_u`D+6y0_>hm<=mNtd_n5ti=s0R0r(@WsW@(P^VM zfYOa&WsqM(FQ^~T98(51@2@llGg&XQhh+qhUa?gj`h6HKaGL$TLFd2W(3xY*e)ucU zU`d=Hms1+Z1+pMH0nWD{!JnJQ5@q2ASqswyC3@**Q4HLr^|7_dTJedviq!k7z7W1d zy<&*6_FT0Y8sBpglxpVys;x7tqczuZr-~uLu8jyTv%`fW6HJW`E0z<^R{d^9cP9z_ z_C?_W#7d(tkT_(RLXO`}Xgv%Zd-F=hQhR6H10A7gaDw+{Q_ketp-Xz8c^!c?NXhK> zRKs0?>Z+RhUV~-wovQUz(#DYy8J@!+znwl@v#n-_svTUJQZr;3H;4I6j-NlsTCnzj z7VBN*ofz`bO6H(s#E!We`)z=jeX46g^~`tc%=DMCKZ3Eu^c^`Y%TMM-%si_#XMYX2 z^0>BGVJLZ#Ak7mx2ig!zcG26|pJQS^vsX+a_0gXZ|4dxfWN)8UbnPb@Zu~CsI5Inc z;e$tKta!k#wu60F` zB=;rQ@9t1<4uRhgbn&nc5yEgWG4oMP&9eL*_W`}gPyyPYBu`C5W{LH2A-CEgkdrGy zbLVC7%^Wz-7B_PxfP#R;Zq{d-$7w8`l=ALctI=s)@-JR>`&?VVyz^k$Hv+WGg za${^J#Mh)jFX{Yz>T>Lc{qOYuroA;lVqLcdqxo8(?){KkHdJt5({;_S zW^s6Eom6*7&13bM9eAoFqHNSz3o(DZ?&Gu9n<+$+U%lBjg=WeCXapa97)gx%WcsV` zMdpNWS~j$7YB)Xj+x4#9-TyA$z+5(RouzW&qra8^PDigY&-SrRzy?p$_)IFB+v%w2 zaBnldmQROA!@`$Uc=MI@1LGL?-?h#0uJV8kpd{b@%ZCg&FuJ;vWkDqJwG( zO%a|j-N=HV7kHo0vjbUc%M~7EWrWX5tsWlMT|GSAun|pAR7IBN(62-i$?WjlZ~rKD zC_hAcN$7nXU^UjvM$c<-hfU}U8$!G+&TV4C@iE`>zUEH4A_IlT_O~GqR_vMQ5MM@< zns84WN5{A%9>jBg*yw7}rqm9rY_5T3diUqBtGpd;qxK+-2NU~>7(FVY&$~ZjWvPz3 zIVqv%^M!7_C2Si#t$G1RXmy0NR*1h5&XL@xHWJ<3m2+HlGT$9B8aT-52Mp{d20HiA zHkt+zGOUDIpsO|6j|wv7jN}_~bbp!VQgaz9H+ZORxAEn7UX@G!!w412JL2^4O!Y5M z^$Ec;8~t^Tv3Pa@#k>PmEQL&3`y9X8qbF-w4Gc_)LZC^2DnabC6 z`UAT2hO?KidF_U<9WF_4xg~l#)7uPPmuh`hUg6~tG0Pet>f*!Hx&>eQGRqv`x3Lxe zSYEPiBrJ$A*r>LGj={!-XM5C#AnG`H+^7SWHE*2oTNkfzkR|^kSR+m<2iHx;o*o0P zzWsn*|5%eG%P(41gA$7CZ}C|0)95SE%vR7|M-8&3cWGSY!KaHtaGCyaZ@#i}-`;jz6+Ff8*V1X`12TjHZyE*Y%*Gkjelh zc%2N=+>P6O-troMeP7>cf25YX5n3=xE+#KP6O`mG=EPU?zA^M_H$Np^=G{}f7#T4; zm1}pj!@Cw3OkO)h>+0wT6?>@hZ08L}TPo?WJtpV!-XqOvS*D&YJWW2WTYBU(vyf+S zEMMc}9q~!d?48@U#jEp5MaTWM6pWIBTIv2=t6mQ%_WBE`TgD=?=g}XuP?An<2>j^q zs}L**h;5qrDL9U6U`)&2?mEVHvV+&S7d!8-DA4c9l%*1zlpSIif3QK&xJ?{tyrwrU ze}y>OHH6)HC{{8t++>^5MTBQ>b+~amO&C{p+3OcfvSysH_dlsVyDr!@nv=ICUHadmmk|2TQ z@#9*BF;DeI<3|t@AF;AT<(df+aYHND^(zsOUmqX8^nJMhw(Iko9_nU1WetW8pEvyV za32KR=`NJKD%X5G-IaP+OG;aJn&^*MjNiax!CB7tbXk3&s!hpi2_H((R)}+3i*;j|ZnayQxKpNr zL4w$ijOZ3#wBrl`hj54FRxM?IpeT@v=L&cm$U^Y4HJIw~bkWDZf_g$Jxh=k|cN12E zy-7_Fgr6p9OE3HR2U*tJw6I_9pO&ZA6(2tawsD*eXF*C%Zp~Z4Z>h^m*TQp_it#^F zK6*p!185`f%Hc0PZhX05YU|r$yAG)d3DOVcHmdJqRZn$oYuMmjsXeI^kpPJVDtQSv zZ~3{(DJ1I&TzLmeWqnb@>SLn58^T0hlq0P7?#UO7@uTR%$;xD$mvo7JFb@;(+#F*oHq4}PWu>7wGQWE7 zb)mP8%XDZ9MeK;3dh`xm&!J@FJ+`gAkg$^`yoFbL4ttXUWuvz;_AKJf;kUm9O5PT- zeaQ^0?$UqbkIJO4+KFvU@M99ihy z{jApY%Hfav`G%F-1uER3q-ozG0yPy~FtIi~9(k2bN0BGXX64xWIEs5Ovhx;1F%M`{?X?HkP3|9%&6d-PcTnT3x1lC+TjY zo1NO`mZy`WZKavOnsxb!Nl)3G!#m;z%|u!onL^DDuN>Vdh;(A(a`kFfmet1(Qk0K! zo3{K8IKsi!-?}PNyq_bS1PBKAzTUq5k3`g9Y0wuA-b@VEb z*>(K{t|Rx0N98OST3g&Za{h;R-5+VYPabio5w~Jz=dkiW@;ug(cS)Ch`Eu5FHl2=R z=6GmXI&;T#p#C<9mo)3{6BXVPR=1Pqk5}V+iJ*w}V+H$eOQ8XdGT~Sj`dH1(6N5Fo zxH$avQ(G&kTGy5x=y2^w(TeN)IfciF@gMB+%Hjel;^H*WtnO*~+*MN6Z_N+|ijEr? zQg*m0Jbn}3l8^leGwI)%t^9Dvb_wryHs8Fj8^pKQF=M?&M+wUdR+`T}g%K!cJ_y3= z*KxBNi0(}n%9Ccka|NS+X}!CKUEI1|@rX@DPE6nCof2*>=RY1tD`kGLfSVy5cJqkV?XQcb z8G9!aF117%lx~GwMT}VaeR_wrOK)K1-VbA)ZB~LDa+D~)?P@S!4QF^C>BpS3iW;m1 z_iJGdPX+TU5sW2|{0Nt6i~%Qp^eC>M=zS@i-)F^TW+L;5jO10&qh^MP>Ftn+R>zR* zDBq1LnFO%O9z_M{F4*4j-dB#yn-AC^U*z1p1CZ1d||r8LK^M8 zoxI7kyL@yMwMIJ3!_j(zgt~6B`q@}h%@DjMH28``C@m=)rF2qy;n7wAR^{iKEYE~Z zmRr1$8LwQ%#sJl9pDuoM&HM27BJEb!t9M<{?-CpNi+h&uU^Dx2iyWRCA*J!$;-f_>DkfU=U_8v?^rYO6Jf0^hvyUPe4yNaIJF;2c3-{&()wHfIKg@(Q+&~XuK?DELxrO}QAXH2o@{n}o#xYEE~x8-=yX9H?{KxpEx z6?4bhbqGHFA&441j>n!Mqk?<4Q_s*$F<|L?W+iM#Q+D3Pw8cf#-Ne>Q)uEtvgw5-@ z=g++<6N9Ez6FuE>ed6(GEpP^HMBwvB*zkw4f(hX2r3}K{yLp^(0#}%1H?A$VK681Q zxtvW}J*6{nrh)2y^{IDU{00p_^@K;tEOYJlm3+3-aZ}onC9@+hKbEAT-WPo#$#t*d z)+g^KTTj(rRG->r40(Rr=*h|n-8|uvTGkOZ=a=KA5RXY`2=SKc(wZtV=B{a)h2Pd$ z(;|C&BUzGUjNFf;pHHQBI{>dCsT!y6F)>tZaESB#sAml`VIOGlnKJft zbY}b@&^=JCrK&Bb2cuR+*VES{uFZ+`pBJ_db?e)t99oNWP#eeURgkOnBesqBlR5m9 z%$>^K%pq?b5G_gDG5xQ3h2pc|c8qcX+}@r0a6xHiF%It5N}E8&DC)w9jVlTXs^#}o zx-g8h1AjTZ*`t-aqAlrNq-gb8v*^juN))#hjzdAsFUPH1hw$dxYRfgQlUnP`hYlO; z6-zF#_4c6(xnnF{Vyd6SPzzQY%9tg|N!+t-IbP%zvRRAcN6k=SggejT>&Av1t1VLF zP(k;XJnJhj8(xK0S5K|Dti>hk35`AqxjGOb(6tn07}=ZC%AUS+*VJCksM2n9y_*`+ zCuiFf6&@MwTP_Xrfp3 z9ciwJ$GB|XX1p}W;L*fQB}D#m8COVGwUp8^=10d^K0Wcj=XVqJ1#mw>F-uV;_DCh73I*0p+*YEl^)2SF3zJ&!)oYY8>`pnPCV3ci zCqmmJ%Us3{_r8@S>l)|G<(QBjKPoY6bmdHedoKH>+f^NK@DIa9q*p$D z82yTsL!2et*G#b#Vwcm-*5VPhWUZ;eQds6%qIv8F35m}#2bM_1s4N)k#v=1?pw|7u z3Fc@a-qWtj&LW`E}!O4SM4$f=II945aCn9FRCXdmpIbI9lF zvm$+^*wX`2f~AfplDb6lQkoLrFdv_{K9IKt(x9On`}@8 z<{_qfoH&zn=a}=V&=6ZWNW*2T^84d*FFqMg(y%2fn?B)8daQ%xP&<6iSf5XZjUS(Q zDDjrCIqA8z8!+eG&AgK@Z?`0jT!2J?HmYHRyeisr3E8@B7RYJPzk`fg=|3~G-x%5l z47-t7o>hj`u4!_)kzsiF_LCyjJg>vd@OcD_fXmVhvDWEx()Pffy+qrQ26nHp5$@7H z^MPcD5_Grj^mwvr{MfpD^~*3gJr38&d2%I(75mU+uaF6(?a%sV^nz(g^!jC#E0<;&9557nm6zlB>dQ~i7ZvD;xVcs_E1`>~`HovA6Q7juH)kL|xTqiS-|&wV$s@h)K1{izv5+ zPIdxCj+Om3CV!j74c0Ju-Fi{g(j`!~Q+x8PI*Zf$q!|$iB)7l<%rg*3F%ZIc#3jUT zK6OWpqsz$>SiL~(3{?)2-VAZFGnk!d;Po+kX&*7T_ta{5DA4d-mD754{tnmjGb3{Y z=YY+}+{K2opZjLn`IGaFh=g61S!yzNYA2sf6|GHS77wa92OPZb!5?-9tl9vx$afgFVr@dMV#870(O(7Mak-SVBEmq&M;v_=$g-EdirqVo&v9FCJ=8gQ+fsdP%4 zOpLZ6n5TonIGW9lmc%{M+mC4hxB@x=S8%dTUWBn7Zmwzi6Gzvu2U(7)dL-ArA#M7_ zus+2eYnvX0mr~D&xjpbcSp3w*wcQyCMg<>6;Q{yeCbC8=}I=Zm+~DZH{!3`@i>1Pe{f1fdmZTl&kzWN>i2&hvLDf? z6sD+wSZhmI1vUqACP#`x)>PiZ?UTkT=n~w2B^iF|J-TbT!K+4EYh}T6n($IIVTrYq zOpztJ!5{Imb@`h<97A~M&Q4rgLFO^1rC)QG@>L11C8gxYe2iBTJr?q9UI%j@*&|Nf zt4&LhD*z4!)i;cjJZOK&QsM$dF1c8lRRP>_r}@BFT9<3Sa}L8 z!;6Mij}(S-gH;&|joLrxx79{vJJ}o#jx{6Z%*w=YdUuoO|>nHa~cNBYfoM)^a%d7{}gxp3e6-+Q16gtVG~&JBGy5Q}kv3)U~t=(qC~IpLDL&?#!~;wW_q9v~FdE2YQ*k zbNSdZuOPd&r%Qj=+4Xt?|L4kVqv2nP$0!dxc{7Xu8mA!3!!jXwY?7p&A5BZ;P~1$R zhrZ7MP|U%Et}u>0R1|Kuq{zErQ_JJnRmLvS4b9shlwtK{alj z=VS4L5(xf-x>MPnrJIPVTn4dtyn<7BsGqUR*4RmZGJyucrZu}oJkdfWfD74dN*k7k z2^}(6-@;I`IjK8!vC7>1(3sPbTV8!Il6mt<*pQq}4SlfukMTF}gRbXOAF5p|_30U$ zs&^7uKYnVB_XxYvr+DF0H;{j#&i3x#(E?iE z?JhvKni!v>oXkp+{0LYmRM3acbWZKrZa}3!kLBRg)1=j76?I_rOERLitSv}lh7ZNA zhnXjYRE-KSDMj2nxvKK6<+qCa)u>nBF2#s6UlW-;qMvKXh;<$~{^Md5*mWtLH)KL( zoWM@*JJgdvk$cEm# zk+tH%PI0iaDmw63_7AcX`HZRn1RyIcy0S z$Zk@)K*=@BsTrIAj4`rVdm~`f~=MSM=U~OA#ByJP;{sfP1 zSV-TG)a$ksceB@e&kwx;l|(*fhURETB&Uvg4>3|} zJM9V;z7dxs4OMNilx^g zL;^|VyIV#&KQx}-!>F|b;Apo`)|OlSeg|Mx`(?}P|Hga~#=}HUqBUP-`TI~E4?9=2 z-p&&V#s0|;oeh~zA|9-&RX8tFQb<8M_y8mjlE(sW4xIc)sb<00h|rXU@bt8q$_}Xi z?KO|@b`IGLd|=(xxykB}Cx@?9V9`Z|({$BvG5oc`m)(cz1GlOks4cvA^R~*ZInAV8Sn;j<_8F1hU;q zHk0>wpoWR%WsHmIeANoETCyGu5(0_O?)QagAqT?eXI#}|yU(Jfmboc?`+C4=S=_MTXeV$CW%Hi?P|n;j0GF)@Ob2f=PEfXS0GN&EwkVy&^Tb4XKjg{GaK3TWv_v5uDU z@Z3%E-dtFG19J~>&3tjRKP`MJa{QtIhj8ID`qUTV0$y4dxDm8e{dv8pGrsLTgdG6w zb|_bt{dg9+HrU3i2U^5;Sd^vP&rTeAcr(Ur6G}K|Ov-xgL8fku%sh9cVFl(s5g>Va zy(~!^%mgJwpOzU?g@8k9+Te^$*-y^MU&so23b*`?;z(`3J;?1Q&szC(!`f(7wDLwX zsoqtsGHacp5hTcJTMLj$od;we)dw9kN$4e`JiZz!%GS@LG?@%^V`C$OBsn3$vU23i z%bq>kd_ct(m0YVp{IB;-MS*QrPEJnvw=u527=B1fGO*-u+^fd*uZWea4n(tNL+8Iy zF6=fC0!JME-km#e)Px}XKBt+$`Dct%Osp@h74(H;e42lVQ!01Oi!EyfzzyQb*3ZtdJ`S&InA#ewM zksf}IAw1(ca1M?k8T@yQS4==|Qtp*G_m>i28R`>6Ro~p+z9@{6{11vBXssUxWCvKV zvfikRDIzaKfLG1U-(9=JHD2W5Cbugv%1iga*c`anGMw(NX9tFmJo{fd42s#j^_Sah z^#ETnM%=Rh9FIAh#(i8F85s_%FR-J(P(9!Yl^XDJ#nQE?OJW67Z5B8iRF)8i_&di< zVA@gRwa@;0`C%o%FcBO##Z;SgF`%BhJ+in20SPX{@sK z-gB>wdgO-$`pL@4-TWn*_BYqR8tz-)ONIDQdG5i^v#HW+~#6`czGs4SwLm}VsC*F z^dU$k`jn=8cChDs%sHU7=k-gI$_KbKv6C2go-rUF0k~QXD6F-$Rm9zt<3hEdPXLT5 z`KK{OwZCEm2*(Cqg~1s3F3n*EI7P%bf9I3?rF%edL+)oSOy2qb>X7Y4o5|m~)uSQL z!`jZn_%7N37w_oV9zE(V8O+$UXnO&qY<~uVY6FGCY&O60U*~`n-3MYo2kk}r{?74x z0f24sJ{0Tt%L(MX-vW1uZv_6mdG!sz9$&{u^M5H(eYJ0Wq&(>FuwRkT*pXT#hjT)Z zk8UU;y&o^Hh6F+I*=b88m248Av{%X+PUrC9KY~Fq5ysfLz3up>peK27aIny7n9KGO z17{!rQ?DeZ;9j~1D2sB6BJB0}Uhp_}mGf#gX6#f})~hQ7ETcRcMF4zJfhAgWq(JDn z&*oTlFR*Se@ZcG+iF094Kj{D^7y8O{b~x!=M>O#wrP6J?pOO029pu!R*BFcrH9vuo zWdR2wPLC^@t1me^z?COGg3EguUFsBY$kNd9{lsyc;W8VW3fY_}6o!y@KB32)c=<1H z2^UfUb`yLi`M29>>0{5w>d;^It|X2HTeNt5~$12aP&ZH7=RTe;uSn3DHzu-_Vfys5SXW(&Ki&l~IZE zzse7Lz&S-lgec%&7BAuVn|ll5!sIM>Nr?w#KPD7r611%Nxq!DEMGyZ*a1>t5}1 zwtX5Vez$F^?FLu>9Ct)<7P1Z)xVugM(@VVppv(07=?xc6D~Eugr7A@vYF! ziw<%~ffJ`C2cxTh4weGG;RooYiSAH|TTkfduw#WE_#I0aRvI|<-H9aG$&L?j;`kpl z`0w4~o6d4Twg#lysO6ehton>~U)M#MFK7kg^A%9?{s?IUqVC``6@ApJP6fYwyryp7 z4H1u{5$OvOz^Z^|mtJQczU* zH;HQOrDh6XCF;1%h@aT$Mh_ImYO3Uhf*Ep6q9Y?MCLSHKkV=of`&`|RXj z(c-#yQs=r$l5A5E9Cb0j=9>VEl`TPe7ppzrufHMqV)FsH!#en)9s7pZ1bgzKNV|zt zGgYtf$&{1d#?PNGaGI!8a?p@pp_^lBI!*|!2cy3#&Lb`|F( zzX_GYE+J011cCl?y6?3w0=j zf;lScg#x2)Ni_D=D&%Qi1MZ(f=T;1b1A#5CvqSe%qZ`Up7A27>%1?v|F|D1i${7jr zcf5gteAqY=QTYlG9D;!8^%$ah**Q~@mk%kwz^l9{hjxFR|a4voV7p6)`97{<>dI$CmaftpG z98D`!>6~=ZJb?k{f{Y`ctGY>~zjZR0DYx2EiH*ktcB4CU`@Z*CrIuLh zI315z6!-(Xg@)D^z){n4Yq<{;awMPoOJmQOjpQpY(VHj9-Nf?waO0)-0S&1hb;Q7u zbwC(6b2F?>obnEr_ry1I6~>4NgLS-jQy-bS{o8VDG_sAOT{i2>OEz|3W3s zf0;igl)C4- zpELm2fh4edtpwA)20zyGVg6Bl^dhEuH(!C*Re!^pdAy)*&7|Qwk+|Qlfq@2RNgZyl zZ8h`tOz&+-3;?RPSnuDy-^%~Rz;Re|Yu0)!yvbWC^)Y{Z&`)=E8Nm~Y827yZc)uC1 z>@trKCD(C)o;)`WdsPyCkG$Z`K0OB=Cyf%Ro+iI&zm4K9>2ZIIvQs6_-#3#*OHXwA z)uOxp!2Vx%6U*UF&+AZ#g$S}lzPQ^=HJa4l_+zWfUv$@#>2++ZrjYJmcS2?e*yocq zL8r@BQU4gUJbswU=KYOV`U_)lhzbVB1GQ=o>&nC%GgL-c#m z2hpOZHp4g`UzY;F*hrl9Mwt%T&iFwyTbYZzQ?c_B&+c3jbvakJ!g@)NeL_Z|cXq?4 z+?bC>L!Eo>Q@?T=E2Z<@zxNL$cXdEl2- zuihSH7pd%}V7~VAE8FpklEFc1OM~_(b|x8Q@b}5>4~C?fgTud}w=IxXrFiEnJ7Pi9 z;GiN?G{K!U!~SvUxW6G-`Pej-CYD)~uswNr*H~0=iWY8Uw_jUhFxRXTxT85bjK6+Z z;h&;RLBqKaxV%EF7HmvKoLl1(1E^yQw-AaN zP8?G&o=9$*o++9Ko_#t9rOA3Lz5F~I#8f?iZLM^l*&}YuAUJ)2a5A!gT)p4SS~6tS zygc6im$!(R7^vnhG}q7e?ES0~?f7YEZ_ox$iJ&xEt^FTh ztNPr}|1<%N_+SM6-DORMl!3FAT5F>AN8)Wb!0A&{u)AZof!)volDinPrI7-MW{!sp zrDa#vt80LreYK|NkSle^!9!dxGpV~$+?Q{umil>l6J_3)QKER{yudH%)sv#E{XR$` z?&!ot#K?W}mevDFg{)`>p)~VTwbN8Sk{5iM=4_S;YA}6#pCRLO@mSK+@VSN=g4$#OQ3f2vnkeAV)3~Ua=;p&uhScyM zotZt&t;NA1I2*C@k2-2%WpZ{`B1)YQtj0-h_nUm>6R=n_coN4gM>(K~;hkBhgfFbO zf^YNyXfV3q*ktd*R2E5FbL+0f_)#X16)zP4;-OfpKX6LysKQpte{*vW6Ky(kQX)`<{S!O8aNLlHHg3!SL z;jof!*}cecea%&q={Y~fu9oSWV=fyi>EYnE@Ot&)8=3N!60`4T^6Hgez2&~31p}y5 zIc;?^8ZPbt7pC%5SZ}U*uC^OAdI*zcKd>GfvZBVJ3{LQQ8pCrsMfmEjlJLQknCjCt zf0=f1%F*B6E1!DE$4qJ%YzAUVvu)a?r-==YKDIYB4UZH3GT}nxJ!WDkA6&?fTCw?a z^cxikflpPn8DwUiZxo@lk&lKjbHZb5{q%8AqMbDv|7tFq{fb>F!@-MNvxc93R<@7d z0w8nb*F2AtBlf4~#AhPgfH4%h*wa5;k#J>AI! z_e(Ux%#r8?qfB^4O}y7?6uIJ6snt3}VCn9w?H9+T>lWBjvVuFdPOk@6+F7{-Y|LL? zuO=5b?$SagnFJ6SpetNamwhM10;>2peGGu3ciC`o-$nvH0H<(T%Xa+z6+C%! zQrtgS_zi!Ro((|y4)GV5}7_8d93_t$V^nq!Cw9Br4(%b~Fs zZ_3+KilG@7uzRou>3XVux0Codig2)4l%(U=xm|xkayLjL9;kchdc3_YY4KxV!qV0> z#FV{P<-A%n)XjBq@sn|rW5(P)np9SXF?$I%ra#u4jOMe!D%&2t2!=lOgh%Q+2p@9C zow2iFv$NXL$&7J&9&y!|gmxAU>rdO9z4a?RY-(=kTq%X{@{b((`jN3`-o0P?K9$g1 z`^;%+ie>`t6I{C0dmy5Og~tAPP7($t)7^%2W@%%{YrlM2@VBZQ;v7Fx2OsKek zbz}Wnl!?CCWo4&7)p9Gl8JX9glO@!1Z#K(HU+M5I1IJ4iQdH}~iQVGzj_MDrYQdA{ z0oMwajLc|W^1aPOB81iw|5^e?;QEqgEY zReXUkFtW9(cdtwFl^(PiHP^TGTK8A2mf!ST!jb-J>vg`;gIR8DCr;$At(&Vhd9Ispclv*9jhgPmbeUYIy0evgP9sF9TC*mz z|AhF@Farzcj!VT+!3ydxN_a3M%8EVxtqQxDcw5G6o*O*Y4wt_<{uCO|z=cO!-`oS? zt9`HC;hW&dW>ZE&nVrbCc_Q0yu66|`a&oEa zJ`kK$gl5F%Htum^{F1)SRVWnn82MPOht&3WbE(Hz-=0c!k zQt)!+UcyJGl?s*3g`Dke%~geF)Cmm25aLn&xIPY^Kx0j!wrL^Fh=bt0?lTNE6RTMd zg&$>!RLrr}%wZm&nW{=K6$gh(I};K3D<$hAT7@m)y^M1+?|zvM!Y#871o$AZ(Rc7d zbH9ay_-?nX{h9)&vo6oT+m~y=Q|JaKZLMmSnx3c6^|&^1KeoqKcLqhiTL!8-QET=s zMm=h7tA@Ru%8vPrs&n++^U`tW_z#PxUS8$^xvBdQSk8q1!8M%Sn`WFS+QC-c+9ElD z5JaJ9G3}k&twNOLt@%VL-gUQ_UI()6B9QgHF6(oAu^#KGKeiMr=E*hj)V!*ta2uHsfQHyq189``go6*iI}e2)qX6SFdaSmG;0h!jl{G&}^U#M043M zslis@8V}HVuA#5`P$}Zdf~Zcxw(Lo-j}zzh&&yb-{A7V{`uCHy{qqj24we7O*;heR z!T-EmTMM)yp;iB%v;OGD))CeW3Q&2e?3y05*x6r*0b$Sruym|pXBPBK%&9%2(U8;Lfkz90K{wERk zdu^bssbg$isXMm_vrL|^i z*8mQW>JR>`SAyi%k2XFY>$JmU?tKvN8bptG30?<5T{@}3BkTZ>Z|-8|rZH4}LspCI zp3;^Akzvps$G%8MWbAfO1rveoF%_NxEnd_Mi25I(u|^+55Ls+MGN=(s(bvh`Um>$URt!qDULqpQp_mFIYrRA$tUwa^~`OT%w2o`Ww@1}N^ zyj08)Po4q!MD<#Qexsug0W@l$`a9t(m|=UI~unpaR1vNp(T`>$d*h{SOmM)NC=kFS0xG^A12jw|Jyo29!}e z?^bKL;G;Y{pR?UiyNK!yn$sIj%9P{nnDx=6$`3_x|0xt`CHV0CdkBP92lPXHXeV76~@<>6ohGF2QYq(&3X*Y`;0t%B%v&3jh08n z6aVR{gyI4f7&pJMuM|ty__%$Vdr|LZHSwBTg+*%V`Y2GtTW(Jb&D$xIqp zxXoPiVR;;`JaDzd=5MW|u_kQ<=JNO5wMUzNj}!Ig!fLzVYWXtk?gFz+%wm~IUgzH* zZA5_sa3nQsOo;#NDZ~(aWK<_IuMfeF_vxJ4tx>m**Qf^8op!j$@C2LPUcUi+!g6TRkV=IfbgE$QIZ7w|B@gExw2oLEYI^=C45lT;0w~$^w!a_ zv1cbHCS=2PP(f2qeE#&gh)M?(oeBMlFmqA-OO5j+d@~LUV8Z7a`y%!qL%4P&W*Xd?4#*C9pGHRxr2LOR#F@r~K zi@=xv$M7gj9skm;VisPZBZdG=&IU`?-y-{4aAuQ%7L~`1PEiDg5C!6Me~=Z;X#q3$ z(HeZ}q5QdUl!pOgMiEF6=u^$VyidFw)D9PK4IWiP0suvUP~T3>`#U;&zX(X+Xnf-n z9Klhg7Q&30w0~jL=?=)n$0>!LK7I-t1A64BqUm@zf({UeAT~4A3BwBm7jTZQBnh2U zAaF;}VcEG|&iXq%V(N8a{`Q(uVID8)>i31MtF{QVjj9vrjk1WR4KSvu;s6tVN z0?G(CSKdNHG>G`BZN3UShVAf&F3HV@Sp*q{Cu$sF3=qwZ)pFv|qo64V)l$8n{O=^w z063^tjn8@VqzDK_IJ%&2LBHS3_;*V%6fkg0fI!{ku*2?X$BS`N1sih2f4`UJ3O1yC zg5~d0jhRAFtpb?*u=UyRrLm(DG8pRNf0p`t2M|zz=yKZ}wDP4mRQ}|B#_kEKP|z>} zM6dg5(VdJIWC2T4R){)56-pMOpO*Zsf}KFO=q$%@2ei6rfaohAh}J7^C2=x(CI}$L zpVkpcDS_Xu&qINwv|e5H%~9bDh!Ok7M&T1GjGc7zf-jMHNU z9H)dTVp?u*3Y^BkCV`wf{09a~<>kUYH;CcUdY{5P`Dgti?W3$QSMSwpCy$lNPr)*~ z+W0X~hNFJb;$j-feg4XTz&ZXZ2DZ!vpqsB=>OJ>ooq2HaF3(_0P&> zNPcaU*05!&W%*v~(-dC*L~7g`d261pz!!bp&ZGcseC`S_(I4W`D&^FsNQeBGNq)m# zQm&&o9lqUHT0NOp^cQ^VWMHhmSNrRK#-`LhBiCb^buvGo^4MK&Z24e65pWx;_b;G_ zFjM#L=`W6fg!2gv9R8xiBxR5bdh}X=!N_)hDm?t6z~+eKs%ybth@E2q*tS`Vz4Q;Z z)nF=xY2XmIV$-kX30YyBb#$b_0|2J7p38RsGh-OpfO&e{j}H^CSqCzRwp_>Drpa<7 z(TR&BBNtHOkh>!<8pVP2z{Jixu&cvKs$Y``Q}kR3NfNtp27NuBWPvQHTsmDP_M7;R zpk<=Z&2IU;gJ98 z(?UlA>@hwz@fYfRkuECquR-8*n2t2}naK6spLtx-P5-1%J3MYF7{w{7CGGqwwCc4m z{eE1UKc{7WCZ}UZ=Ic*5rgj~VWsjM=?Rrpibe9}?Yw-yyUEo0QdTJX?R(~eIEG;?* zOTiy7xFD91GAiKZbvm5=?ZS?1s_~ja@Z6=j@h0u7*zKZw7Z~rSVz8dUe8p6YRD1PU zvy1@+B+LN^h%VMf`yq~hB+km`TmXN+aRPNAG2glnlrrRGbuTpZ?Uz~Na| zH}eiSo5kRX#r-pNRY)Cc`~f>53dwLIjHyVLsFzr1xP;I)VN$;AI_-;{8vKNO52{#P zBI{mcy+@@bo8nEB-Zahiw)C~av4E5y0D-IMfPf=#P#X~B1Pk=Ee`c0;A$1vB#e9FW z%&>tmlI* z@YIwcKB|=0iC9~S$r>!45zOD_4)t2h@JMB7;V)5@XemPfL7!q1@O_g`@chA%wt&p`&2yVfc_*k zQTRu4{xjwOkD~ua6d}%F{JR%yG1knPuZd1yE!w4S%6TGsj!Ama^{U0tm!j1T++W;@v;u8v!)`S znWjv?ki1^uAn_G{xB+=XoB{c5f$Ba;Wgx403gX`vHx<;2E-vXncL)_?*mSJub0jJ51yYbgWzSPak#uVh1 z>76&o5u|}RMe&QGnw@Ha;6=S_))NgeBCpDQRA-7R#Jt7k$@X$`=|s` zW2)+qROusee@HBFI7Bwksa^03zQK>aUW0oUzwj7mda7HDv>st9>~RQ3=*X*3$8{BU zmebzjy?Xy@=6fAQD(T|)G8+2Be!ohFNmKlZNIPkC@Ugzau5rw@k;{eCR`j)UOxo3Jy>)8vZ@@fJjy_IaF-wYsl`UGl* z8086VIy^YukDI(Mcs;tg!V_U(9TG0jtF!s&sg>nYnR>Yi%~k8+m#h6;jiu4>TsS7hBrY?F)EJfd;vtASGjNPBJ%pl40we_)Q zDJpUw=40(8MJaBFP+$7VV_8ak!=t5XY-N#~)tvOnTDAcBR z?dRLB;)BM~^#HcBN#n{;#J;*XZn~=NA-mGJ=4Pv$AEX5szHV%gtEvyZbhIZ|It0LCUY%`jAwYxi84G+cT z;>m8-8Dd;u7`lSyZNMz>gKnv_X$_$(TMXs7Hl)^HXe)4m(*>s0hHBw8KiD3>Ph4AQ!5vFB70T`?od45BWsGT zRdY|4Hy$u-EEi`hL4F8a6It5qq$A=HZwg=9L57c|lb1!QQ+}9>b@LA4Np&7-E%2}& zMlPInGkb&Y-fN(SR1Hi)4mP&g1t6mhH{G9R2@G%RL7PTO{ZkLt>D%3Ug_&*+RQzI2 zx5b(gEH=9-<*4G4LCR9ns&gyGNg;zj4CXp^(X={?_^HYeA7N}x^}4%$!|V3su+XH{ zTe;O+ie0vPJWQ$dkMfG9-)=3@|1i0b>PHlDk?U>(8>R0&bk~rNDRtPHUz}Q|qa<^9 z$Td2HBT=F#USoV$QfsF}pnX?OSuAsGIzF7Ip~|a?V9BkR1}Y#9$%D>}o3Iz^#KL3R z<5EQ_eHZvv^#_Bmh&6;3ev)IZ6qAeDfKq-*<$imCR>V4|byuGD9 zfoyI>s48xF9x=Z;$@}TL9Y>2ConAUBYiVbQ^eYn2P%f4^^$hmKyoC*XlPSM_u|>@W#x@T}m^2R>8~hY?S>Y!ypC%x8YPHZm!?jJSz>mRRoH6pol6C@ZDW1t^B2wZaXKE(c#o7- zzh`8~s-ADVe+v!oao%it*^0lx@`hwNLX~7(=Sggp=~wqFmX_2sIWcb!eOrj`o!_y> zyr~fWwt^l;fH5ew!O|69t24DXvm@}eEWy2U^Zu}&g#NYdMIps^>rYO zz6rd)<@Xbejv(8i=S2AzwQz6O7p0zGtFbGKk`(Btc59O!o~`1*s6VyM&ba9r68B@m z@I)HGmvPRodGWmR;`(q0{flN#U<=v@H*IInQQg&Za{nad>eWMHU+5Wdt<+PPymIw| zv3wBAa1rkZp_*q7pjigw=4ie@YygiWGN3IN zehqz#5H~RQ@~eO7S?sij(DOJ&RxIh7C48tz;Xy2m&3aAJSIktJR0yx~n^P@{KiP>R z<5GLjC{naqI&Iaokw+Y!$~VGfYP+A+PQhceS+(Li6JFwCK5qE5!uIQ{{t;u{4AUAY zbtWQUv)ZgS@6pi9ui>v`8$;Jkuv*5>Gn$bOAj;#(b9pCmRz$g3`Zix@M4E3fMxfQ0%{Y;(rej}SzH7cBfO#Qr6$o{xP z{f+02peih^)uYAuYh3AQIMAtKeVva+oJ|u`KaV5;X;STJ)-Yz8u-0#Q==A?KG+_)R zcr^NOeD-+RzfI3MI>bSjV^`@93G-nSo#{1eoi}0U5_W>uYL>HJ3>nc2J7AeNMFnQ; zRD;%M82YX}mx8s*K4J^q;~yR_9hV+zu;Bn!j&yOc^eyp~LEO_o*|7^wU-2CAlA4N3 z4d$7Cu#;bwR2MOlG$^K9L=?K_SQ_V+qRl&cr>nlt)Xkh`=QfPrI(clbz3R10 zj_Wv|q~Dc&bJAp40a0;cfg$`_!i<;~!^P3GhFQgJ!o9U!r74@s{-e$pk9xHoKTz4+R*9#i(SUVCJ&n&m|;P1^uHu}z+5E?q_RTz-6@n_f*c z-(TL#+r?{hT3<r;weLDU!x+-%XtXKzj*vHMx2h>dYe}$%7leRNV`#}XpvUV zdTqIN!n?c~y~7B)fdyGuo-Qe9`s5S#vp!>mLBZ(HJ*!`y%zErom?sp(;A<4~!$NA^ zUdf%ZYK!A*3Ci?{F$loF7nQ?EZK8ZrelZGSSi7vYL78Jj`{ZFIvzO$2619z&rD z4D$D|1mhQC9t!3}$wyaRH@Q@kgMS$Z7am~Fl%kML;h;2~)0G6v3_Ea*(%}+*h}C(P zZgi=Y&e)IiMu}R*_~(t}r1jRz`t3n}CT613o;liD%R9=Q`wcnoULDLw*IrphD$){A zk%AZt@*Hv&d*pWcbCf5qWDf|BMM1^~o$kDW3ZT48X+_)AG}mbrFBsYyuqg+bt14`N z(BT=;`|TkSr=w%oyt3SC51J6B&BScI71kppS%WTw|MC=3hORFNkjM-WCjYI={)s9; zR6yX6e!acZ77e{<{PH{4N^6Lkv4HBH>t8d%-0{!efXceZKkgBLntZNUpTnI-)A6TZ z*dXR2^Rq2^{_jPz;Q$}Zcu{n^XC*Y2;Mkx6E1;AEo-5^opM$_(BmM>Y0RZ~*dOL#S ze*+p0N-yPj1_&n|{<9u5T){q^0s3SZt~ozVv%)1rW?c@oY4`X?F!{%a4?4rKj8 z;x+&4jJP1N(QCjL+Jd&HXb}Cu$9EPGfXj3B^ZYkn_1Cn7sBA->4O4^dqYrTl z9pf3{we+hO)w9ik{{dscZyn*{$10RbPu4nq2|NW+y7M>MPGa7`MBT#yf$Y#)g`1)J zfK|>te|8Esaeo9?5O%Z%HF*%n>NPX}D~B@2M#8hEIbdhFzQi>x`6iS9#ZfN+coa^O zNCbK8xyxoYCLDRxM-(NM1T^vWF`n(eQb-a6zBuXJtUj%oNfepSA-y=D^L02IS@_=l zbp?CO6VL+y@n-Qe6aEEyEH;R{(s+*QPKo`Ru%Baxiouz2xTP*%3{?p#IOVAN(7y8s zsgRE-*4oAGf*yYp0QUSM;`ER*aq+q#e_f`T_{LQwd5pbX^C12lGdqDfuFEc$hfjFy zJ2S``obSl%WIg@_%$?x=_yPe(n!bN$-)Et^n39gMD!;5 z*I0{4F|f1P;R!k?*(1m!e6VwmMiujLk1&lA6Ts_aD#+MMyF>Jd3lpzL+M8~74!gtr zXf-ev0^q9}8$?T@(K_rG-UGXa@-mAv#tNU)8`rMO)CzGzS;7LcnyY>BL8@R?hJGQf z^CYq_1t?eIjxw6b0)&P*gvMn)_6qqPNRj^PVx^o&ie6*kw5_Hc>_txMLfu#GgRhhr zj=AuD(uas9oE;}W*ttu53|AfA%}~(YhyNn_V$QR|2UWLE#eUm6??Ev87z=;E`+`^| zZh7W|V?2;t4dN81F%sW+M{zt7-h?yMph$25VMPj3Gz@SGHSMLi8l@?bf`U%=e7bzo zudk-%8hUoz`oqVb2!-4#7DGWLUMqB$W>xC*L_8vJj^L+|`uZyYER^vGaR4HWep;_n z^=SX$Q<%5!W(o)}-aQ37k4+@BaIcw&8YM@Ge{rb5}u~!>#>L zJ(5U*xLn|bZldB5ZT?|2930LlIfFPg7P1di9MNJCGf@iemf6*F%qy*DYeCt-Dnl%#36wN+@UjUy-HB%-(-jl;6(DI_prYQ}? zIW#B&E`lg4HyPOxDf}7o9W?G@QE>ICMw9hQu|Sl3kIi^?B%*%*jf4w@*+IfYAvB0D zfW&?l-t{cB;V9YrNf87hevOj4qb(W&+I6Tt^Bs8e&u3-9<9C&8PUwygiuB*!UijNC zCwZO?67>J-LX_lrHe7&K+kjdcZ37Aizorm8XgIh7DC}8n!3hcfM%Nf{O}|6EW~kA8 z7%}kdsoF_q>JJJ$P$<}AOG3A7egLuGe9iEXh)(i=2v{Q|IV3ExNC zLKFm2&jG%DN0`ddvTxp!pwq}{9|<(*QFhJ`JYSqNQ0VT!{xlvcv-bl0L@rLzpoh;(;Rw28#2hXJ_7H3x%e@zZ=%4++l@O?a>E0@-p=~McJ%EFU zPFyp`()0J6kY&uXlbV%eJZM;a9tg2~Ab3X`JRtr93EUm7tu7a>fD=w84t)`w^>ihTav9HwMLtjJ2D}b*de#eXG4*~gb;)pE2 zQ2g@7Yj+~)P4&V4&gxve*ZvlE>WbO`I${=aKpuBvD)z6^8X$+TVYA)oEenH1viu+Q zoqv3JyHcDFZL4{U&SWGRfnf6T0!Nj9CRf2`|M2yxO4iPtg_4Z_N-{cq05b`s7Z0u= zu>Ki@iOFT!9;#JjLY5I$@XE&I)yYKK&P)B39t#e0|LzG4*~gSVhSp%WlrW4 zwFjTOb}!WZ(%*F`GGL#wuw{$TwiyLe7#YGX&8a%`VM_y*?G4a7+x=Z{rvC|N1JePO zvBEM^IJV(_FGYz0rZN>RxU*1X3@Vet1a9U2wL}R(ZR);n@BOpa7Ibz9vC~cQ9!pw3 z)naVQGf`cTS?<3%gASIG4iJdN>r+HqpjerK}?>zIwfTXQI_<+&02kFW%VRI44L zqHA9q+XW;Ag@nZaUk|G%7Ja}VA06ttHq)6r8LFz#^jm!&bLew_KsS#^PP_k$)~67b zuwZyjrelY&rr%i#l{>O(e|cWoz^lnh<(>YA<1->_l_ja^K^c?XFcp`{%UGu_=_Nq| zbPSS+0<&vfDT&1Ls z5hL$hM=}3YeSJ}$v!flU)>LV`Mu~zOE{NjM37PDpU4w|DYJDn)GccH+Do+mDAI2Ra zl{i)mWMWxeOQbcbVN(*;C?NuC0My?*uo;nkO#Zm^0FU3dvdCU`)u@z8FlKG$4KIof zFNo+JisWeVT9etK?DafNOA+y>Y8n+kS!-0uc$07s_iYvLJ#R;#9Zr;I0Y0O<`TYgP?OqFc}V!zmJ0PEAwD}Rg{STo*ZAN8`jG?jLP1Nm!^u>1z3+7^S= zp-F0{2S~Wh16k&9Bst;mQ^d+EFK|hNy1s@`zkUNzt7g+VocKGP0hYQC1*VdZ8g@qC zxT0sewr$<{-`o3l!G-S`uusDBWf;s<-d(4iT->19H-7u(B(@k-DJ*;;0jGM)cUq8Y z%bp3{LWlSsLa|{k--&^G9-#Yz{nR&vzx}pBUTBgS(^KWt=fAC0P=Q3g_i*%yS}JZP zp27>@2aJr5t{tY=;o)Gem;Ga)KENPUGB2@D{D|;8w(ubnAbVjE z>(LZ;q6RN8D=i?PXNNVfGyN$IJaz%Ym z5qBs!m9#Kogwtd4F58RrO)Y9(%N}k=Iaw;GRAti?T*}C zue((t{MuwK^BvB@Pc5#^2ci#BZHk7ecIGA=oi~KMk$%5MLQIe^_J}eXU41->*2d>E zlJ>equS#2(`;KP@TPXexseT8Fp2c(1_sVhX?cF)4ZSl}MVwo1gKe|p^ZIh2})Hzz& zFj0%ay}hY50$Y7Cm5i6BDiY=RiLxsoxz$v2>qr+i?~r{Vb+(uMhb6R-Rn+Fkd$ffe zZv#ES^({J?M1fQmiqJv~APeYgKJx=Vt=OG}`q5J{*H=hCG14x`S)8oBQA9>v=Jf*w zviedw-9sfK^#Kr~?>MCV*n3n{rOhN(wjc|=x|0}<6d8IQ0x71s+c>Mu+DTqb#gZsR zB~6mi=PI7))wo_M-LfDzhg6e(Ow%QKD6D?qzh}sASwjoD!R%n#w_Cp_I(`uKpj3qv zVOV!siSH%;Aow{NWcx9^;o?tEzL_u$mitD-P1fTULyINxqwTD}g(u#1ap3P7jWD_& zyKp@cwB}x;I*afN#BuX0ygyQIU66WMa)*av%O)d@-I`Gv-Zf6xyYE^RULw5{mle+Y zJ2(sneR$<~#)YQ?j^$4hPVgE@>JV}MnPQq}S%dUkn`@>~LH6hA8>%hNn2|^XLaSW& z@3@xrtZHo2Ib}}25r$vDYx%-`kQTkWyvukF>6GBaL<8fC*RIViXuB##u z22WK|jqVJbeu$k+dEnT2Rx?A4e2G39>dW?akZZR*Ul_RAx@&6?lvPi9V-W?Fl$3m#7!crx zfd+&c3IdZCO5aYr7HThDT9O*Lv*fWddR>V6fW>k>7y6QV)|>l=#oD>{*Wpc)rrCt}1fHzI~vz1e5l#GRIA zs&%8`Ji@v&ZZezwbEwnlDnnd5Z%BC=|8Y{?;* z++p0&J?;W_z`2>FrEo&Nk(-=1okyMfpW&u)KeG7s^IM_AWOHS=vy;=8P-fMYlA=U+ z%-J~}`IZw&h^M9rM__$-pi!Fj;n)cho;$OaShSz&m@NAI4X1(r1i{RX&iM4UCx_aa zV{$^mJvOj_xwy&l=5s&-iT7*<;ij)`V#-)!hqLK0)NJ?W8@&Vhra%dq(uuvptoA+`5j7m_<%oRd#jT!`waAT z@Q->|@!+M4(A;7OTMgAhO^soHM&0tg?Irw*b_<&X0mrlu{M13E?PT$X@(CikCF7nR zpaDRcv4H`Ez4GTBQ?nY^aVVcFA1dtwk-$;O3|%N0wn?A5G+M-dr|TeaLA!4Mt=?!PzX^F%yglilmnKxCI)6w7QBH)eM*It6*EKkHQby{r5bq ze40|%V#+_`pQYfudl=zeX=a_&KiJmP^Q!|yN*kf?>1O{*8Q5kZN-f4rt$^GaKd8R! zgb*`6o$gfqZf|p3zy0>AoB;5;12-On;K0@SF^3j5*s9d57)Q-+;o92ThsOCW=5 zD$P`+pX216e?oOlQPg=(P3wbQqFm(7I!$D@{$AD%{pyj!0~uyFU`6eMElnvs+1ucb zI9(09X57?;&Ch_CuXS2Kld-k6t#h~R&kF;NK663l=_`+GOQ8)fDXN_{EEo~m6{_}! zM;!jRY`7dGVX$-SK`aZclrsVDk5jF~LttbsA|=-lW}p!N+{R!}M}%j)RaMFZ(Q!iQ zGSxM%Jj@L){ahq&TV&U%`}a?2bqHB<6uM?@?OzfEH&#K{dkd%IJ<2KYMn^|i(na^A z6@=G&zNlJOS>~19o9am5$gf-ps9bKKN^-0*gUfn3q{b&k_(~OHrD=r2d?gjn(#`ib)O4~H`q3YV&+D5?LFKpu- z=BF@;P)~7QnXQ>EtU$NOcX?K-Y$f^K(&C_tc$4GHKw@r@b1_x`wtwz z&wsu9jo2#)G=lxOHBtqV425;LR9^<(J&NmyA?nUV&fl4gb!@jE!{Vs^m87cfqY5?- zUc@#tA`WDkod^|B(`wb>sJw4qFyxzYl&+j?kt5;?nK6HBS z8iEP))tZ|{iluqzCVBtc2QUk-_@tqJs6De-jpA&HdjvS(X58i|wb^)kjjc>pf4eJU zWTKdjR4u@&9{YA7mG%3#$V;{~PJ?HuB6*0jiWm$?2g66r{m!*U8z>va4CO`1GFW12 zY!rOy7d_bOkSG=~X^qTjYJYCkvcx@jWsu{^Hv%nE1pZ>SUPXy1XV(+w>a8~0A(K1$ ztj3m8KbPZnJjQqQHh{KiQAiLOE!Az4Wea=UF&W>DO;Qx`z(KkXI}zNW0o9+~el4^V z$>W%`6^*c*pPO@dV{K{K67BZPKFyG2_4b7M*83>aeR;5y~cGZtT-cKJY(V02mX~&k_aI&QA!E;&^Y0c=fD$H z>8&y8?W%o1zOxm+J4+Rct>Vzbl2``PPr>=nMa!fpDrJdj?=5hSk3PEX=l%{-Qaud$ zWLn7K$6Lt&ld20WL#056>rS=DQE9D%8z%E+oXNV=Oo+@GUYk6*OV+l6SjcK&uV`%w zI+LB<@440~=}#na(dj|4)nXZU(d6@MuY=rHrv~NUx$%h6GZM&5Gj$n`ogs@$kQ(Pu z6y1BTg9_U^wBBm`Fb@SKA|`Ju=#0LBbCKuUzP>Xahi*C-zFBa;ycGE2$``<1Hx5AS z@5L7GLg-qx#Z-JdookP>cAr6hg3$X zOec6%&nUi=m@X~)j7>7>OR7LZI#Uueq6y`A;I+5DRL(wA;!oo_Fq%dY5RD<7i8{Wv z{lQV=L?52VSu?c1|HA#!6&E?Rpg_1`%1 z2@p3f-ZTOwmc3=>Yhq=k@*&38ME7PD;|%v~DwpqdZrN3e6IzU-VwLYj4JCYZ!Li9V zwxuP8oLaF4VI^)q>rM;ptYehpjs&pv7Ocnb)a=jK#M1hXriu4iS1gt&{A{<$?$2%t zUBRI&wxu*G2dCjBe3S&OVw(!>{bb%ldum@)Z&soLsLOfHNY727ad$)jk>o5}!p_Ka zW%hahog{HWl;z>NlPXRXox?O;lp+4pJgCoTV#|-F$9mh7UCl{KHaEL|b}*@qnYxdI zkDu&9XcrxZd`YPA`wY#+A~vDL_aTutZf-h0Djn9OZtu|HW=e(W>s)H+km;H2ePXZj z$74msZt*TWZzNwKAQ&{Lzek%(F^&Np5zsYDo6q01pKd)gaoEjN@oC6jW;TQ zs@z%9Tq<}CbX2x&4N>|fb-9ut9ew|2I>i=YI>PLmpRprHMCOFK>$pqbjNy=CZ&V>& z#@tMiOJ9tgxy7!IyS0uuE){qVl|;2PmmbXPAGjOEhzYG<8QM&$jt3!Y1)dO!$zhxM z8HBGpja3shZovQ@|Kf6`hKBb`=2eB%6Zhhi4((ii+Sa&e^#IA)?@SS-5}8@wBv!{`*9>l7Tx8MT|>Tn|hm&T}Uq;ZMcayPG)W`(q+*5{Gw zWEoO9*^v5@QkO3cJbJTkT#VD3+gk$CTA%rD#1esYfPEuQyyt2gA35pS<;7K^`g9?O zqBQOpj-1G{;2&kv@s6(Zh&N-cl^;eTBmys%L3^Ho!{UvlHkxv^hzN{yaq8&W%u(vC ztOO3~K1I=rlGX&#aW{^RD6U4gUOIAXfz?(n39eBYwlOPaVGaN9-AVPQ-529tO^FI} zCwit?cjmC!-okK3vb6A^k|~eu6izZhO3{#1|DS;t9CI)&^QtsnOuW0$_#{lodhkR= zqTWhu=^%C%FmzYto`D*iOIP~6A19A^5q^E#(v>P(mAwM&!%^LS?{;uXdnWHR+ssW% z!6)%-Z$$S$=sM{;VnOt`nlQ`9=P42&^4@p4rd$A3&FIa1f8=;?E+WY zgwrn0mu=eRS8W8`&aFOOIoT;mvz{loO6Kh1Qa+qly=|cn=^Qn@=#)_UgJ^<0Vu#ah z#mJU#(qDAfaE7NXnnxBSa(?WZAxy#V>PN{MawTvs5XsfG?o8x&R$#VOngf!vkn2WT zu4;3(t0%K?yOFVg|Ge+4|{Xz*2mI3M=2oaU~jKU4)r25xpJZ_I%1+8^0mkCb+8BE zRygTp-jTpk+#L6+Sxu(kD3h3OS-!|M67R}mr$zHTU4_{-p~Mik(xKm3Y`Wx~gpW`o zjaQ{EA#~5{0D16iX>}>XFSit`+vgqcx|Bb%x2_G8=%|Db%?2ai1aqmrj%`)3Z3v%1 zl2EIiBRB<75=?>1&XJL^!x{CG15458vS5F7>8Hu+0k;;baqgh3ggW5;^j9xHM)MQs z2>WK=w3DTR$bSH?h4U32LK~bglpW?Un;L<;$YK%0_ef?se8p4Jj6q+iQ=p`kBu*hn zzNprUPN!L>2U%R~nLgEe;uquH1wQ6Y-#v9FwLoPrHeHJ#*16jY;-_HFnA-wzJGGgx z%iBSZ`}CGC35Fp3YIQ!{!_Hz*YD4-JlvqP<->_3J4zv#Ts}i4^rH^ro9%1EMJ@+Dt zxLMb0;a%gCL8YbFvCsPTJlaN+h{M+}yF`jmk0F6Aw!;07& z!ILc!6+1j1H016*GqHSmMOfSEz5v@$DY%KYtkI(H1>fpj>$OC8dKn%o*3NwG;zSGu z>Zk1&2``x|NpLN2$H;hbd)The^(NJ+BKL|Le^gu8%>#C-s?hRZ722Lj9(C457}8dW zi+FAnC9LBxPB`gvXG?K4aU+ZZaorV#7P+S-{l~k%_TFl|JYhmR=ssB)?N<2%7_jk| z-DeW3>v_a~eh=i>ux2%u@Sj}84dG!jpGt6y`8Cie!mu6Hi#%g{Yc;3|7(py1V}cZy zF4?)Bi*#U~D*-vvzuF-f+{8@94d(trw$a};iCu=!`=SJki7SL*8v^Mn0>QszvIk-N{kdWuOdS>vjskLiCN!XkY+pw}f z&FC+<#HB&UAePDJB2Y+d@IJ!%T7sPEXuW78eY=KB2wWBGhJN%~IG zpf9D=&YWfqi-McvdgS-ILyiVX6r4rDDz(vlaWfKhVa{#lDP znOCtC4E*;32xF1PD|;c31Ct&D^*It-qc zsjTcg2jb3`A5bpR(zc|?$~g*~-fQ$bc1}cch^^J%Um@3*pk`OZ!YTP7f<5kauoq{=*!GxZQgT|L zkiN7H80}+AEIT}9o+nQ0n>tClu zHU%nLps(hkowP%?o~u-h$%eunA_SD>%E%Lx)%k!JUbH8v7Iz=MGZO*!#%ZX z?}(=y{X15PcG4E-b_Gjj3hcsIdJA&Z#w|MA=_4L|7iwb^-IDIy>{s;KRu!1bNH!{J z<6T|kaFOtj?k=(FOk7&HEI($r>!imetZ4IM2io~<4;9p4Y?0%-lHfkaEU@0A3SBia zZwRfL8kuj=C2kJgZs4o_c9uBfqT^Jny`SX!M&VyyeH83iTt%2FL}z+qZWZ@x=m@NW z1J&KgDELm|dZoA*uCdN!$!GU-#9#%1Czw)Y?}z>W~=c@89u0Ud(gR#(YvObiuub5 z0htk((5)aOuaWm)c90bn4Eiv?jnJYgAZJN9*qx3uJaC$+0R=i=gpidJLZ5{4Lvi0- zN>qC7#dw_cB+YL~w>^(jA=Jf+jOx;?(`O_@vh5d4`BI+R0+KztLcp$yvINE$FnEtp<8h0~ z&cZd&-DXU_dQy(zn&yLZA^}hH3$>zdJ@VuDkb+zn1yEJDR>BQ;t%N}MEL!~+u=cw4 zwt(+qQ9=I89mFJdN-P=F)jHv!MB~$dlc$Bo-@-de+gDRz>~Zrg7u%k8O;o{jD>`DTA!3AF(9IL`f?H?-I9^D3&x}nNuhr z1`g$2=-B^Z?7icue*Zt<=nx^5WMyS!lO5t9duH!Z_Q)pNNl{8$+b1`Fg&VuOxk~ODjXg4qax_{w=?DbrUN}`%Xff zU7Fg43;^cjG@tjwD9qydFW`ReJaUe8)2Yy4nJY5(TFP_=wV;}n8a$9!dKzhJT5and z*+coAy(a#Oz=UHU4{v<+YY*K(eWBiP_JoEv6rcO>NgclMZ+o?NP#f1BWB*$IT3lN=w96)fj(hf%N<{KdzOvjuj3pTqL&hZl`*NO@J&WweWe z<~o|PX*4GwMW-3k-K(sG_xJitJalR{dxU-K+{f(&rfWTm-G6!Zbdvw=EF;iXwrM%<*icM8?$J54Y@y7c|Rb(can~ zm9s5$R)2awCJB`58f$^kiuDD74Pm`Yxrr-N1xPMeH|}NeKF$z-&FQ?S&(cD5A*p>_Zp^O(=5SfhnG~AUdQ)Rj>j(K zuoRvKVU-aZe&>b1Kbx!ym!;wqX665LZ-?E9M2E2hY$UhrgZ)_|7|99^nM zV%skHE!xRq-TJ)nYR6fL1()?nD*o@uQ-KqT%*jXvG0SHm*w1)fY=B$kmTx0H&?dTb zdwOAL-Js6BaIqw1Dk5=daWcEJM%)&8RAT#c$h3wZlX){Uwy--_@C?ub@gJnj`8TAD zweob=bYxHvs;rW*LGGb_m5k!~?i|1V&zFb(_cQjrQy2v6VaDQDDrl(R~#Y?9c zjNGgO0}q$53oE7sn)a0TS_u8%FFi96RYHU@C_kI#HFTliXODhQP)p;&0F=FA(Lg#O z^BcRqePwjyo41?Kmkf-i*x-ilqf#k8Kjo_T1sZ=4OdH6qU+R7T;#NwW$67-4+cmF+ zJDWGYa(tR?Ty*pSPA`h!*(d_>)yzATQdrBir`jLr()7{rsdU&hNZE37ab>%7dY1S4 z?f0k7WBE1MW3k&h+Rb;xmzlL+wXB{h2EwlDKIx`iUE4^=XWK1Y9J@Yn;0MMg&SB-R z!S+3eZOiS)*Y@`#Q-5VFnsPU^l^S@W8{+t>lCJ_bbe z@^i0qH?}Y#UX%w6N)nYT>E%lV*iR{R-|{Uyz)KLX8{ZxF%r;^NHq_r?bfz1J#Zi50aqvSvWQyT+vOfN14$5L3muu&;@ zbS33Ve#mT>5_kWt^c~6KLogQ7ImnoT-%dv(QaJz`Y}-G3fsc+g$7&~P$8ez%VIsX= z5>V5&82(IXVgv+1%sdudpC{LfR!Y2w4O~KrG&V1>E@*w$6Kjddb#NzeV2^KHC@` zv-C@R*^bcVli+K8JhV1Eihd$~2lf@4`VELsyFyQXxB)tSz&ASNoG3-W4)SFN>|6BSERX=+G&FlZ|bd| zzH16Ac{(XnJN^Na--_3z*Dao7@H1P%{6ZOd)5EBJ_nbMTM&?do%62S=;j$TD1MuXO z-9Ldyck{`)%pH~BV?x zsK4(wn0CFBdApHQC^f1PwRKp~s%7JzcF8{nDMs*F4UNvSEmZ#C`S;aLm@@ii}utYpz=8hC~k@h}D9X8!oc7#-*zNzFjt*;^=mbllHHyUJ7 z_{{umwD%O4(s}F(RPWJOVAetoR{>4T-Cv1K(rwTbO;PR}RE*8CFra4(<=H z76ppl41lgdT#If~j`dup#8wS{m_ZZJJcIQ)j=Y|h{WKj7Xy#U1E{kVeoIyYUR#qv1 zEjRBA!chXUHSIvKvs_Z=#9>ON3yJ#XGrUNFSxLOOwB!`5T@O6{q?$f#Z0rf;#d;|; zdV5$G6i9z!fgun+!mm$Sg2;{FTAQ?sl4&J-nfuo{nZ|3h4ms-rZ#;)aUK}_m*L|@m zGE1Ld+SG zppff02MTi~O?Di$rYz{gL45t)B8hg~g>gJd{Yo^(I_=)7EIgnwb_1 zNPD67%67cBGJ%Ns$bxVQH^m=>9svA;!3H1O&LDmRiA~=BXu0St0(wIwXoU1b6m#_^ z&i7nVf#uL1ekD!1`H&?AN$2$&@Y)Bc^Zy47z6yCyrT1fo$-O${67BEQm*if+vYiBc z7Q*|Gh0QjEEhDII(%|xcFG7q)wp@`WKkZP=sQ}ef`IiVPOdNdafYqLsMJKzrc=XIr zik$*hx&!%?=iLAmRbJrAi`-uGY{W)`?>VLdJ$%Z z1G#$*k#cJ`Equo?@V@DA_$y&3=M|jmeW*%bhn0h0zxq|2L}TT4?XZo%%v9@zO~inC z-XSGZuP{wh150|9c*vc)|0J?Yj6mP~kp3A0-OEH6iMdxI*T1_xJSI{mKAeT=QxQ;* zYXF|e{gftfo_QCEopsTY`pwMQh4a66eO?C^>D@H#{%JwyG6e>~y~@y3X($W7$J5sX zdeHM4KG-&eJ$oSQh5(X%K2F}4bL|fh9xz0!JJr!`opv75#9$^cr<2iv!rSGt?8ijl zhUtK$kpbW{-`_JPN&c@Jn1&f_c!YEI;-7;E+U^1{O$h_uQ;`n7gz{wasy_uXvx2q7 z)8ZJja#!bUTMO(fMDJz3H2X=J90QV|4F`4WYFPipuhR=mVi$i0I>hl2EaQ1YPA69>wrtMB$~XWZaZ5q?rV+&L%%{+ zz4eUwWQmF4h#53`U1fp~VS4r|Ibmqq(w1C)YFp@`@V?7_0#w3@a1G3LI>Q@D0od7g z{3faj`*%!bn`p?YPO!%4O$i4ZK|_4z-v;i{teUzT%*S0^N&UJQ=oPe@-#-2{@&go2 z{P~=KwX5a&Z{pAAZ=oYdviryDZ9jw-{}hkp(7lTP@#>qOHTg{o{NPh0Nq6L@?9pGj>5c;^nIePv==jzenwcBbU}Ky z-R#DHFmUur_H|+2cUj{t#~Yp!0d2cFLFT0VVffmoxHIsp9|1V}-v&daTaL(I&y+m6 z4Y<(%o5cQa`ytI`zk*oh+ApkHJM`;HFh5jE}{iHqX?2oWK+zPEibMmt^t$)`N z=em`Bv_|d6(bzR4NjpAd_~BoWG)pl~)!=T&S%looyG)0^9B6-3TIS>y#n^%?y3C{9xARDXaMNG;k5p zscW^T2*|%DAe1(I$|MSq5~?P3v0|!)I;q$EMR}4&LA_LET)0l;?XTp~8^a5gG6sJQ z=@3-MoZw#E~L8mCXx#o1X1~9)J+|Hr$Zck zf4E+2XhnHfOGuQrU$=V)^_wda(0=uJE%;qf=*&?{nycJIM#8G6qjr)R?rD9RE9K%3PvT}*o`*L*}lWf2!0D;IRZ*XW}*p%i^Y^7$KlwgC!T;Q zGSZ{Kg2T=4vA^X(^Uq9#Z?8Q<$lcqT$Jc@Ku5?DjbBnxZ<;MX@b-C%Tb=Q+^0iHh~yp8GGcl zi(}ajNS5g9s>UaisY=r->?%082Frd}Fb42-LO$|4qgI|^XS<5tIs~L~O__iY&i7y~ zgR7tF`9Lg-|Mp!hqOBx6Yq$P4X=A6qmD>O4p!nG+KIW!1mi*p;<-i(BX9dE zR}%afEEVo&Jjb~IP(k6P;YHsy|26ZaBUD4gZH%TP0b}CZEzBoaHK1(p=Z3pHQf<#N zO#o4}5`yi#s5^L~79Y);0m!(LQvuzMVOW)Yn{W$VT^gFMbC2e{+vIF6r zkv0N(^9%dt-FK@)%(&gFk(Cd_$Ur8cclvm_lY8UH`$$M#l)jLWf~6Ww)j2E^1|p78 zJFR_V{hJNFA{@TOjr600J@NdK2X5V>_v*=1YV558IzZRcVZ297NZgU_PTY~%1#F+- z(Y5oPiZk65ahrob3*x?cQ~Rp!8+*QiM+HBV`lEJ0R~zV%LruBN{iB@DVJb1gTSweh z8;ZRL@4lpps3e@Q#k@@oG467FOzGVnMH`tVWD>OM{Q1?B-7)?|DJg2$s}J1w3TvJD=qHZ%FyKDEd?;465(pPc6Rvs_AY3(pY!d=FuY zq22JSzRs9L2z3xw@ybA;-sqE=^O7S>hz-u35IX4AXUe!~`?UYH5i()ip`;GX+3j-l zxr=u^R}rae{ltZFr1B@BwxC-umDj7tmQ2>k&Gk1%52a5%H6~-wH}Z2$@akXf`A5sb z?%`TpkLvlDk$Jo(U{;7*@lKq1(yAI61iP zy|DX4tyYpJzV_I&T0h&qAU-g!ywkAPaM9bev|#_+v3zPfd1nEau0A7^)ahgAqyQ;f z9!31_X#K0i(AKE*s;omr>3HhYEh@KI7ZR^j!_<#v<&k!j-3c89x@!elbwM4^ds7(M zDIAsyEW3ZnO`;|&@`D<8M2<>uCGL{FYXaP|igs4MjyY~GzdUM)xt}Q5!aX^IkMZt1 zmPe{b+O(POlEM9|rvwYJYirSb9*3bOQO~~q?#Dx0svH>F895iPeqtK`oS(}VU@@UX zSm70%l3h2HL3-};NFkszUEkUIOfYF~_&FgUtVe)v3*PEG?j{wCTpYTmdxd~$m zTTe`00Mr~f!sohbyrCxxra8SweQ^r;Bq~|ESK6 zb`{OZvCF9sde8qj=w~G1{0BsyyC0jk&hfYdV4@txpQe2yddlZv!wiD2n6ZBeu`XVG z96SxCJ1bW~V7X81G@x*JfW<8f@1a^U*-00N*dqqc>mnRE5(|LM@X0;B;!U~>oy!zO zR9i_(OKGhW{mXr@DttC$3lgb7wfy4az`7pH6C(iSh9KG(85>U02Vgs37FQO1m+vq6 zZ<~{Gx%6$Z=`qE7Og)NGSP;6BQ`_8Jww;>{lV8b7>;7mgI66pOakA-o0d2*z zGRbqxw@+HJ11xr1qT6wiQqkVCfFIZ`_%{b1&-8e?GBE+UTbv!UzE`TRnirK{Gucq_ z!{w=-d0Gpmp2pZz0r9!aD!r+daJl{@{_+%fWMRhP&r74(t)64tC3d-H-CX|4{F(97 zHlH7P%l6U{j!iZWzrKsQB3!L8KRIXKs|`fYX3uREK2V&zQAquUWTe-u;9eW2r3Xjugu^E5FjUqhostk(H3Ze0u$|1l}0REoOab z?QYY!i^Fuv(s(~le}DfZ$~!Niv9o4jYO`k2e&uN1)jUh zaUIIEfR4)wgP@PTikkY&4DZRuG1?&@pt>o6u?JtUA%G?n?=Z43-!CVEK7WJtmp;m{}yu%KQH zG+~c7LmUVxLFE?$N)_N~w>cNn$*|ppYgNv3gr?TeA;n9#xkk+Ncn!QG`Fa)GI*-k{ zFZPTpUD~^gBsbXX8X*Y&q~ct3+jZ2{y52W)vGsumZo;cDI9XA_N0IOAde!n*>pW^b zggeEFtSGi;QseNmbIO%S3NCz~Ppc6X9Nu%Qo!V`nM=lcQ^)#hK{v6u=q^~XCnr@q9fhgs$jK_j>F!WW0G#l6OzM|9Rt z)ROa!mm55xD=#L6>F*rN-x@JLY_ysgiP{~izA$h-`frSc0qJUb3q4g*=ua1Mby|Jc z4+^&Xw5e5B&Ko7KFht4<4)SsJ`wWvLZ_9I9^|et(b+~%y$II`yEuLTbl=!H6zNxf& z6uHmUs2hBB1y_Elo=>Ptx7rgYSg{+6_tG)01s1IWd?Ibtjy&5S_O~>uk1)EOS0C?@Tpi!aYJXKflI=4+ZGSqv-0<7f0# z77MmQ#>OX+X~k$|3Qh=Z81Oa+7_TA>Z}}Gc<0A&HZeS_x4yvTev(Mw;{=RM-P006e`8#5ikmks zT9st=<9~@>@(d{4yv64~Wu~tfZ;?FG!IDr1z3S~jhX2dK`WU56lU3sE)?WQQvKnHdr!XmDrN87lobF~lu^?dJSDkW8)nV5BA za=xEE^&@k2>*=+t)0IjO9Eyob&wDU34V7$ZS@XI~T9pRv@x)t@@!We*noc_tBBXO6I1BHH}_5JI~~gC#hd{W40}HI=2r~8Ka|iERR)= zw-rM_vPNnS^f@AUw+#PSBlVaGInc08mi0far9Pvr1NYp zwf7pwJHtTF(rw!U?UsvyBO^!osf1sdPTAsY&<_4sQ}9`GIhZMtZ2X0Hl#HYgFJx}r z+n=fZ=Fz)guDnz)OLw1FGOx8uJQK$3J7Pz?={T8y`pAc$V&OhV%tNU8_>-Nb`Y=WQ zCfaFVC|dTi4ON#}7{ZOC#av5oJvS!Bqqvn;2{`pB853_lDwE+Z0(`bXB|37e<8*6H zyB<9L!!o0@SClBOL4x#Y`bXaoJWcQ%-N#G1)L9vawCgZ?nDsjTRhJ)THCC^~>tW9n zghdMf;gtLvxJJvj(UZPx8rA#2i6owzH#?)w23`kPs92q2;T(Bj+=#2pu;XV|e8;6& zkb?mYFeb&EdA?zmM| zx{^d=D-F|beoS=`nXu@A7jK7bmPbr&UsKqA#2+U~;%bMajCCbxbYJCa6D*F=I&kD! zQhrp5Qe(8=S7UlNSa%cM27DS2K?%^jnzM;*nJ2P@L&Ft;DfBDW5R;LM$jT~8QN+T5 ze9wbm>v7b~J6uoin>-8O3%v^$eKgn8H|?d04Os|Y7DYmNm#04dPqYQa0GoiNI`U5n ziUDUU<@hrs+a?p@Wv8XSh+5mM zhj@LGjc!ICwr)7ybwhQuW+p=B(%um4^~p2)0iw4#l*+lo(gk8M#~cYKkL9jo%D8>Ln&|097f?f*93jHz1@<(5oMjp zuW%Q#tVQ$Vhw>Ma(?-bVX^<0{GCZ&*ZX?@MB-l9=kd*wGR zaGLtQcP6!dMcT)V%nKy{Lzn&AG~j*jJ!kOllK!dv0IB~DEN;W1y%aq~#z@?xEoO_| z+E3$JJEN8V>cG<6f zW$ToP3%(9PkZ=n6BK9H^9yKX^NReSCjp!R#LO4%T%ex*$D>tZJCy$w8dGky`F)y0c zy91J$fILcvq~Fhe@d0+X$Jrlm>E%K)%dB;I(p(4!eQkHR_pKy#ezpd7&hs5pX|Ng- z7yk3?4}%sIs3siNDikZk$O7{5G(kFTUz}N49u! zY=9_0b+KHnYPT|9UbV9?9}Df#XL>WbNgt`G1A7Ctay%Y3E`O4sJwX?*AEU6mC1*>O z+n14h^~T_`K~swGk9L_JFMqrt)yIsgSq$9XMHQmI*yWOi*^~2)cwKlh_l4{24jxhr z$#$MMF6U0=jmvfs&s3t03>J%a6=4Kkn!;`wSk7X-~8soVd8eInuY!`YAW7eQ5sCJia*e?#QUunw+w=hYwOOSy!nqEXOib zITf0zjn0JilxC1=8Zk_c(o!}ZsVq&^=_3r2JqIg7*TX#I4mOY9bYJZl*Y{y~IJUgI zR-ED8%aL!mBD9sD=&1D4`?-h{LF_`sa_#(sVa^vBF0w|6Jz7^fSD#rjl=_D)s`{>{ zuP{eqWVY15j`gGUnp}?%aaJ8O_R&(abl%il@~f0ffNR@r>LY&qv30u6)+# z9MDGL>i%0%ScU@YDfI4z>)GAWr}3O2sxdQySp`PV(K9CAdWfkQuM=x1#c?!#>>P8T z-}kLn5vrh2V*koB^g4l#u8Up`=DowpesBN%RqG|a&AO??!tH?{bZ7%kGLcQeH=-`T zb;y}4rI5|2lei>tpNif0^y!?`l5YDVgIWqQFw>> zaa8$Qo)v5SHQS^77K2yq?g$f#Q-%0Dx`w|4SuJ9e>Hp-kPwmX_Pf+x{ifG&ji?gQ| ze1BQn0isw1uKMT}Mf_@06#DAudfeSUOV%0xUPq?kQI{QntxAO;%Mt*?bdIM&;k#5l7 z7)jAz7We;%eTR~8|7Jk`TQhhNme%nv=HC3_;7v%ji08=tV9bwTtd3z>=JG~XKf`~{ zNe3X_%~K5L{mb+9=LJ-cF+Y|;ag+y-53_?Gbm#6gyA*p`$miZKMdAkieMuBogXpHF-z7zWN_xZj&Sf$MiAsAtqKZ@h(;(sCXyil?3)8p*wFKGRdk z>Yo!Sh4pyXC@MHvlz7H_wx2@f4{t0fy7Gqc1-1CvzYhRjan}%BV0`Xbu-1Mu?1-*U#$->ney?_xdQJ7>Dp%VKSgpVF`6qrJ+b9X`>sYZLED zBp+1$d}T*wcrXZh)I#vW#nmRxa@-zWH(?UcYh|kJeC0 zRR36{8juK&){olxyRtwtNkgftNW5o?2$yk{2fNp@=%_IF$%df?+F#B+Dn-5+b~-ts zzcfZeHqV64vq%n>#cKru~o}l#GTMHUPuR~T%S+>jV zIZ5gs&Bw50wThmcZm=wifIT>`8t1IbzIC;4X28(czhZdkm!@|d&%s@PPFtLT>;ON3 zv!j!Mj(~zurw%lF1E*AN8<_!@)V1synT^x?tf%p~o~9zjFwdP?0xl+Sngbtvqd9$u zCfr3wbF<*xwW5AwG_4f)QtYMif1J-aAT$qzK$qspMm96R)&HF09AC1?iE$td zNB-*%fVM*bNqYO~_UUdlUN0%nCY|aXrao`%W_1330)x~Ly!erD^63jeO>`jdTM;aw zr$`PO)dkFbxREh=hS#t5?82$WQqTN?@rueGZ)Mc7VRJIUM-jDczv?BR?N44b8bk}4 zo-RFS!bMBWu1a-FfvEM9U;6LY@e_ezBcNa>IO{Y1Q6i@o#CZo!P7R;VzrMlgG5YTl zDL+`i zIvRP}940Etni)mb- zb|=}h|EwLpd_eO6d_?enSaHu`&`Lf9T2r?k}hi7q%+@5r~cKb1-KADUY)fj@}(E z2&^sDN6&SHyZFI{3>;}s+|q{YSX-NR{#n%eRU?pW_s7^dTL)>=kx1W#H+riqCz!rg zH0yWk|D83y&AbtqDzY|uyF1)#HZ8j6yiack&=;Su0N&3Ma(!986e*XsZ}(T|`~hC) zXPa+U?Ek!_`PDwQ%f;!0OWUS*f$_rGB1vK~|E$oY^F`&)D}4rf^K0apl)&%P)_e|7 z+(2>U3C_LAb7O$FFnd35nxbUo|89<~E%>uf0b;1#|HQ*24t$M(=-(+M;HJAW@DWMRW2QcJF+8hJQQAXM+A(RV z4U2y!l=}g8gSmw|mFcs(2b@i8`i)QXh3E5B%pt$%p#eiG9*K?rsRypdu#FkQIoY#M zr;0(D{`qit24DBXLi*5e31Vwn!p^6dKIoyVoJtNcXwuG-l~V4JJ0rb0=(zxVuI2AE zPA&yozETo+ACi{0kmATNN7UC_c{n-*UjcUhxmAh0u^(rZg2M$&^#&;&)Z2hzRP&1W zHyOH<$tO4Q=i0LW$`yLWFfLAPZ_7v-i&#<6+!(agLgNDC{eeQWts*oKJfpgBG|(K> z%kKyce)FmR-YX~pjuj0H_pvml8lPkFyAQKh`pywg!)8k}HBX)w1M&-iQad<368uhe zC%zn%M~e3$8q6oLbB+2RmyHyS&=l`?qB*&FfH8zuU7kk@n3-^*J|v`7cV6UQ=pO>| zK2!7U^^PNRdKItT_NT}3MZ~5a(c=fLmsTQDfs8F?u6WRuS}J@I zIxW5aUik@NA127z&ZvYw1!(XFs}?Wm_DI>sk9nZGE`g{-tbe~qJnsPp3xRUs(VjZr zunflujtcGUN=pN6CjIFw_HQ5n>Myz5u+b;~1gsZz&4)BxE60u=Y_@?2KNV6$aX9|? znZ1X?(cnJ`mJ>+>OWKU~D}YNtRu&fa0<4N)CQPUJ{S7-K?ti=i7@^$+y`cLO$a_;cP=bq4ytFWC zMe%oqa=>I~bxkLVcmIxWL_fzyAQYZe8fT57Z}IjfZ2dG^_Ut4J;`7|74j_l;OlMu~b$BVsx53oz7$`$p*5 zjepVMhNccIj6k`ps#LlfITAT2Igi!2Dc5|LOHa}RIc~tb=WPpr-ad1*|I@zBC3w73 zu89>B2k`BSy(z*Qn^NhFOxtaq;<=zyncIJh!Sm-kv)XUUiD|dLnSoR((QV0Jf4Py* z!MuoOj%NqPDKpQ`Tm8?!P<6wc?X=JtRo!{Uu;a^S=pF3F*^WP=EXtC>i9YMdRC)J7FDtW6gpHsb`&Cny!Eb9BO^Xch5gjC=aL+gvgj$T zTH0`-Y&wXRz4=C4MmqsnWBkPaVZqu9714;@gT%tQEMYOuG$*3MIgnRnFb^ty$zi}o z8+rd)3OC4f3&z}D%T%U}-K}De5P8>|x3W!rSH`t{`)WL&HYWSqdkZ!Q+MggLC$Ba> zQYOw_WD-_KiK=vMs2R&6iRUo}SvonZ4&E0H4a(`-M)s5sP*U3DMY=3suwEu-flL|O z(xzqdwgg_np;_$uUFwjG{vDfj!{8lZcbzA9b=A}4)Ljlvb?GvCaD4KFZ0-Nd82~;# z7D&a}Kao(_zj6J>Aj^J8SG~bim*Vguw_>5uyF!p;Yzy9V8JSK?_E*O@yo@NmNDa$> z|GC#wH8K?1<6L52z>{HU1x9Aqn?J)13H8|d4GCNn_Wd~JtN#^Q&^zu>3~}Sgx^Vo# zOjl}6X~?_T%=a0ia+Q$~Z_1_k*omfF#jlIr6-)IJH|%5M^7GFP;)V8Nf0>{NM8g5g z6qw97JsY!oQX*U}>ng=`mp6QtqAa~vPf|5DXs(FDsXIj{g8S#cV6-^*NQ`4iqU4Qh znc9hD3a_^^s^1#nAJZE;r_M+v(2}b;07kRie9&>Kf0d3_C4FXoY?z_0@O9; zy{TOR`cP3?O@>ge_AuRz0LX}53l9M=U-7Re6~|wvrGoo5Swb0!x(vA)-^!K4dA$`C z;JLb(MdzX#?h>s;DXLJSel)~BTp8Y)8t|AjD|yDKXSp9#S&*wJ#Wn=o<1u2z~4X{vE1HwdsH=4fHM|5Qal5c2HvKhuC9*M6 z3UTt9MnDv~GOhRSB95kyQ=j8r;+w2O;9k!Z9%IiP zDvLPP1yek1Hm>!tF?lmJes6y`nzp+NkD@ZiBcr{2Uf zkUWY%Oa{0Lat%a!>*UggpK1RivmFuJ#3PyV0^WD(1&ds;b$HZ}Qrw-muj5lrm+<0g z4_rq?vlmtU!5I2{Lx+MMJo%;914~4Rxr-rE7KwLq6o(?Uzjzi38SOvA%`hKdM%Ffr zk*)4P74mtfys}*$cbM5SC}ETS|Jp7{n~I~2D{^9<4P~@=BpCbgJC%{jN@jPGT*4jm z*iVTLu`z&!Sz9S+@Ir)$YqKtA#pUTADYz1AEESz*9KI%3OSpR)HwKn^oUJDAv8pQw z=ty3snpjwM;auGCU($&!XC8^6xY;*4QV#c(qZC7UyGRYW>Aa8WJ%;f#Meo$vuCdnO zSA`g0-8w-zve1FpA=+u4?etjv_emoZYW0hLL600D3E$Lc3$Cm`gU2J=IaX}&I!KVI(edKgsQY-mC>SbE-oR_N7K_)A+RG2D3zx{`&q zvT>D!>0mW^P|si+Pc7%Dk?$gUNbs_up-0#9)Db|J&9)zFFP-HSbGvyt zK~OVYo2|?%(vE6SfQ$H&!lt2d1e8};Blgnhpm`ZO%QM3szGN4xC^BzGv*f|(>DYwA zklKNOKvC$L3G!;gR~EU|X}b3FsasQg@NJV$rfk1YeA*hztJaov*qK5p{-1P62GTBK^PR=XSa80 z5I;nfNTWkn!~YVsI@&ZGQ}0)L>R^VvHu2=(VY zx`G9gIl|)3H9e%0?tS6Q*yp-p$>1}JpFb>Q`AT%4Fb^@wUg|Lwk#lLzk{kM&KPRuQ z^`k~)RX^jMwrM52qeI}>V4E|v^G+Jik06b##4+miFdhm%x$K3%=$rF{N9Ixmz?=(#OLhF*i; zDek$eGady75f$o$m}$6?QpcELynEd3^tG8QmD2-~)$;3-)$x&1AeDca$&kkLJDk_R z<*i#lkJEr_TLZ8-c#Q9(oXEet$_!d@Zrx%aTzA<4zQyepcw*A_Zk>a|{x4nIyH!VS z)<&MGl?9{fZ))%uN>Oy#Mc%X{V~p%5Q6|b_5r=LfOC(-z$}RGWGIW7Tt}X~kw5cB} zt4Y*Vls`)X?q?S z5-}4TZ?`xW-%rVk_-SuUMF*&)ySBOfpq6Zs)aMC88R1s15V;u12^~22Qqc3fxSJte z?Z_SGU+)D(R&oW|MXt-QpIaDO^G6xhPWiPOxlaA5 zQFmG>#hYU3Q|;zubYFhf-kK^uW~M4_VYF`Mvn-zb%_2v<+_!|mZT*#t)gLV3e1g(^ z%-;GZ{vsDl7iEUwCr3X5Ea0^-Eu#1m;RatGJh4u;^Edko4X-RnF{C;+t6Q&6}+A(+9WjA+ZW_)=Z;ZSjJ2;Qp^0 zTHtVMLlt;RTyvP&ceo5|Cmw&}UJFwxE!tQiYmyj;XH9yZ{Te~cvro#7R94wa z$x-dDvOe{lJhnH-=Vy+_t%5G%CEDF)7b-CR*>9jF(}&NKa*MPcpmG@r%l|G`;>jAhahBlXnvt) zWw8u9ofjvoNnwBdZSfStqoB)5>(l+--hb7*vJn}89p?5}tSozf%FVBBKcNH*W^)bb5pv|kVO~<$%&r^)nD2^t8z*Y& zB+>B(8%*vMFz4NmJ9({V$7IFIn!D9{RGh*xShuer7dlRAyG}|#j|TAYC~CrM7&vLn zsj~^SCSfeB{cXf?HhO;5_eeNO^M#Q@igAsA=Ib`<;wH`BAgwvOAOVW)>8m%shAsE) zJ?q?z$h=dr1IQH}3%_)|jMRQ5Wit2Y2hR6p8Ull0@(q#bnXhf^ock{ekTrA!{S2uu5ib+t*gA{A(RT z2+AKG0`iRU+{2E^Ry3n`vo$*K33B3f0F-pi;-G#-G+97?A?{-0xyFOX1PQw03z8S; ztJ)#ScxDV1waza3*6?bp+wsvt#xXsG#PN4zw!*Nte1wKI8MS`tP*ypVeC)`%vD$Lu zv~MvDI_*zD@J(uLvjqPevlWJ;iK>+!%!}O}J1x>Es>jsp@NJq{`yEluqwG?}eMF zSZp$GmH!9?v07rhooNWE^KKvt5LLY1e_>p_Sf(7Bj#nJBPKn&9sZZOvZAWD$z}`vS z!Mfsl4?tby9N*Kkev?$0DEW?UJ5maqj{}3?hO{493urB{E5}ngd`uIp zdNK|-M;{h~oh47!=+H8U_x&i^-vqY84Z@KinVFBnWyj~dos@p~KVL%emj<_gSgTX* zK#mmh0Q=Ti=FeW#36v)(6@W0TmXP~Bko_^%Ysebcv2FQ4@Jd#+?rTd$u>ilwL9a9~^DD4$DDVNLs7g6Pdh$4&C&S3KMbj5k zeQxpX6kXk-r?B6_^r!uI4RWq=AMX<$y@;eF;deQ{hNrj$BWGJZa;)>rX=?3qaN*$8 zS>)})#ALW=HFIYUlByt*lQgK<*V*1hu2?s}ThU&YeiDx2XjDT#VyX*mX_h)4Clkg6 z44g+Nj?t&v#D99rFi_(orpCUi_CnH6?S=lqD}h1gZCr|)*~RTyUr!!Snw!F# z6w=UE!40K)$f>nz7%VZcXL_-lCo;AsL-IxfZOcUF4PV9N&r^gAw{joe-Q0es@*qat zqUc85mWLRxsf!FRK! z#=7H6ThXwwQ%}Ls>{5A`Z{NX}#fh+R*8c8S)r$k170o_s&;Dn+^B{-no8~A{92O1I z#^UgcHaN&214Txz*mm-%(2+?{bS1@a#u%i~S~q+ZN4ofVo@&wVaW5Qw@@PCKMj)5S4B(Z5hmuRXeNTr1 z(%UO2H=8w5nq%PHQLX@kanp^`ie8|K-CZ%Nl+|= zsD{gWTGbueywY=WN;`sePbepr!hmyvsgAjajby8E%&3njHU6!xlGMFZ5obrmVC0;N zUctxN@*e^~4S`Pa+S%^pG5A#1lh65_=hcNJOLwi-EX%+YCgw_CSMFv|nfI$@Vg4o~ z8{$|?F*>JjIRuZFd9Eo8YsTd}knpm@3wvRfCCngFG%U+?qCt<#2|apS;8+DU_@iMr zOG|c@`GJWRvh>{3cE_qDAL*|98pBk@(raIie0h{MojN~DbVu9c!9W`YL*%&i+JpI; z2A#rUrN@ug`N$VUe?}g^9u6)G+4Ut~7|k{qslwD@K%PbQRr4Xsak_L-rU1MDROzqD zXG7f^{9JOfpQTJ1jFVXl>`g1@~$n*5weiB&FX`6{oG_@gT-ipwv!xg#4=M{`Z!A(&`B)u5l-9H( zHZv+XC>&_k^PK%LH)70tSB1ss19h}L^KWn2-%UsS!P4fET^d};|BmzLuV;zKiO0;dUdPVAvxM!pkQT7!A)Rh=o5^G(P{_!j{&8DC z{?J1+dk)Ky^{~^wULW5NTKCNH9CXWDiWKmc=Q5Qq5&npc1Z5_=Jc0OMeL7Jgg7faJ zszQdJDZp?x9pERK_0A?fbQ;AB=t47nNM?g;2`a}1DTNtoQ$n^80V+d}=>&(C@8YuFo~ z_nemf#CZLT@0$yp*OUYpRa+`$3fJqolPGN);k~@KU(>!Bi6xHgrvn)Shuc!gV^cjF z#6WG>!)Zpzy@_jY3_cn2cSodOQ!kE=>^Fi+|1Lcvg&m~1a-%ULn6B53QjtMhGXFH( z;4YgR_#!=?c2qg)+}s6mdg#ZNajVXtG$APOy^ZumJZU0Xh$y2G>(cp>*!?Ev-16)K ztNahCq$yj0?wCuYIa2+ewRB+MYoI88`1#mmf+QYaSJ}^~HQ_Bqwlv*k$`zYF;Zp-{ z0R8xeWh@)E!}bO1Pp1hq81Jw~jMUe!hGS{BsANfnwEGYyxUd1N^z*VTo}7&R3@{_Y zdK{9RuDUi)&*Gz&e&0+56k4C5pyAq;y}I!fjGfvD?~RT-k>r1WWwez?@Q_#95&00Z z*G2f5o?tmW8n;5AJ6V%SBdJFW=ZYgq){fv}iqP4pQu0(Dr^j5`f&FcRxzmyNYNi%~ z1jLJLqgO2=k(j@^G|mkr37Hi(1!etlUZ;2EDldd*341yy~Doy z3E$9gwxK>K!?gH0p2W?}mfe7C`D)Zc6<8LToyH0aeGbbU!YtjUnjj}2kZ*m18|c`W z35G4V0j~+7<@H*!_s7cD~{OlorqnZE8eQxO1mx^d_S4j;?L=joxujx@A<>e8ZtAB4X>~1~*!35^%t(iazsZzO9LK z*A-=mspxs@IbhN)u&5h2*4Cqu8rQ0=uK8Y!cw+Rnby<%xLk@-ewzZ{AQJ|$wg>$Xv za@{wS{`*7uCs$5AH!$=@T?Pa0l3Sy9dj)KtT95#6|Jez$(SQTD1>nWYa?LZAvkxwad%73i?F&NkQ+2XKf z@v}`wO#x!(1(v>vXMQS$46Y_v!jsS2DE_{+KE*g-GMaqE_v?lr z(ZX3A$@ylKd;dlaxDs-2qpSCQ0UN2Oo!6at5SUYxmzvQ(uq@(1mUN9K z3nV1XwU2%}A-n&M+-f(#d^u(UpZzqYyjt|xR9nB@G?%4Kj)75BXx%YA;8fg%a1Te1 zng{w4Nt-XZ5>!41HfH* zkyP(ZP6``3MM=6#i;`dd%b&? z@V7L45C63ZVID*T`5itD64II7=$)q+;PHd{V>tT3x7yc#i1J@BCiF-3bEnN{!4nq+ zGXV@XH=cRR&I>>LyCQa;mZ-?Kp6}Ccu9IPc4frldZT^GWBAb^m01dyjg14k|1oCcc zB^e|*!8ovrdSe*`0A9%aI?3}&0Om(__Wp^rz|Vly?EMRZzKdS}(nez$_~l$Ld^Ujz z#_+L`(TgXe_xBY>kmH}bYmfESPI0!2@@7S$a7{h z2OnLVDBPC6wa@P4sG+|vkc^h2t(w}*0~X_C-(+RIRNLNZcXnuW;BwGP>Wb&9%ba%C zKrKmsavy)Kon+Q~m2DuaYP0BDff?7e``+{n*}cx~qYf#NvL?x%D*-=@l8nC|5cFiR zp=o1WT{@m_YsGUGJ-yOJgA-&{egl;y72knnk|-61&_xI@v2%jDL$Hy zh%=UPTlga|<_*Sol)n44=Wz>s`p6wxrEXnBIpB0NDDrq&Ij?^UxHR)2*LidUsdV2( zcqqFFZrEyK&1EC2;*7xdkS3OekXbltXE^CF$5dgdP~s=@*Uw=sU5eiE>cLK2YL3PY zl(uF}>G{;=-m7`IweO>jVm^7Mr(0-fWAd@@7{W z6-EiJ)MNw~N627iXTd~E z>+TKH5fS3C0;p$NGUuaBVzW{2Rgd=t>9q_Y7Fm`rn<}Ypj2>kJCpYT)$(yRr>i^VoA^A{V591Y)m|=yi;PLYANJZmb4Ga(q>msJK@3|O>DWPfjrSueO>RpCd%F(SasOqoL! zZom5Jqj0xHYizyw6mGxBb}4HJ`O6!09~r@**5c_Gpu8LDebD9|jj>%;&Yfroe4kod zXHjdb+@O-qZDO{kX0ps$_Hn|NGJ5aqevvFN^!4V%bDETs}_lXUt%s_jat8p3^g&$gjn ztDD2p$7Y~jBH7FQj7|TGD`qJHV;4$-mIe&>KSk!H?5n?FVcRtH%H0fHeD}z6?vQ0y z_03Dkk%mC!t_O;~<}F{eIh)da1D6_n{cS%?Nvw>NK57K+#vSjTgcPtcxhP7T03h%F zx)}O`C>_h{hd^o^CZ71k?KQ9sJTp;PI0q%f^14t`j=Q<8@OL+nz5w3V%P_9N&%19O zYwp!Ajd|vM4^4MLikNnDn9z#2w_Y9H`~j)Yt#cLeVirk~1~`$y@H3CL5Iu0pA{Ohq z_4SK@Uu_{=^*klBCpmgPa`85oi!XqpNHt@7^eeA^)=WSHM&XW&%Q%+i^1Byd zCrT`+c(?1+(Y32E-omg21=hT&Iy?_4KfGeEQ@^o(MY;r`lMNWz^#X;9Ax%wBBZEj> z)B3S}H)?*P$S#p};70c^$K7(;ssLTx+pplXZnVYjvQ((cjaFF@qXz5^%C4%=_P%yU zoycGFBU;VQRN@35P5-U0!A~w;7gEP+RossdaMrwtw=MI`hi8*Tb1?rcRZ0D z-vi)Tago|14UOEQ05~;+b z8}G&!VVTOiq-P(&1BdeCyoR(z@FfxB)YKl{Ak(|eBTN-fD3?Sp$Fjobn|d%MM69?V zn;f?QjwE_&jt49YL~~2JCZ3>)#mGh~%u0W$ z>v^1kO8Gnsk5kK;Kcs&OxBExjN{jf?g%&1a0}i;tC&w+owvcz1=9d9{&v@_ z37#%2$AXSp{oc7G*He$XHu(0(o_0aAM%#L?!=ajeTjb%qK9=Q(V*(~x2jb)b$MAPD&1%Pq8 zSmnes*KWb?W1N_OXY|eb0D#Nv*whx*PFy%hE+fsFnpFHQTBh%QHaj{u$GcF-+0jTB zCyN1fpjOncBq7GHQH?TvUUD`U`iTc$KEcwDWR&jtOf&C*q z*}&c$yhaP0vrE=cQGub`VwVcaLT~}(58jULtUj95v4^yCI?{JCtLtuXrX)jM>rp5! z{MxDwbKhovqsoPz`22Br1_uVQ+MdnO4aVRYp#-qu_5m>vKwnRA95Z%Lxw0$+psg(P!zH(x(zZ&y zyr(K4QRESO-L_ubbiu%I^=Zp4#7A0x@BRW@$2_Utrf^vyO(>R|>#(vDXsJ(^ZA{q| zQ%;2(lGl)!YypHd!gW;V5>^ZI7v9jc)M&b{lKz3yKy;^@03-6Sx~!C5!t?0aR8uk< z&DE3C99_@DJ2U2BTl{fG^46W|iF<75{@&EZ4sA1-05+Fy-1qXd!-4|c`YHkTm-#tb zzOpyyOWCMy?JnrCOt*K0*UfnmB(dJApKo!awr4-;fl6H;&~)w|PvH z)!4Ya4d17SPqr#f@m{^`y=^V!QK!&vx+6vq1p5(7KGf$rR)yP!NuR)QwI}x>LrX4Z zRJ}{8xM->{Ie;OnRzy1O+zO@>b1#CgdIkwcn*s)daWb& znLLcL##iUY-wN#vmfDSoq>UVD_8c0H40_5AV2u8-m`;VsbA`Ig!lIz?e!MuZgvqy^ z*@odRh_!w9yY{u2{VN^uRXXCMx!fuCd#r$lBTNyKmvJ_C@Xh4x=3TK|>9@nHZlLRF znbQb{2KQQ4O=S60hxd3rp@<)g$I&aK-Pfgs&QhePNYCYz$AP_-k;*l&zW{< zD#KDySCpnwNDNiKMAyH>RzE+Xvhwb!qQL8@Sb*rmpiSBDz^LSgB70t&;JxEHL-901 z?9J*~& z7E2b_*!@YQ4d)bmpOKE!x~}l9N2Iz<9PGBQ>K8lV%M_ zh(l1hW76`Gs7ctIOI$jynToT_c8~Rnu3P6MEjl_Y_3hrn40qyV^Xrj9;)x5H?+b)K zdMr44752SYvh#+|EfFdtEnM%1o5hj8wC5WkGR6NVTth!%D5z=XJj@*Z4V0(Sm->R1 z^`5GX`5mUG8XiioXR2EhUha(Ur-7V_YgV;JY8Qby?Vsj_?;A4Ew>Z10`5=KmPFmZt z#iv&wi#JXw6VEbfQ=NpSmd>`pvoAF833le#igih?nqN23tw}d2n=}zM$8ez6nToX) z;f%vHiOp=HV=crk9K{K!dxMACL$aft-nzaP5uB42Q!csSR@4<@OVol0# z5-7VVmu84#^9X`+rD%EPSzOPA@<+#!zcXx1E(*tSl}2bAUmfBdxP?r)=$@K;*Sa?7 z6)e@JYyO~FRig`lvq<8EA5M#+Vxfh4nd5VU69F#rz~~uK29)h zs3Vf8-kd=-&FYSO_M%WW-)#p^4+QrJk>>tU#z19`` zk;Na6?7AP3&BwrD32FJTJk<7=q>Sg- zu%%s`s7i=?%*hqs2^*eWcR+ zXwI9x#xtGjIR6uw+c|5?n#Dys6UuZHhl52;vH#YE9Ne?r(a-d9Eov_5oIACLR5Eh< zP2WP$3@H}v4756*fglN9Kgb8C(+~!oL&Q2q4p>>%c*3df*&}7qMLh*#ht*!+MgxNY_0~habXxJ6R6hXX$8&P zKL>1eAEdIJ4XsTX@2u^=UN)2~B%i6J)_Dj@-Jwl>U~6evErW?s8pmuT3>2l6PR? zLoUgj@}Ti-1)mwb?;nJQdrfT)|V3QYf~0CeT4E(I3|^|jEtCjr8%4p2Vy!s!rKplQWim2@1cDC_JTc-Sus;_ z{QB(2-e=neYZ&%J`fPS0BoE zt=I?LCPF%m)ytTnQuzXz^V?TZDNZ8nw_aa&tfK}*7CD!RamG8B%`IXB7j2%SFxb#o zC;2?t8~g_ruCqF3X@5n1uuSD|7Dw*z44amG)Csyq!^v@vJWP6;jV1FvcJG~Ym-@~O zwvdXj3c9RBW9(nKY>^N0K?m}j!^R_PPsQT1%!2;GyUqu8!-t54Hgy*$t{3lq_uIoySw???6-w|C&lFgeSD*$*>hvmetdy#D-pF*HBC)*$-H zIMH*OYO2If72{I-y`3@bx`)yAr`uSR>8Ix8zgyX&$5;aJ9N1-w;njvm3P|`fq7*=Z*;0@An1rj_ve&ls;=ZL`Ub4gWXD<>$86K?mj} z*U0e3Uf@P}ZH`Y4ML>yW!{c5}C#_>KJG<)wtt7vdf+eK;PzPaIHHEtf^qsCAbkpdw z^IQ*&C5|N_Ld2m;Wy=eyw-ryn7MXkFGyCxS6(5Z*9c}L?#Z&CC!RkdQd{NOKlOqz! zch`Fq$fk|W8VA+BUf^rF?PbAL;JDK?U{*g-{~(U{B`m>&^25NlM(1At5nrFz*g&kT z%#RAt2w&og@Z6iB31=UONy1|vKrApBA*h5}$I=#9m3x<(bCi zhU=d(_%~yJ2;5c?>K7{R+CBbue7Nq%*nc1QKpmhHs;rI=HhbPCrMtQ9CDj;V(L&Yd$C-ge-7;+hGDYEq(@G5OQ*^RYAme( zc=Kz#nVtw-vGV-tGsG@~NSHtI1neexZISY3=UiL9s=wCTAn;AR!udYw)66<3I&lYH z1L;pZ2hH|2Or2n&=g({^i9rte3k-Tjn8w0B1=>gbX8`%@4l4$LR!rV%mYvkk;mj`9 z*Ie$*-SV{Fse3+^MD;H79~{oz^UM36+0;*B8RvEI<-kWVd_Q|)9n1j8pm%Y9`+S~U z3TbfqVEX+3>Of%XBHuDF!C)3-BK|p=i`J&|JSuQ(+7V`tyeeGZ;o@^QYSUrQH5LG{DH?kbQUpyM6)#{u-QFI>NHiqqFdKTX*mRVCFq(Fla4^f? z^u|#@w}gi3WaievLB{SKRdFgm_&tHek5-_Ik!}YThYUd^)Mb>OsNC%DlP*&+i$(S9 z=vz%ko5eH0JGZQSd3P=?j+lz?-s09j8jWk@Ps>fiDuMsxk!mWi(stYV8nirlLLA0F zpK<1z%Kss20q_!zz8JGk0_o902xqR#?r5)9Y&B%MHLT40_^81;h?LgZPkAf+q4ntg zDZJkujQrOcw4d@+e*5cIMLz|J5za-zfHf`M8aqQ|U6GCxLd|WwHT2p|;H=v|&Ijqg zm2mI*Gi236>lrnJ6QoMI6?m;W#HAkn&*KI!9PmAiXgc0+iW?eIC&0$C2OgxKhJn+! zaQ0q)(;iFh=8$&bGC1ofjLUBS;9#@{2W41?$l-v+hJ4HAx##G%SXlkND}lSezyC&9SXe+_`%e>6 z;G<1H*OTIarHaE+2Hf9%mpJ0VT^TqNwk$tsv&gp@HD@nfLz_t<cE(-WHl)DUL{uRbBm%d;V(P+>TGs572sORICJDr8sXed`;Ka^^d| z>_n31-dSVki2)4lPq7XiQv+Tdis6$B0G~h&wgl9Xyp`s7bu^dn+JD3!d@WjvhUo`A zXW9~+1^;;HabunU{G8&F`@-;4=~#UT!u=!V%iyl2H$JMH$Qwmg@i;u9A6*RM-#|_T zAjAZCu+&YbNA+jwYK1o44YFd@4o= z218a!^7}V(iR}6j;`DM*7C{A z0@!$qr_DzhRQt!%uA5KF@h`boxh|Xj){^;sv3(MB5qDv|LxyANMsKz?R+_k}l5a>N z(k|~`xZ2<&^kO!o_^>YXE05bqL^(TP1bW?(ieR2=y@)k)r6j0t5HD-XzSIFg^klPl zgo~%(_6b7|_XS?)ms&9$PR{W+?aaM|SF-i#uj(KBm;tZ@r6Oa{e*M(c$Iy-4{d(c8 zn_o11rM>uB+}Po1F~Vv3B9AtH(&E$ku$d$8nyAc+sP9q5P4%RfO)VR_NnikUSLYuE zzNzi#NU4mTy1UO2vb^1)OKk=`86RSSC8lhsor`ry{B;{ln*x?{lzDSC^0A!1AQq3u z=eHm0d-awSBlbI;PVF^{5Z~agyCyEu$8VLbQm@q0puw`E#yanb*cM(D92$PXhOb%< z=zjGE-y;^#lrcl zL+SY?npHtQP7aHkoXOx86KAtq*W>VL2mAS?;WPx6Y;ZQ+- zw=&2~H^}?GECEc*kPDvn?>PG_?D}5B7JtAw(+?(dh6mMY_M6 z#_#iL&IWsq2XuPjL;&$mW^p=_LqG{0b7jusbd>p5oCN>jqX0zIb~joFtWTfCe-8*; z4#7;14P$utpRMThU(+xGE(>AV{}Pw^?*T<&IRC^5)Bg`Yy_pnikP} zXY@bOj|^O+il?aj&)3QB+*t)ujWVDKjtwJ3m6YlMHU=!s{j~0*GDc1H-D7IZ?eaZN zgYWx9<>&2-cvaa7+@c3eY<&7b#fPHOIsF@AN+s28euRF0Sl&cA24c;D%f54;+put_frf*}SO72_#Yqn}UBV4nV;kTbrlSR2V5et&h{R&gi~Z~q&jix(V7CMJpGg|1Pi_wPSKN3HkX zyZuTH*7f@hS9}P5;q4E%El*v0Fg@qDzKpP3G;Kir`kR0(bRqA@PgNFc=E+>$JaTD} z#ewQs;(^`y7Kl^6rOn;$a?_;1ZpQc(irDy-xucY-G;(ah3pJ}|8ZT0_Xp}7~80i(4 zu~wXDhTNsmm#|b2N6hJ7zYhPM10{w12K zh)bK=;o2J4xZ595EMF8TiPYfRv5BW2oDZYo@Zgf*AOew47pv}_f&)$;f06BpVjh`X zPcH6Roa_lQ?}W`)lxtk;nnS-A0NDKSWf85OS+S>L#bANxa1$-JP^TUjJYOTehw(~M zBft7k0S-BBqo|`TQy`kEk@Nm7HM+dv%kOhDk!l^k6gWj8m?hF0qjP$7(J^3oE+VC#UvrwLyN&T zX0J3UVOD7CuH5*tIZ8aNYMTc!V7)ig0D-E*Z3HMLZIzKi;nJH)3Y={2J$#$fjPZfp z4Cg0$dq@V#*PI46Ic#13f%Ten-h*9h%%XN)Lsg_eM(n8uetenC>U3v|2xcK0 zQpv2*pyxle8Y4M?U+M041m(#-bxSqfX7DOEYG>P3Ea?@5H)k7*eiOJb$RRTv{I-Zc zFSai}=lRxWB)cwOpE;8eBd|GgjeDq~Ypj<@tDB4J_--$$`JH>p!($dGxcwbo>n4?< z=IAPZ*?Cgd(dmXoV-{|ncQfo`LHbSuCAExbo@wGx8=)mQSO{- z)LcQ(Y-a}_k6~Z^=XEQjgT9LX3q5T}K*86Tl3IJP3Rgq?^7gR|(f%ddVcs{0{)9Fa z;OMBKQk~RsvJO5-8GKhl%MwY$8uaeNZ3}pW^lWogUEeQ`bro{b$+NWJyZKu9eNIq>%p?cNMjd3#2Cw@5a_wCs_i1&NhCAxsuM58T%BkS> zpxqX5j2e(y3As*b=lcDWbBoxc#x6Jaml9@H_T2vvPmfAYuq_>0Yzvhkk{+co)8l{{ zSgb<1d3z_gYueda%z4&EE*+*v_o$KPvlkC*Z6~1-o24%Y{@ZGGqu<_FLB+~ zRZB6icX4KDbD~KOokvcy;FGIIMQHy7Hgq2^8wa*zY1K`ol#OAJ{@LKbr#rdLU(&wB zY}gy0Qu#ZBb+M1rIZT@b`z3Tr(`0N`*G$=Zku75OHs$kAaIL%PSUVe%4_%1fEj8nYVAes!Z;wiKh?5c1T zs*6U!YfGawdS!ZpOqAA^g^D@016g&f+#_HO&8mklq)aym(A)m<-GXR~x>S!_h1YL} z-$N4BZQ>>EHS7ZoOn129wvI*oz0uGIvoDUm5(zy*xJ?%h*H3Oeg^%gO%Y145ThTsCX)}kXD91lLEn~au0`^HwhO>wnc z9EI_7Dc`KX!m85z#7^dW4U_3erE74Q4ow*HKSqw1M_bH4E1f*lIw(#YRpVIL(bFsON#D|D`S&&42JVe%+sPP)~@LMxkh>iB@Xk(Sl zLHR;Wb<+cEn4iFDQA;0Dkf+J2JxsAYSxkcsq8a>Ykuth6M<(Z#Y$m<8{G>sHE`rY^ zsmXTWA*X02{q!DEC&uliF4}b-S<$R8;5@MZdce*>1`o=>D=mERt3AcGlav`qEhyNL9>|c;EGNpr zzc#@Av5c>o<(O#|$5EV6i7=V@%aG>g6R!jOQ z%W=+<_I&Nz%FHv}=g8yRm^5*Td*fnJR1FB4`dq_*M0;US+Zy2hbGs)Fp0%nX zHVy;z-_)ut_>dljO$A>xK=2*aQ==UFEYu&QejT4=j}iCEgpP@?9P7!$#OyBQ z#)D&nLQa?7n^=UL*eY703^hhCXX-9w^oP%DSMym5Ilw?-*lI;6iwIjv%aCl_V0CZ5 zDusfqY+XX=Sk_zdLlTix?tTnR&!?cbhiGQj&tffbUBw{JX5;Drr`J0n`4$4_ zpp7eKc#jXGU~$TWUrnaw5F)Gy+q|Z6+9k2sDiil14-inFk2wyMZ69SBhO~Qe6m%JB zo~~riVn>4ZG;^*?xc$j_-_oSZhzo%(T#m$gw6Nfa6L3St_KSd%FO3vm{Kh*7^0}Ei zbb63lwwfYd`Ahq4VzKN@Gm7}KbtN_(13$6&)EN#pr2=*rt?mab=@ha;Ec&EHf$ZDX znsHUOA111kW_7n@$FsMmL^nHboX#oFU`YXfR?LTb@r09{?jl0Zg?2PeG$GJ4As|;tzxTD7I98)O0-ZAsh)dcdryQSK9Q@thjs>-$G8av zEHx-0M}<1|x1btM?(qCX*_PQ$p+&i(u`+L3c540eR%^Gr5nhFtqb=%Jn7xIDak>Ss zoR{9Tw&JEHLjUN3(^&f&k~mG|xTGdqR-Gtn{7fo70F`Rh0vo)5zo2Qd9D=jO5=wd1 zZG+Z{{CS4oZVi-g)V|C!+bFS@*xDGO;c;R+Tvy|@s;XUZ-YH`=mSrs-O_(=Ym$+od#;zH znD3jM5?`l#j|FGZTbG%7;NUX;sC|k|(_CmrlEVif(yL-n-ZWBfmD@DY(8;R8Z&A@i zqf_FWrDSv7;P4+hVkp#Q{Bg= zic?i?k_ffNhw42d)|TZf-0$6dy|^CugX(hp+Lp6>%dUxxjTck3!Xi8Vbk#zwa?8$U zk@g}*MiC5=(`cfgXIJ2YAUrN(+o!goWUyT>t`CTXZ}V^7lo(YTZ&`|=m}DQQ-jefv z3ZF-bXW?BVujwcAk{iICJz7)vkF}5FOHq^7T1kA8kEh7l>8e<|$6j3(U5U$hOo?VI zh9IRj+2bofr!}<`Sqvgb4+Xi1&*m8JBxc#zhKzmhCgt6_l$}7+&5MZR+MMR&CZ6U~ z$QR!i0>*~Wfhx#H+~kJcOuyk}PfKg9=ErRiuRNq$agum(!}~!vkGlbjt1MG2d7n~x zfsKlR?H~ayC)~D(3+#r*P`yyf8>Ka{g_>hp)4_US^G^B}CG?)pf;PN=@zqx3!ul-v zfRM#Ro%hHi-P-NVX0lhvx^Qki0pzalUo8OU+bJS@nXk9-$l&x;i~Lk*oVu#uB_T6~ z@Bdbgq?^ThbgaNXOP=m}svf+hoL)a(G368@wRyga5W(s7%zO1&zV+dk2tRVv(@Qa9 z=wvf!gW-lw|Ad+xlI%I*3v7R7LS%)Aor`&ic*`o}UZVlZZFl!eUekjd3Pi`STnAB} z+6~wkUwlsxdYwaeZu6>2U%_VwRQ=xpV58(v{V_^<^DK-DrI|Q(vIi-;_(P@?_pKR_BDqEKQK)4!B+I0HLBOUdwLZDShbGXc6I}e`uoA*Dx_o6YQd=8AtGzNUE)C3^Q?CD zT@7282Y*<2zq)Rs_i<1S#~WL{ecC9Jj*(srAC_=C9jjR zjH8M@IA!Dln{sVFNUDJNO>r>K<*7$Xm(eWYE9Gzj5k@opM5>`VblXW^jJTXBUD+Tu zS^cHYutUDrhNcmU$_nH_Fj-sgKGwJ;6fbrLRLTi@_U>fMRoJVzqq~+6`$H>yckdaf z*HFk@>GONTq`*2jjo3{!UlSxSADwM}_toAmSLk3;_aQ)kZzt{V-KgA9TD2g~i zvHPbAiT~+l`Z80bx2eWV8RY#7;`(zHv|L^dL|9EY3nsk^9A5JD+~wr{?)c{z>CT{! zidcbX&z@Q_zlz)lf60PYCsvSTpBR9k=;(iBNB-xxC^~}P z))U6tkNzC-Pv`aG9YqZ@l$L~Q)t7Fz) z2FexMRX>TtCWqy{;f$k^CA-D`v+gb!pzvKFAmus|-3RUv@vGI=&S$AQ#n)@b; z>!kNOBan?@Tw;KPTXx@XT&_1nX7BaF!d=hBDXy`iL1?pK*8U6y2z%JO?L>NE@%zpy zBBlkCgY^~xLJ}-s*GjmdIhxu`1HHdpkkIi5%Y#skGwMym3I1 z1VIO0Q-mEw?rF5^&)3uq|IVG?)jpJI(^bf^F4e8ykwe=;+0&D+8ft6a^sDBNky+mE zsupsuDAE(YNg~{_T-IB-PrZZ(Q-%wfPN%Fl%*%Ouy+22OkMjAKuY7l6P|LhR#Z4%o z*d$PqdTSoWKQ{&aj*Ve|BGBDRxe|@iZt8Iq+OF{2D5S!_0y;lY6rzGWuBdbqoJ2n{ zCU;SnIO3PN+cD5LI8jx*r0BZctRYVlNzgB;s|!7^qnuvk)I+a_9s2b!ncz?0R20J6 zh|f!Pmy-K$nD``R06MhpzHg2URP>YhGm)Uv24~UV?zR3^oMRUczv2IoOmSVQI4i7L z&vx6=93JkD=Day|MC%L(E6a*BZ-pYY5?*kb(>{0l>ptA@?37*p(=>9jao6EOz2f|% zGYTpEy@Q{o<$1!iZH!)PpxHK)QC>&ajt_8cB_G7J!+IEL?oty`T%Gn4L@N4&Y>Ai6 zeZ`U#GUmo_`{}(z7}WL&3|VQL*z=K8lW6=>2hj)L^550BF95X9{h60|MtG(6615DmprWZ-m6^DdykJT!=R zC5UuKWmiy)vKzzosn9O{9-Vopvq3n+@FUjstA#9z6+gn08Ul`Jlt;svW~;(KZYSv# zXOEf5?>$PEC{fYa}G(nLU(*fkIjtu_0B!6!boE1K^D&WVvGzG0vwR=5= zuB)&kUXM>~6uBY_DTx=dceBK)*V(<3wEIe+o{K|uh=Sh4Y9>S#{i3k7>g)X3ghP0; zAH1@n*zlqE8QD%Yq=%fGU{Bw-^udyku}6F{BCN1IXnBu6x${9ubv->^x$z(c&*&fv zYp0>rU%`%xEFeslR)_ojH%vw}i5~9wBE<#jAt;jjChzo=5^~KJ9GpMF1#DfL4Ny?j z=Pwpwow;)GQpd8uZ~2xG-rOD%L1P0?15L5b7r`jv8FoE>`%LZ7?Q%ckTYvVy^kA^# zs{QDOo$6l&5M8f3Zk$~S2*ES4n_^+<7E{6X$`zSE!-u(NcPZ=~xNDzn6?jati3@}B zm*2c;HinCs*uQGnmQUDPvQ7-|dl|ihrVz(H6BWg!nNIg=v6ZnMa4Cc5; zG>ZRnZV{!si$yVYY&=xP94{jsC<^wcLqFy_23V>NG^=AwuFrE@*sTO%Jl=JZQUVu$d~>f(hJSju(OB9)HC3{U?R|gz;XqG)QtTl~8R@1UVOAEk z%hs&(*yz;Yk3KjYKx~=B``)EHC|h-wD`;XR8-iN6YB;9(6h%fnC^R zgTA`G!gUUz@DiWWO(^xDzlL0QmRqZhmma*BZ9A@#?7S>5x+*Tkn|awsjzvx-dlz$# zX;wksE7%Z-8PM_XGe@Bm4F{V}G9K^Zfq%{`37l|9Fj=`#$%% z&$-UEzOU<=G!+qvU3(upWt5!GVUkyL%F35_F;u%&+IgDxLG`9x_ei5SJ&$Qp8PIny z@B95+5`>C=ixg*u>d)xj&sYwi4xm|GvI{L3)elSD`mI&)RlEI8(|qveBG5wWUsb+) z&{WxzGN@M9Pqb3KShP_DGCc2qu#RCOk7=Q1_pKwawtP~~YUGSE-RpzMzuaR8iA-Oy z`5*Mdly}YQh$sZYtGqDn;$7d(=mQT7y{eh@X+w~cV75CKj zkV?Rx3no0&zxec4vF_Gq22Iyr(c)oPd@BQ1s8aG zAW#o9jKypdPTJW#eiIjFJC}9g(GdD{dUj^t(cd>3UgcgedbrfvK`v$3pW*|wH3x0y zstXUBt=2Vj$xfh>&|zb3JT^qF6vTs60bGYcc{p&dB!4r#*>>n0s?kLFOv{ta7$q^@ z3@l_6zov*tw}3vX^oQih&wj{Q=+v@D%@{;-oA7N!k+J9F1+oSnn3OoPR?@z5kr_VA z{R}oZyl85)>>#ZteSv(yU<3Vu`m4M-%MF~z&7@=dhqd7L+Y2W-^R&yaHmGM06{I|f zQh8_0-1ft?+jK4Kx9W4>-#r9*xI;1uK6OA^Ln!r(Yrg5y3ZPxcxTq`*q>? z!x$}7G@?}|D{HjB_=$Y`^s4E7sV&`+6Yb04$Cfiv#^~}K&2Q8ZUzI@=#|BV7x2cm4 zWcsxLn-Wx#(3009?cocb^ol54W4(ByP=h;%kJmvk@SDSe2)y&s3S|IyF+ZWke3C9u zX!NmvX0r{LIJ39Nw@0EhfpvtlhJP%T*qGU$kt8Fq-0--3cA{uCgNHyTIk7sf;MN19 z%C%RqT}mvd9ZR=8zo@jbUpqfMY%%5(ZJj+ERcvp^o1|NXyGmtJ9-h^GrH!+4cn4IA z$|J2Wowu~YiVM-I-RG+haeR0CV+1G!=+;jg$8u`Uo!|M{Ks{$;P0L)YzB1fnwFUIQPM$7KZqf?9Tvv!};6=6=MuM7{(N-{d?W zB;+8%hQ>n1qAa|l?IbXP--VeW4jkixA_v@l=MClo@*pt%T=k)-$ELbD$QVYFI>@$7 zNux(uXZnvhTsxcyPW-+k1{~{>(GetzTq|UPImUCfkeZ#BdTYu%GzUGOeF@zmAiCW4 zt(e`0ZO8u+M$+Po$^fj zx`^T76LvoRWCb{nKgE0jCk~cu-xfb|WfBV*QOQOC^RG>~b{#VZJexurT=oCxSTKEv z*G#Ed`{};r+S*hAe@3t!e^E?c(a)FtL{xK6)zLwKgj+YL^x?)Ieu9#`fF_XCziZuF zb|C41X7<}B{^8*@JD*0*d7Q#x0H6zvCc=_%#YXn{`Z+PQnZo27(`AN(;H=eo4B-+Cfs+bTT$v75GN0L zVjK4^Z{}zw-gC?y8Q>QqS_?)>|Im0wEAQsy>& zy+=Orc5lg!0h@?V=U>6iI9_9aym=*(nwqHN72fO{n*1NgGrL;~0Te<-Ty1z{+B??N z)4b0!3~$~>N)lrW@MgUpxkD*S?wIEozal}B(Z)-;$8C1SaevnirF=dy&!{Cu z_0FV@{y~^RjL;p!?saJyjMtw(&2%nt)ZMl+%{Qln{5eB>QVoM=Y%b^$AAfi6c@!#u zrY(wZ1RR0f;+6pR1M(Lp`FA>Y?r>rZr*)W0PnY|%?VFna#-w)tMHGNNvLVr>Gxc=pLGoHNpSDOB}5&b}~8!jT@4-U+U$k7PWU8A(FV58e5Wi@` zp8v}~`O{<7s6o4qu*|gFv2df5nN2f_*gl(vOPfx#ou<#;AHl(pp z#tJJh9wO`X^UP?Fx+#}kuZ7o-Y;|?@hMs{fTdR7m>W`bt0|WF?(^1v*j`bT&+Z(?2 z^WPa8=11xpK5{*-|6O8nkd9$@0D~t0O<4{O7saa-H=1YZ zC2K}jtq=G@g0)?`p7zHvD<*Uj_T6Z_S;-rIQfcWWDRnntHtYbZu4y#d5N5zL7D(gS<~P+6W>#XjwO+d2(`!E#F&e0%#nsTzkdwLQcI;%vDjyu4 zFG?hs@?#~Z$25zq=?ygpZ{bWq3jb6_Wj&G597?2mG)u&%GoJ@N>C)@8(7^jljn1;> zl3}p?vDM#+QKr_jJWU;d;Ea}uoLyGyv_lssJ9<2A&);6eD(W|e^N)_l`tpK>Oby}G zz4I_K73OyrMaogs9PYZcYIx6eCVy;deHm5#6%1n}oFex29M|}{9mmoxExsVd#urhRgb5y!X7=@F zAiPrBX0<*uO)fd{T@&1Pnp966Zo>nq@*1?~YJ&_OKaS0xNKTnbU-ejTD@zf32sOg` zt>Kd3%KNtF?Ml}*7qZix8?^06Cj!xgx|+i33QhRXa7ts6gf|=|o#Bo%`L?a#ZJv5) zle94XzEu#}GMQd#s^1i8;>gfRc7A&*yF-h&3~n`A|5otZZBWx7C-`e9{2AeN57@(o z!v00}SW70%n1_S@FaOVQbm0Q!RZ?Cwhk8!^6eu1t`YE@cyERdkPxkho7t?`5?^HyK z<`oq@av-myL@q)xLaX0pL#bGmJDtm2ZMzK_7v?I-!z_lsLHQUh^Rcs=(V(lyW^}De z!@61C5TVu2*NimqdrURfqag;N7{$cUu_)L6jX@$|PLATg{+~)9!nb53qtw9;a?^z$lxuF0?U3-UBOkkjL<0hrBMD=1yK*4jJQpOE1_>_}u2LgAp^Vt*uS( zua+`w1?_Rt(koO&+qsQ!eSK}Mel*O?wDH#D^{ghL^-jxVyCBG#6iKQ9VbR8Y)f#nl z_-uovmNdmkiB|tg_@1;o4>X0qQ(@8va$oWclg$;WmW6ce=0@+BkO6z?hNa`GUCBbk zXz{Fbx(VpEJq1pLA!WGyprlW((VPu))_dh*VxC`t^d#*}OflacT}b?sx$hS5(QAH`ZuV%{aZYx3_oq_S5=eS6tK8#W^I! z8X%dhs7q-jpeScny7OWXw%LW`CTnwAT$85|M`5qAqkXAA`trc%7U} zjkyUKaWV^9c-X?cyy$HuUbCBJifSP~v0;|Rv-*Mcs6s=jR<^0&`i3os(-maUPW3WR zoU#U8?#`11CdMKB1=xE=+J>bB13(8NP$_4CP=M#+8-LGG#aWb20VsxIEGN|is>Wz{ ziVgWX{koE)(r)IuuI}3TDf?LmO~d!oL6kKLat}}(i)k&r98wdKah?VYaN-n{nkY$f z_1ARQ&1bIXIN)@X4&~P6Q0f&CX1GjBrdly4cKv6UK@?njwN}O5YwSxZp>>-q>J}V5 zSunwtaA}d}SI4lgu&JXW(~XgV*4D=hn?Q!hT5bHP?irD0&;Pmyf&aTn!S6rxKf!D^|m&3d=(6 z1&}}N%yV&Nh0+rj)-rva>*<~kqV$2Z&n>sj^*yW%gNg>{h|J{csY9kn{8g$9{?W*V z5#CSu!Qnh66X}~V=r!u0F!ya_xN-X{qtMVOop~QKr%FO+p4&nG4irz%6}P3R^bkI>D;iZLkzSuEG;7zKxD*$sTZom5 zU^7$|Sez%}rJuZMm*)OBMDX$#ZL$vTH6Zz;+rkxc2>;ffxk1|u4-RZ;S2r_pbi#;N zRlFpf7VUwbhyuwQi^h@*u6oj5^bSc8>QXs0XZ>CvPD9;iQPmpt@^eVaO@@po zMmi*SSv3jv{XPQX`Naeb263j7M^fxy>=2q;rd^-)MoYIQOQ)PzL4ten57FX~leflU zv4h_XgxA$vO@lP*BG^<%8rvwGCh+5QH>}y9;xQ1pnIeryIAb8`{L`l*f-wpinGXf8 zY4E!*7~YGBsw)}dMZ1DTf4KH~HvoEAb|}gw4_$>8Aywd!Agd|HZPd?FvixVX|7b|< z{ZKoWkAGLAFY#46ydv~%$VtTukS19o`rr`BGWRL^SbF{_jX-LG5RuBKhk9ODm1Nl# zA!MXXL<^4&Jx%}M)ncdUB#bzVf#T-l@+19Q7Z8jcLSLpiTvy-7bHqNn z+E5xZG^d=ZI%JSYe`i87qf0-m%O1lsCVh#yQ}xr(8V^35=&N2sK4#@ zfOM7jAyFMt;+TVhL1M%htLn+cza+FW*Pm;2eooQWAGYuOV=Rs%KFz-!rCV)hrBU6W z;=!2GSDZY3N}sRO!iiqHjhv0Y<1Zwj+3mdC*B7&$s>?6DoZr;t1sSI)m|ZU2E{@2O zs>B#+nXVHvT3G!+nL;l2(BRt?wvrm?uD{ z4$gRRh?(=G`aMiDD2QMNP-SyT)$M?9ns{YdhCEA4xpPdB#GH9NM&}*P^R}AsXQ5n= zJ*UqnXq`y5Q&nhYkK*Yiw6-_=SgF%Or#0~!?(lbfa5Owv^O!@RIZ8g{O5lj(Oz$9jt^az6t$np z8-BNyS^m<8mXA=SHR$5=LHxor`xy7xPtqL4*xTz3O0qR31#Qbn!(g)HNin5eR*)1N9FYoDE*Y*iG&U@+gOz#&`GfGc~&M{WFK z938&6oww?`J-fZ?m4pJly?@(o0%d9`*MR*?cDA0Ww?>mOqRBRcrdf$zkS91m+Gt!k zHm5~gX#SL=?3e4}(7!PXs>jyA@iZ8lZ^_TnZ}6^lcP9P{bgmKwPDzo7`Z)ek>=j9J z=o^i-#dKE_ocW+a#iT$X*~3GC=^r53@N_=eXI@EWds8Sl*s-0{jUAnqzudcF-<{s{ zaB-q#Of$!laI7YVI`+)S!4au>pfjkhWm;`Tc~qh!9`)fh-~p66tLo-bDE5xYH)Tj({u~Ao9bi$w2UfoE8OtxaY|eQJ8~@!x`iesx;9e1<`id!Id$E3K>O|1E8YU-Ei?*Kn{IWmA}c>`tyBx=^xW9~;V3>UeIW%e$8!R^J~!w= z&a0`Zxir@*D=aw8)1fF``O;-v^dk40vuvMdpN+)xFS&Qz8gx6>(x`V>WCQjPn2uk_ z9goI~1Iu`Oj357%TiaSLHA^d+mnvoSf!rl=db+|K{|FvrM9&OW3&0j4$O`)U`fQVK zH9U_0)@O`=$??v+?WEjbnwUVL?bRkgd!u}#z@*6X`4-wb0#v0^wH)J+zHK0JTZIZ; zDJiW4S#gs2+~PaGDxmT`rH!$1%F*!^5bm*xCL7n@oKW!@^;cM1ZE`iecK@SJFpkr% zE=W61C`NrA$r9k+k{{0XtS|9S5M>YcTExl;oTTm>DA7}2n7M-Q6 zxrI}+b3k#H#zyDFv^8>L0TjZ>2C%DS6CP3oUVl1v`)@iKqxBT9c6( zEdKf++nulNX~hL~$GIT{t_M3fFh9=%bx^`GA6JKhLh0RvVFBBJsxO)$$8;3zv4g2` zs|Gx>furs_j*47XrV$5Wf+kuc{sBU3=om2dqq>ZNEq7Od+^5i!=9??9D!P@Ro!E~N zY||N)Cv%8iJNVbAR8kt9gpQ=;Jlyq(|hmNc0VbO6uc= zDQ_w^_d-WI(lnn5KekToPnoDXKgAyyLNs}(C3XS5QO{2Is0yD+C5IL(e)Tu?o6;gv zvj#JX1hb9}aHsTmYAmK5PY4P+TR0KvvHm61Q*|UUz`mX#SmP0U(cB$jrWWAU@sRMM zvwW358v1O5Ua+f?Y?MkPhni#~cJRjozP={`iPAI`vc=InPl87S<#oi0nYmkm^s|XY z_xRvE$LMdiWRgK4l$=%zEgLg-r8h{= zUaz_R^<>j?P}XOyIK|ox@{TdW+P`ni(~!>kGA)IKpWCQqi$QI&Qc8Ub=rF)(&rvkz zEYSpTFLelDpcpel+8<$)-}Y6!mIcP7bdCU7!OuGb|=j{L`mT_eyFJ zwrjHjYFoqfHnEeUDB0lNnP#PrRLpelfY8KT(GpHnZH4pG6vzs41!y__xhFvVj^=lt+)H~sx?vCL&HjtY z6M-dLawCsjEeI&_TIuX3^j6mgwo7h?_)AMdYghZPk7d75o|z2%EB2m$S1wdi)@)^= zb@gkH1~z_OHXq+w4+TL+dh4Vd!;6Mb@u9{zf6`yny<2yRk8wLbqGF!>VodqOs?sTb zx<>b2rD=Y2uI0_!iK5upJCzI*@@dHWL6!X4dK;jhqplJ^S)YlvTenLP%))~X#HTk=TyHP<<;j&7%B>lT0twIfSh4Dpnu5(OnhRXKm z)6?31%J@7I%28bbEs9-dOICj8{SL^U(Dr4{`LHkYZP9~7o*7FOEIwpuPI(t8i-y-GhN*!03{d*KaiMSRyy#R`m<29 z-_)2`qf8N3#?(n}5hG*4+NCQ*5s3_Ee1`h!0#+?`g<@%owaVa_NJdMqsO9dtfZ-x2 z`BD~i2Py~RCqWENRs~JjX%$J5w+MG!l>+bP7O`S&QCSvqacnk#u2TfIQk?Td#h6v~G}xt5Wl9gnhI2PyHrBF$o`se-nKH2>rn2;vPvaG|9k zM(XF+qR5~X7Bmn&N~UE4(ay0u=TqwAcQ+2CH9-{N7GapZm)}7os~&s64-u~_5L1m+ z&Yb7CACyIr^Z<1e`eM~Msi6aw{Vox~FHFnwXIXeERCgfjr+x5#M6n7)x|D89TONk` zUqvI3v9hY7@+VS6-cv{vA+YPy8#jKoK9*{h*L$$xgN6`)y$_q%RQ7K2%$U6Ato`cz z`Kn#i8te~n+n;tN#q5V^U~&-BEl46Hv>Lu|DxD?0prxusG^aL5FXw=AYwXXv1 z!rq~DglMHKQ@&H8iU@wLbor@3Z=Tzu#QC!zmdAIV@i2@OAPSqJ##R;Oq*;{fsZidK zo(E-pu0!N!vlLFW1;!KGoBYhW#KC@246TEba^igft*CI2JFCGjvzH`a65`!_Rg3+ z{u0Z&5S}snAihXn|NLG)AWR0l^-pU)CenQhl#7LoJpK6E2+wz|%GRg^NonT-@XfxH6R`TO+6#g^$Wu&V<~tIQPxB$rI&R1qv@wzG?-ex9n%>V7`|`Vk8L^ zOtwCgiG$enFJ_8%ImIcMjA~g@XUFS_^QYPoL5%0UuDBr(=+lCiA;C@d@0&csF*hkt zF~rD;tV>g7&S1m) zJ3hWi*-lIs+^^91wuz9j{01bRItdHba$qHC_EHta@j}~9ayM^_2lKl_!5VQZ>14-5 zyT1Z*x~L4#&|Zrtsqf>PA9=w~umKv4R?`=h z^+D)%v>cZw(^tFk1d!^9)2|#Z-YecF)I)wF1GEoZ&26a&24Pz7P3IVWv&&SwNug+B z9O=7BoZTmV1e_GP)=@;_MkPRewoP5MBN^LE-L`Qn#x(m1dRZ3h$UCvAVi=$F`A?VJ z@50K(5A1T^n+S_tD}{okHU$$EIv%V|zMS9Tu*h66?}?I9?8$@m4nbtP+NXL(XMBs! zus8Bl%wJW}qIW1r6E|lG@VY?!f@IwJ%%Y3WGc%0N%N1i{ql~u*3j&tSE($6-lJyz+ z@KD+_OU@W<2b=QhcThGCCoUC0A2pVjEcORtN$Dy@yZPz%nj7KC!C5~ZE_xTxmt%%{ zB^y8810`f4V;!zmRlRNU%g+;!rKUtr#c4i|YGq%i@M z!)BQgtHw+@Osj37h+bjZ7?+O<^^R0$KYCY^yLzk10#1Tk0N6!TK|t_4KkD)R=%B@s zEZQ)!OO5qYzG{s7Ej3x1N3k^K@78Jzvgq4(fNfQsNzXjt^vtepH;iUN9tEIo1Ltj- z+$?QdV!&*V<#RG6028aVpw1I(A@Fn-ZGbUa?84Ca*?xVUL94eb1mb#;hc4#L1 z5)kd(c}>{U`@4KD2$AU_EC2F!fiiNCiKR(niEOdXP*dGUI{vyS`^c|z1V3SZlk3qm z?)gAQ4^fGq_rFY$+vF)lK1|)-BQQ}Yqc8&>+2w#bIGlrLY<2ztefp5TfajeEW=Jhh zKz1nF+cap<{8(h2OY%$`YL8er^(xDaCSsAk`3g(^vSOzr|3q)zw3W^IWDO_Y6AWLs zjLyEA#PJ%>Xh>sxiAvjCm#Wc~`JJpzAi0)I+aZ7j6D=V+ zd7@1x&XavH@-jjxSwN)3g0gv1vphJu--F%n26mn4oy`f>GvFk18;h59mcF#uFn(1z zFn%)t)xoqos0F3YJIu{~s@fL^XITzr{O*N9{f+nuFSupC*sQ`C3>WiMaC=CHGVBxG z0`iGgunj@&r837$n&*81K*C4`HZ%J&W0q}XBkY1Q;b?Gj-!d6|=M~05b{S73khQe(H* zK$TSAfh`Dc^Lr>^o{bb_9G(J0htHnWV@*Ju25O4~OJG)R0O%MGt(iY=O1&WDzt_al z0+flON-Pvn+gBS==ScIkB@NDI28d3+)Kt`CvJ-tC?bFPX@^-YbOXbbhhSJ8PZp%4Y zD1keQk&WYd(p-K|ZC8ytesVbbcLPzG`*Z<#D5F*5R}Qj43pZG=-xSkf!8arH-YHWd zhm%?ijHCN|?S@zcBL8y$w*;k^mSVZUxwb*7+wb%UlGP9>CHUNFoeOz&fP_>tkcB%u zV?8aVP$4ZpBF05kF^wBLlj0Y;!}P9-xbY|Wn+G!i9=t0JDbApzc2l$?i@dD2A{%Z6BBjRY6A#RyG?{^*z<(8j(w`96g8 zgTQrTcdOhe0|I|`bjNMMIM0$d;B`RYq>TZ0_L54gN$7)F_14qkp4GkCC(du20Td7D5VcrjMrNN}_!U_&J?LCSlY`Cm$hf?6r z_GVF0>r+}!pU9uW?-;xfOXk@Cf(#UXlIEaT1`M1a-kOn;&T4kCUZ8SAA>?xE&IVwU zfz)|eynx*)yNKhFOmeWP;4-5xEP74x8I#F0@g=q>g9nUFTeTa9BRg)qXn^6z-}nvG??=kM={S}b;$`}531)tB86 zMFcCxuR1o-Jm0B^CnMr$0tF}nrUkq|u-U2g+Su>Hl40*4ix$@@^^T~`=?LUwBc#WD zE~B_IMO8oXjs$82u7K}*(|JOCmux7J=IaO%s|B+-_Y6z?4l4x-nxGn*c`)XHFy)@0 zGPE~?u5eP?nkaD@(I;O%YAL0uq<;1If@`b0TjX~a3?o_eEa1Qe z1?nrSrZZ5bV9PO}zC(cv@=D;+hrKflTZOD~NvSHxCHEEw#FV%qFH4-&ue9A|P(Oh& z#d6XP#eJYJAEzWHmAMD6r%VZ5NH4l=Eg{S|fXqz0C$xW*IPe$md#_!(pYNd{f6IbJ z8p+?an@9^%kwH9NG3?nvJRPV3OOx7g%6ss$RBvEYYqi?tnU(?3sy3JQlhJ|KH_1N! z?{J_cqw}oJxl+p%tDO?ksH-^0*L+{>)cyfOKmp-nS;f)79|Mz&jGCCU%6H~tCwF%YO#)`7H1r_;0G@=%C#axNuWfrKRXKbY zj=0w#EYE%=Z67RLb?VhxVcP7~z;nDD$+<{|T17fT%Uz=f{RId^sY&UXgLx^DA`3bK z%wtExTDFqJu21g+{SX^a&oxK-YM5Ln`SYLY5B4&9m_C8C(X!@$P%%|*5uz&?gqup4 zJee>{wVaQdlIsVM(6IA_<7b-(EH(wnr2{{zNa;oAB0Dea>oh+gHq0FNE>;}PiwHQk zIf{pOaYOR12r>o=4ot)%9`sXfn<@|X$@s{1Y2_2Vm*s{bzJg01`9a{~Q^oj?*oHP0 zZIg%U>Q)N;(_dO~2&$<}5Drh{2QE>oTO7e2xMmq2K^f)S!J^Y#18eQM^(Z3xj|+e3 z02i-3b8ceRG2U}d@ly;$u}}IeS^CfLy&`c`q&&(NeMyIegGWjGKLF50{O^$mML4O7hxWWm6nK#l6~tXsg@VI>rF>=Z)hImF_m6h^wXDc~;jQOHT@a<8 zf{r(kI{@6gj1HI)QwSHaa7yh{)xGi z-sPK0U({vAh<`E6;V?~aY55u>7vedbzZ>`FSv0TGRHPRd_butHt#yJzpWu&Ss>+%i}D=6H(!i+;oQNbViQx+ge))~ z;U2uT`?e=QavIhg%D<~5;WXef>gMzpl@2VwJE}+s18k~x<45ZtIy6*kuCe(&m9hDMklrrL~jhz##zCKO~|AFMn*PDUApp;|CgVr@!iu6RrCJ+6KZXz%mLa7(Ti(-H~`$nuq^)wr4@iGg@6iZ6~>(z<>3Ie zsJ}Wtl8~0xD10rC6MzX6wJ&b1H;_9G9vkhoY9foQ;0BhoC_4DnrfkVD63;JFw zt?vLbu;KRh#)8K?3tD>D4T}W;R`e7KVpAVIiHl=;UK(_eBwLJtv5PP(!LKvLJIfC^ zeu>&ws^_1yxH5k~A z#peY>p0xGd+0Pn63?dEdx^NwjK$K?$IVw(X|COH4amo0GZpDk#F*?_cKchH!WZyCl zr1!fu(LiuUT6@jk(A2cCK$aHp{my1*;L1Yv5@^U7CB+W^wYxdt=&^3fg>>%R$NC-v zB98F7>Y=v76yrt{fL)0b*H!w9O4t?)%`ddvI>d3DZ)6670xjR{|p?9S}wx zbth%#Gzhowl%0FY;Z#0VFqyXH(CO|uqMrHaW#5If#x0E?lDDGvG!jGZGUu`7U_Jzb zAlz(hY~>F?1vkVg=-=gmO=q84UQ-c_vG5m*5fwGb-IEUUivmvgk+G^Dd;bH@W)F6_ z9cfu}8yW_9(5fF?emKNI_8wen}mm`Cs2h0q9`KZ*1HR{;Yka z+6N#DLC!Yu3~A&CdpZ!BlB2=zkTyy}2RLX<&RT`LP@Z1A0FxQH-Vkv`Wlz_@b;06E z`*q+IgAE=}LP`2&Aioa=)Z%P=8Gr}7#FnhflIj8~On{Z9p`ob2trJ$QU;CtyDX$mJ~7y)A;&V|C_1-d z31agtxC4wY!(Uqs-<4$&I8{5YpMS3~E3jy<-i0L__|srwXq;>1t2AhXkTMRkfy3|2 zeI8PE8l`-y2e>G+9DoWAKh8tl2EyTnzODeXd7mY))?o%j8z;;LK)b*VJy8D_Z4Fry z5ogC|4n*KbCe4z@v+6x>6y0QciALsK*b}DbKrJp%4teUZz}dMRpc{XnN%LHzTFfBj zyh}(Tb1#tV`7QipG+`YvQcGvRdEHS@2*6ZFRRI144KX~KjDvpIIw~rD=z5l3Oo%Iw z3!JHLlMI1%m&exR|a@T_d* zNQS3sF?^^^;!b*-&y(=`oZ#O#hBQem_G5XrpGd=P+SNt@U@t{2-!h(mEZpiofXVO! zoJajxbz|MprYJM^!YS@7qc)a0>!Fv_YognFR_m1)w{)Uq*7I!c(PKo;{ z>5?gv$c%J9!vs*-h$LvZE3y#bCoA-m&5P+gMS-ImRtj*Gqu?C#(efH|k|2M-4=LFg z;RplB3_dHX?ll9Ug^(|c#;ky!UO-`tIgL})K)HH_H5qgnl1>pj;OeemQS>-qM|i_t z$yJudS`BB z@Uws(=q9*>fCnrm4wI5%6*j$q!?pupiZL#w;LEfrgSv~UcFvfSZ&MIa4+ZF+x7pY3 zD`goWE@jg`pyW*b%1Vv}sMA6Nq{&0{ibQ+oi9v4c!O1XZ+fD_|_fAI~Xb;QaPhMX7 z`lq8N|Ku!n>W61VmVz1X>#CrALCf)^4lHYW(WAfGeXq|jc3;}$_D}+Ssrj$S^+9ID zuR1g`GE>BtzOf*az_72OBmwEX8<3sPC!*WUSwVz@C|%WWQ@OMIn}QZ2@b6qvlrXRD zlj%4|Jx0i{q8zz98Df$=oq!yr#|WuKsh`WI1H*P1Izj>16by@v@}~P^z=j~AWSeI? zw6BpfBI3)iz5MIl+HcRqNv)86{E4%Rm9C@l0_aKUNup@Elh@X0$6NwV*=W!$oeJ=- z)I^`yp_l*E7D47e(wwDnK`w8|1&daCYFW97!S_WdMSF+u*j=0&_#J1YTw8lBFWFyX zhQ+U8($?5d2O*=9R=Fx0AMe{4cY#6Wy`+1n6e3oywev$?10aW`w(4gj)A4@{$y8tr~GN@I*yNe&}Uop|$$tlv|^0 zm$NsjICvL}BG%ori&MChEKm=P5tb;HJ_mOt_!c1==XgRKhq{X}b{yl+`}86>O`<(} zB_if>vj*kP0-$b#eHuYkMIVaFku&H^n1Ng8GA!3fG@u`7FLUPk>}}2InzI@T;cbus%=*@I>e3z=B1}le9n0hdKUr;R$%naP zcRTo^=7~XNo7;^Vhqx)Ab>*L^6K%p=Mtp$o&hGe`nh^naQYnU1BTDMs#Y@HoN^Noj zNDipHjm@v_?&xV4La8tPn9;7s!;Qf5LV+)uGM##t2mHJO>L>llrw%xXiz8q}s0A6} zD#8fQTLICVo~IuP?>_VU*&4lZp*A^TF~GO$;ExG z_GS?bG>_+fjD4XvEnBA)haDj^=Q0=yPuAIH++UbfwM;P zVwh*2b>g9(ll(e^zI>Ca$x#Bcd-QAgU{%e|MSER^B;J9i{pw|vI@G)TVi7{+%z7BS zz!v+&W#JyG{c0+pKN;m|zm@?rXu_UB4D&kyPSp22*?@xx2?C9W5BfZzE%x%ss3^b# z<1e=?F0N=qxd$ap^@FdjY6Bm~{Ol$8feVy^34w0L3uKS~ zmI;FGdNGTTUjChffX*S%;FQX4w=8+>26Fi5b`^i%umVzXn_m9T;p?6zIRRuDKl@*p zgi;0V-S(Z+{Q8VJKGUX;P2H-p`_cO?0moq&(~?*CxBExDA_=i^Lx z@kF8>`mMVAZYNOEE;(mbIQvjBXYzNWoskt`fhgU(!?2SHkNQfS7bypDC?v%(r>#1e z_9A3~AX=()-Y)I%tjUc)2$GXG2o-VOS@bt3@!hr+Qn!g6Z$FF^cepCchLHy^2mEVP z=AzsA!@t!=u0G#k`XDGsipbYpMJngAHI5`0@5%+(KU7Bmx)`^Ial8>D#uBGAlo}XFdeYAf9@Po{XczTdV_FQORKndxQ_qy3G_NE zLpGEW;QiAO2jA`XK|U%|4usmlH55tqU`^oLr}6&bx5KNLd;!7E-?Tb3Vh#xR@ILe? zc)_*}X}rTyBL8&NZNO7&hwGL8>BaWHLxtj6Z7gu$QskQ zKKS&%#oAfB|MQErvOV3!MTUomSHZarFicr2FvArwHz2(Th$&zQ+y{GnT^3h-yo3M$ z{R7I^jdRf==gz;}bo8PcQewqbQoP;Kj{dO!jT6BOjBggz;UoX;B7_#=W34vc(MkR5 z6@yn`_npH(?M?V!f7_YXJfs4-1;N{YrT@I*BEIg#u<^_P`UpR2iiCM;yqyL4&%41l zE5yW#7Y$hd@i*LiqKe!&155Gn{_9JYi6cI)-BiEyUmx)c`U0~7#ry9sDFms&R(Fvc z*}8w74U7YJe}n_?zwQ%k?i59WP^0pHU#0(%0$7{>kphl_{znS`BZdD7hHt4rmN`lJ{iprrptT|Nj8j?!K`A literal 0 HcmV?d00001 diff --git a/doc/assets/model_structure_guild.png b/doc/assets/model_structure_guild.png new file mode 100644 index 0000000000000000000000000000000000000000..d9f8e4a3be02cd40fa26c6141969f01a0ed48f0a GIT binary patch literal 281048 zcmeFZWmFt%(=Lol@IY{P_u%d!Xb2YEA-KCkfZ!4w1_=_}U4pv=ch}$y4l{Em``P<_ zpM8FO|G%})T0L|#*415hT~&2Yl$we>#w+4iFfcF}?-XP{z`!6>!N4HGpuj<&+0Kal zvx9a0ATI?|HA#95{YS<^=bfdpG7K~H7zGA4!UhK6&s(4mV(0?~20jlK1|GVH{qtNN z+&`}(ROP|{bBy@c8z;UBh`_)|z`T=@{OARHnuBaiIxs&pU+#-Y@RM&~srY__s$p6R z6;=v09VR&bwGRXKPd5~7N{J)3n+>k{Ap12=q!^OHPQ|*m8<6FJLt|T=*vYy1@rzNv zx98nN)_nkvFTbOe2VZ+zF&_(y@R(h!$B_RM*0MaMkSMh_TK_Z=N1C{(ckQ0?0J%J6 zX6WV|!a&mfu#t68la{N<&o2nd%4)1Ykxyo1QypdFBJpzqKW2kCn)hr)B)mf|aq-@7 ze0MU-4=gPB zzXTqis@#~A8qQG4>0YHg&n6R~HD=U=2JD{-r|q{4)X)WQMm>n4atJ zud@k^ja81F21NT;f8d_H8m`W{5#0EsnGsC>$U+BGNX&rnX^??Z`;SnzjY7TT*)9@r zuO>u>_7lrJ1K|}aoPW!6x^+G{KXM`I%cFk%;oQ{V4A*kJ$_it4?cVUIoSUPat3Cf$ z|COMw?rDoEgzA}Vmd2$~BBpL<|GI>MBemnlQnHlTh;IrdK+9F1y!Bw)kpXo9$A~g^ z`-B~)umNZ%ieE7b@VX;sXVF5W9e+vcq5B(j2OU)(_|M*ht4=8G{DtLbkSjTMI)X@GTPh$q^dl z+4!+|{vg%z`NYkpASrky<>^9%0k}V^K^WLeEXNVAYSNMqSY5iOptt_~8T!iQYEzl% z@_b|Cw!FUIR9IF5r{A#H?)~{RfIO3!L~Ip~>pf$WOLx;BnKk3&`Ge*b)Io)UX{9PR;OPw~&aWBvK@m5UMC`E1}(a}R(#uX3IgtAhBz{3> zh^LxaZ-Qn8U7K{ERTW|qGQrenT6xgYW%Wsaw^B8P#ksYsh8?uJzEsm~Go=(fatu~bm2m>0Qj$5yj0 z_wPaa&n;~viLaYDaO4X>1|3%(sE34?F;iiyF~R!5xzPPFc^H!EAfm}|E!?p`c4#XL zKKMR(%+4Z?EDmVUckt1|?q89SK=_m;5nz~`g}ktWST2GyObLe(tLyH*HZ@NVM@{)> zW1~v2;*pifz=?=#p?pg%x>JK_n&ByrfO1YRAMNP=z(qgB@F-Skw0qA?;Tv(l3ynO`rM5u zu?RoZ(@oX)ocIX6Qpk`*~>0B4nA9hj_~UGx;3OF?y{_{t+;FBB-%VB%S8?WssI_vN`&jL(P*=uf>NUY zI2=I2Oj<0Wd)2c!0JOl&&ivW>QjX{4!wLZ`4hnTbaC zf$M)%MU5>^b!vYJm1qBYy3}~Jf2h)70jCgig zx76i-F6+NH%2))vSH(D0AOE_*|B_8&l{QbU#(p8X;x#M|_J41|ysy<5)8$$|u>IfX zw@(g|>S@yDs8js+NhAY6a2lFXGqXbD+wufn^ zRHDZRb#9G+#gR-DXb>~a@6@DL+niQ&gjAP14v)?ZZ!!10FLF{*QE^}oZ1rw!{V~U4 z@ZSNDNE!OAa@vkQ+2y6B6%}O;)^@!4dCTe4Won$F6}Z!x$vHW;y*=64r`azrYnWt; zc)0XI%GCe+6*a}U!wz%FhPFN=I*5?+GFxb>TAY8&$;2xz&LJ+|O&LGcaZ38!L3~9C zX)B#RYZ+SVxQUcjG}Nqx*$=QEiKcU8zN6jonkinGpP!$bU#x@vnP0@k@nB>-R{NA) znbBT8-9X3uzv0xD6dL<%^}jWbn>Y}qe5+(E*Xfaw>5ajplhyso`{@gJPHS1Y?dYiX z$N4#Y{5|)RnW#}kE1Q}Bc9rS4KQVvk+hS%hhn|)Ssm_NFbFF4Jj6W~2yy-+6Ec~wI zP~aOH0X=O9qTF~IXI|d2`*n3+JJzzv*M(S!?Iy-#EdC_V|7;S%)X;Q5rn&H$r?JCy zV8&u!N%(Z_N6OD|94XvH3oEYHbk0Hlq%E?{v%SlsRuS%imsueW&|m`HuJ!Ts+911k>8MzULy4>%2;?)&6EK=eI@G zN%H-_2V}!@gcmpr3@3@j$%Iq#6#fmJA5ixmuo^hvaG;pU>n^5$GDqmr#uPtkJQA?H z2BjiUHU0}lzcmrFxg4JRX!g#=`JMCjb=}vMwx`I?^Z!ojjCdbOP^i^34JOL+$=*ln z($B>igv(dgkx2LHIGk2L<`dDyd{Ze%i~ApJ4J8o)s>zE@zsb3x_}d#A!WNY-Q>@g; zPO|^G|4=RbBz`A>E3-KKS#|n&u%fJtn4Fh|or&93Tv`<=GYu~+7r#OHS48Rm7lr$@ zr*0-G86X)Dps9@E7_|5HCZz}pI{pg~kiy<#&cEJyO&L&H^<)1B&xXN9`g~9}Dhkng zJzxWROP>0tNmW~++>omSD14-SuIl)tAf0E8Wo$J{Azr{Pm{9i*uWHWJc`w4v zWs6@JvR7GIxtok__;1Y#z-3aFrDCVS^aWyI#J097jqdf-tC5;mS!Y`4uHgd{H2$J# z7PTJcpE>39{B`FFe{k?*+k*cGJ^x3>SBUdSAt;;!2}{Ah=!9A3-X(2?<2f)qVUO@e zscVVs|6x!1<9mWf+LK*re_8XBee~SHazp~(e?=}>oJ(kIx>2)c;Da@dzH9?%{X~%5 z=8|g0^^Y|6<7$73(R|H_&rH`~sMs9D97&^Zeu}As33@u;jHzRbWHg6p(UgfuyK9!>81AW*uycD36asUEOnLP4H?0n z?z~FhnEno}R8ZmO@j_5r?iH36SHj`=fZT|{OgP-&^qeDU74KvEBy&Pe4$r&ORsw^^ zDMz_@GcQhlhQptr`?jMOo;J^u6*VJOUWL>z6@&UuS%~?3y4pq;DZI1zn_uR>$_gxf z&u+HiwcOt~GiR^5YQlILB9EZ24SP%&7$a|1Llew(2^1=OiMabYZlb(tN!l2`LLl~K z)F#W{O~X~>)9(tz_R7NI-H$w`^I9u>+ENY<8_eHKKd;tvkCv&8U84%fi}VC_iIx;*>b65&(3c5ub`JJ6P4#jM`0F}WmJGG(wpCGr*VlP{TQ&FgAJfX|z zA_s^2m8BH{yJuJcucx9b+(1rOdGAKivX>QVOc399E%b@n6ni1k_E$ z6JXJui@m_IvKlo^?z{xYRo6eZuNsdZZBWkip6)AU9A{t`##ujviG{{S)(~$y-aET< zEnVT0|97|&V0pDXh0o19c?!=bER6s;>Z=e=)(?djOE6ey4=_M+C0zvG>H9(T3RJ_roRJ>3xTs0xABK$wraz7#QgHSXxZ)k6Qj z9bgy+j`+=ZkfKsETCwVd)=UFkh9oO4Nv2Fa?ny#&z8kHJi)(pVndM}*vYXS9J2%J1sx`u_dl!7QFI;JVf$*HT7T zc{FQvI>kZxvyfe)PK=bZ5CgvzI3zr<`(}ls`^h7I-NWYryowh6N?O!Kn#M4nt6^XV zUz?REK3jL8Y0htp19WXMmfHQ?!|;YhE4_KiDoY_C2kifRf1;tIGi=OJkNRV1ZEbB1 zFC!bjLbhj*B7m6RWTxe6Gp(9&)2lF+k4VUCX}ZUt-AzTsGiI$ylvS@Qd*|z`XbC*& zQtu>bA=ia!K){u|zX51L&%WtOj)ZcD_1=7JjK$Dzexy@R#7$@Wv| zLV0;5=qB3n8hAYvlarpDyxtuMSj%4fy+UkS`f{3$OyT`|m1`FXQ>R3`ntT&-2`r2k zZ(d)}vuk~vURh~Vqr_8J>^A{Q3BDp)=C0LgC9Wr|7%nT-xmrCH6uA=cz02Mo8}lI` znCFVz&Lxh~wq~_g#vdGe!{M*ln343Mwbj@d+#9|K0yYV{9^zg@J;ywVmxI4qqXhHh z<^mESjU{rvBz|+W!M1MFr`XvGV;;#u?zb0()4ciqW0R>Y-Ib=KByKSev2b9{^sK@*gfmBvDCB)@L6*>8vbOasFkdRX2i_vwEGE9F~aBd z{!NMo3LN(u+j2L2lAmw)hX4@g*&K@4m)HS}tN68m)Q52h!`T$>GLVZQ~6L15ZGSfY_b*S>dyqiMUnAK8%39YZqW@ znx;|~S2Uem1i)o2xO){i4y+=4Jk~ZFQl`Z-N1+^=#V7YhBZ#HfZfl%VD(F|#Zm8`c z6~?g@Wf-?iGVo#4)w6p-R9b9X`2u@V={gdA=yx};=zHD;T(P~cs`^o+ z{h7g1NJ7Q4{#46GU_EuMe<&OH=JP2KGq%zh6Bpmiq3l=2Q|!`@($dnF7V8N6W^z6| z&d?$mdU55PyS@}aRrlj$yg2B(0Bv%`WVO=NJ{rGSt4Z+FYOPW1(_KsQq~GoU10lIg=w>=hct8rQX?CwCU$UHIPGG85G`*4!#n#T9u~}H@n3Ta;#leLz<{+bLjV_h&xsL)V7bG z<#sY-v2(8>1~Q#na>hMxJwpm(l}CM0ST;?$py z>~X55xD?L`P!)ZiNmdoztlnG+3(+C3qzY1Mpkp66-YyfrjR9JLo=A4beb1YK3wjQt z%YYJ<=j)SS)m@M0NkwD1{yV`?5bopE8PY)(C@Pm?QHAJwn>E2P=x0fu>a|;W??~u; zXv07u&p2E^ifHL?X8$+VOc;R{vMgQBiU;|HROk(De@iVt^dEcVl&LK;XOlZCm?s

4_V!2i&iF_gQGirwiDOst7yyA8$_mUN2{$2~fhexG5~H2|710Fd)OCcg_L`I--|V zw5u3>t&RTCax_f1{VHyS>p1K)E0ttCw&;!WnG2a*c0Q6&^$pA;N18MK>JVLyM2g!g z4f>Cw`?$;wkz#&6cRY~k_l3DR@_=F3m;7M=dh_sD>TBT9ga+>`B7+{b+2mq{rl(@RY&&G8_&ADwV8&2<2@VhlKQk?$&oHv<>IlJymZwhg?QgHL5$o3u69@mWJg<^<(GS!@8;r&kpa^ zSXs#@!_jcwZA)97d92z!f+J|GCNd{b1n-Go;6r(vs^wx0)zw=EETLwzaC?zva6h<8 zaay|?cbMbf0bz{@t`yiiQjFP9J;%0#{~gOtx?mj~#*Wv`yy6{Gne&>(1+>m>I!S52{g?*0^TJCUAG;@F~fk^Al2{Pv?`tkTk|?S0OW zAOF=n|Ay~#r|RhFC?6rO@w0ss?M&p629ki5hK4fJjh3({eOZN3hxteflSaG8nUaF8 zj*g6-O;a{Zj$Quioi`V&-HtXf~O&X4|uH` zgJEM(zIt`yd$$dbWgYrENPxfZ{b|pvQ2>@m?dT|TzZW{D{Wy^&UmkY9B&$BqnDVlVMG2z% z5WVp32H$;k^;9*U2y@ji)qJ}k)|1JSciV2{+pZ-N#yBLFv7niqA}5KwKd$$#z0z-?>e-X z)D!MMw{JCHhWFRiAIs067Ta1#A@jKPbto2wVG@x&_hD{S*HA(z_}2$eemV$WT;4;3 za{>sHEN@2cDfWFLaxo5^`wK$S4E>`K4ZQNK`(w4^T0mWChQ6-HNft##bTTN~MjqP^ z?RNY=rwB;+I#=PtVN#we&RJOyXW30L+r@=Bi$hBuRFAt;?+gE%t7wky5w+JmH%Cb` zzt()K4MtQ>74OL~@ooYM3?CPSDah}KXdJp9iA~35W=t&`$-I`F(M|Q8N4y<7pL+rU zUgv2>mU#GF1FZMyg_xatA@yIQ6~~t-3$Z*>CR}aBUy)whc>% z+usuId|w4EA8tByTs?LA+-;G?ASB0y9kv){%@Xw>|IAmHQ~7RHC`E)ow}6PLsiXj0 z+-9gP7+)+L>Uw%=k09?5Uw3$+&bbvg^x3Zr6u&Vl=H9`?4dq`m-U>g=@%|a$n;(2< zy!&}D;l$7zRQXKdWA_p^BFD<%85#@^3D_{gS_`BgXP2!jYx^DYzS=_fed<|$0K^+; zfXkqo|CD|@Ocut%A?TQv(=#K#|XMymVx9o>T$wk*#++MRf-yw;C*GSySxiHP9 zH)L1%!rOY>a~BnhP};D>2*C3N@K911Pig7aXQUF7k>R-kq%`x?Voh5m4HUiY-;D=J zn@9$;2_?O4R4sfCxK62`>dv-lKqjlX=yvGt;RwSpC*^e#Jr~2SQhER*n_BLezxHO6 zRna<+0QgOcqU4&dTAoad41-d#s=Bku(z8^H1uU&_H>;q|Xk-Q9b6sF1c=Z%*JrZEH zpdKg`TMAC-{lnazs8fcslO>>GJ6QLH|avmb|dN~q) z1nr%pF-2Nyv6oX~Tlx(Mk2zbY;Lfr0irLhw&WT*Ce~^{;sducJi!Qi?A|v!?<6XP| zUCZ4lT5BDlL+4ehZ!Q$$t~?>_^zm;#VsE!SwZ)MMbwLhXp;g5wtD2d=v1DE+t{UUg zhAn68T{}BTBEnb^c8C}MuJW)+v^%t5mMufcEjq@Eg2T7yrkBw)BE))RQAfnTbv(0A zfXA7U)U}w=r+T#7>{&##!xO*ly)sRt*Q(4Q1%=u<*TA{$!1l|=#Z4kUyIuFEyGQ5W z4I|<&Ui0q!IX>6U%^U9GX8x>GI;|mZlmq=wab)S<7tMP12Ev>7qzIdLIvk+Gmt9gtslOhCOsEF*^n%ZJltQ=G6 zz0zvP6b<~dipQ-xv8c)B6RlOksi9U*?t{g$+T;@GOK$eUrJRZR)+ z4=tY_=;?Xu%`;e@OA4___^+Og0<(at+1_CxbKH{aqW7S~gVl43EsK*%J_fPFyZo}!1C2?%nuAw5qk;0zgw5YA9c!5lc%nt*l zM%LDba{bP)Z5`OFjp=4EymcJFA+jRgUL)TYU7xo)T)*H#DDi85C8e>kwC`kQ$;XmM zHxlG{(!(*j6+MUFFZ(ug$oumIyxfaW4i$zrAhY8WGwg;T7*w#XSe-SFv%woLA91vd#~iu#!)=C82C)CK&+x{&>qVBCaUkX5$w=i|pSeV(t(q z`;I9cyb&@PEO?>wLUv$yNQg1vdUibOd(qg*eax@c(i@U##}a?s`Eh7ynP9)!>2bVj z#m2K_>HuzE%S}q4%AY$J#VK|avVR%}1Z4UQ&>fPyOka4NM%2=i zp%DM`fkiRm$H#OoT#se%_R~RP);z3pp`8U`GqlbR_=5e>SGZl`-?+ZhV3|05kb0DhBIY)ykuyxEZl4>c-Lr^kb?g9v?L9$R zlOSCy4Dsz2n&*o3EZrO|(I>oJK`ha0+NmZkKN(L*Czg2Ye%$l0%Op|YBk)T8YBVf7 zI#uLlv*f(x5H`$n)qUu8;Y_t53MNr7+8)XP7h<&R8@8WX9!IQJm)u25u@eo5iz?Y99(D=EZ!J#+&Yj&Ze)&VAQlfM0 z#E5TEOU%Z$qGy6URx?|IIiyMt+%w9HFg^|u*}h4}5;Lagm6GFle0Estw}p7K0LxW{ zz(+hP!ik8fEpK|Wrvf)z_%F8C0=ql;$eOrt;VLvS73+HgGq{qs{SPKEi3Ngi*IrSb z-@SieV-T040$y1eIZxyYzI>O2U$E%E$Ru&Fpi+~&n-o5DL(cFR-U4>{UIdIr4PmZ& z6+)}l96BwRhE*q!&Zi!EKpYvrO}l-Q0EmO(^5)_(O&>U4%4OZYcVXp9@wK)OnxP&r zrn~-b>zETldm}_z`9Itk=0e}jHjEo3=_!P&4_q3bcRWTL`T|7_*&bFq9%b|oFfd#X zoFM@q$rJEZY2T67HEllGYo$`1DUmktp6JD?tL}$QH-q{*@JorA&bNT^CDa)8HTSVw z*35WL(BUlD1+K@+TX`be8x3XfDEGCozT3v~Hx}l_Ph%CVy@PwTu9L)K*TCwdaGs}z z*|^o~1>YGEXGcNQZ2F|nz`!BsEJh<;DU#1LOB!)}4EViYNEZ5E}I z9N|FldHbo`N$etLRw7ecr;i&0^}c2p#|Kf0IJf+*K!9JIhpi>mh8spR&$KGELRQV7 zntlo5;pn)AHa2LaBn!n^PMX>w6U*lY?S>w|S_$PeutW<9{4akE zt8d1Mz9fJ$zANsOIu4G%xZkybp0D}d6J&W`pQb1FW$X7k(Cb{wtW8sqI5hlvB0Ml< z2t?SQny&+i`7fBFXP3v=RvLAS>1(NAC5Hn1?qoBC@UKUa`KyzjnF2-6rU3Cz2m9Ka zKF{k|sE9VNEm7ji2bxWo<)aDMOg;i|+y@B-dGAHFnw%Z?OZ%|8jw#aG8x5Z_ZQ25h z1BkpWWzz7&FYL7S{n#tRi>$N7aw)_hr`rI(CpsRR3~mX=rFttYa5|^bV3k3;6M?Y> z-ATtk)(sateUqi+BrEnk8m6=1BPf4ZgzSEM2$19V^ms3JxdaU&z3*v`8hHw1$RSn2 z4!$Si)r8z$px?|qv@KuvGpW$pzKjpvzF91TyzXa2NvLsVbkI+i z(PGlp?MpHtNgz44h;5VNU^*($RG(e*h3XoqCVXhW&lLuNfBF*&`QC@B8u^}tE^a<0 z2qah^Zo%KJcvj~mmR4Jtey)!xcUfBd-9V8+F2dsRx{ouv|L9c&>+#iN+Gfzs5e+61 zy;IC;W21~|MaBpUr?gTe+oy)~;e=ngG+4rfCaN*4tM=`5I4Frorj{$YIfV*33Zx31 zYWe5D>CgcAF_Uw4N1B=>OME(9#3}K^n>2uLN`xTlpFAh@`{Y-ezIPv$e9o|RzEd$P zQgYzKmDPgS<9e4bHu4AFvtYr_ltv;m=(M0ncZ3wu3E@|B1g~y?4+c!y^8%;-((DO7 zqS3;dBgygAnhK4>I1yc+DG(OVso(i>37PXSY`P-p@B|#CTJw2w@rlbDJp9x4!>TYT zXqoZLfO}-A`t|z5g-uJ?hli5^FGHr%-br~1-_)Rtpu_5tmkj-L)&pHoetaq4>9K+v z@|?bmA6)8J;4pmf2{8zoQ}~@JU3Hmely%mfS1X+&WOW(g=Nc@#->JMQY2kIofNlEr zNt%Wxz`P=)@2h#Cm-oSOSUZ+pvy5L)A&e>E=PY-kSw9<~4zE zX3Px0cu9$*@IO_ZL*h% zqm`=la;rrL140?%nsXs^pXoO|(Jkq9rf)vam@#36o8@!=KGFoDm{!?rxSbrg@Fd0< z(vyG!c!BS*dqb#QiGLrHt#Cz=<{GdvGiUQyfwebdEU~}5ksOR6Q^gx%jtr~bx7ast z9#~}@Pv8FPN}u6>SHag-fADSJZAdu9qec=94jo)C?XJ( z#=$3VfEYfXHL|R0XjGBK#z?+}Q|QmHWlHRMbEr@zww)vz2!E>8+@%VuaMd4BUn$^B{GW~)+8{CXW z5wI19fsA49tdh%X_A4!3>@h|UrXPNfj_@bnLc{4&EzK4DD^$)%T4;OgEPaNwd#V4O zxS9}F8ubBR-E4sv(c^%bs_0Z`h5y2*Xj!-H927O|WJ^>roWAg{*I&7kf5!gsKl6@$#kxh=Miyk>><{ngwLDe=7;pXs~7(LtXgn8g_sb~dHcBnlgb`6 z4?jE}+934ZedTqY6!Sf6x-e{i#H(0gXG@;ghsAlGER=;-*L0h*-?d(J++Q^u*P|dR z_dBpeBfCX^MI&G{RP5YcpJ)a}?|TT6)FpniiF**m^=WrI@6~n4tii6Cr*NTK5?dRB z;l7UI5-AyVpAba^(rq;?{Ze<3^SixGhcbk*K0x76%%SdPqwoDen}cls;0xgkaw_lAe3z5K;u} z7c_Uf6gOi)zrHlR5k3#t0&;X8*yFR>yX>IK-=A)PppCnI)SsllL85aD(1ArdviU{7p_=f4Jirn@#YU{TYm?BF1nM0ucYFC#ow@Nq+1Y+paPuK^Cxj64Hthpm< zH1*ixk2fscgz6Oe;d<}pbu5f()u24-B7)q^!OAcI?C5&9?lRQbyy;h2PJa-%5dh42 z%n`!r<_h^#Z+Gm5Bz?abq2Sf&{%qeqkALCMoAg0PInDFEQm#mNRCN1!hbaA{!I#%K zV=Y(7ItiBB1+o>yqbCnaW6c(nkKF`Dfsh<5pyF3+@0fsvIv6E)W$t`q-1*DgBHQ0_ zf(0?%SDC)!m;m0&co{7V8{V-8$`25LJmAUVH*Yg~5|7Qg`*Vg`&3gui>2af)^yN9j zI~fnB*~jP_tT-FO?|l)?`Wr5XAVA=iyRP?Rlb)mZ?{`Dj3%IFYEi68wD||;8`DcPl z?$6jRG$MW~(Wl&XZ!uV0rO$LRC6_CnMRqfDiveO&)m(!$2>`1W{&_#V(_p_@!H1MkzjM|WZ!^w~SS%4Tuc`>j0@!#$)*`a&b`IB7{IaXo?~<7PXw zL;0I5-k#PkviFm}SPjA}6I$c#d^E38Q>0B0FrCcSdZHCDi9GnWxM_(}>RAoNRBpw_ z_ah&Yq5T}_G6zlt65_*#oZHNmOheHLU2Cb@vBW;C{)b&M!xng&8yE(R$(;jef(#|sf(V;i~-grnbF#ZWurCCPk` zA?F#p8ht-yww*wa7Il28q(aTxANd)Zrl94fxBh}H4ebe&JZSy9L zkK}6qyX3TQ-#`wVqMaqwwPCTsh$0q>?V^w!+v16I*rlxM?xZ20zTc7gc_*;AXe_n0 zF|s9>FwhcVqkejX_VM@=)z2TEmOe(l1BlQT8xH|ygbEK1<*gK>Ub@jwFw2zqCrw=o zlAu;VaAO5-^j@(vKRY|K>2aFlke}x?tZ(}#v~7FC<0uj(kAo0HpWE2oL1k0Pm1y>d zReEWp>P`>G@n>)Z1rhnp4MUYi(m%rV?sNwatiXZ2UHJlQK-rNu>$0j8-l848zC zjzQQQsoaHjD7($LmDkQZY#|Rk7!arMq{UsWSmz_={wys78U?K|js+e?spq|G4z(L3 zFCQ&RJfHj`Rw+);)FGXCi*bwS{fKVy@-WK*k6huoE6%=azwU;A%%4RNvdR#t@4+kV zKWJLsdwkUp?vafL$VcDaQECH;$NWNaTyXvZ9$zv7iTQH#rE5CwNY1MB4d|HY8KXad zZ>+_#mG3P>SV)Xa_zt*h`iQeA{I3OJH?;KZ22Ui5TtLT3WFLRT@eUDRKx#$JyK3vi1jUB*xy3opU?46s|tq(M-;&vsj^}Z1G+@Ebin*Gd( z7??5UvmvV;3wC%B0zI$0ty{1Lg_*>-aSg|CI%7ufJbi2TpWX5!IU}|AYa2YCOd!Az z`$5>g)@{6t@Cw>c8PsSTA2)w+Giwj9i9s(8H^(C37C2zCGRdA)^;t#|PP zRYs#SX8KH32y?c{+?(Zm$mgH`R)w{f9)c+QNmDtb|20Qv7Gp>*{pACNssIuK8a`*IQ0l z4#gzOMI}zk<@K8RhP|?r5tTuak?fcZ{%z!#6CjP`7=~WkzpDzlm@ZWHIa$X#zBeOp zO(@w`U&p0aw)~}_bt;o2uNj3Oxu$cN9B!g;MKD2yUw3=4nC2919jhHvvxj~hAa7Sy zd|WM~!zp-iy4@d~-S!aDeS7*V)H^>OS`8i)2FIK-2Glu`d z*%oOz@W|DV&M#YZGHfo8*sMMPx;$wHOPssIa)Xk8r%D6tv%zmRg!?VW=~SKlrO?6IY%u6@s$1##nQrl#m!2l9-(c1XJUWvJQ{ov%Xxl*Qd$W!3rk{|&(1WZzs;-Py z+Pee)dIOo<XrEIEd_ZE>LHC^wQk>tRRqpv63;5*^maN*Mgc(1o~Nvs6k z5cv3O>StH>mY1@?wU^P8i_1DOEtAK~${t!A_ z@D1hT#}wRmuN~CS{jo&lZriqg_wUBh*IDRh0y)~g63o_~qfE72+wJYDcK;+{MRk#) z3d@N-Hncqtq26c%ABvN`3%eQKor-&O5b2^EiFmAsEgMr{@>;C4)yCKCO@p$Vn4wFL zwSX*ezO%||LS4Og)V?mna|BvHxsGAc&}?@*9UkL@8B?r0^#e`lY#c`zS);w#ezjwe zv;AdCSIQdSY3EqX5JCL9UY%89l!Dkxnr62g#ke&T7tCA412e3zg}?H06J6~UZDyfK zxBEgE_);tX+Ib>F90F|{_(1v+5sdrCSv#MXlBd@FA$`2oC6yy52lT&S{tUA2#)+EB z%F>lzXL~Gc8U>tH8Wk&BndC#8R6$QW?$O$i-PEZrLW`IvURFEf+3qHHON-i6tmtLz zhtA6xTXX>)lyO@&um{+}tSf8S?;`<%P@fapJAB3h6mH1%7&>#oJHXxFGuxEADApm? zgPJyFWZZpP9{l^nbzlcGI-5^tzFR}B{*u0WnL;BTxsp8|H6Ito97wYqg zW7pV+ax{F_Jz27t`pSBhZZz;kbp$9NQ0RV~`@40&*K==cpzzsUJB8OXNOe&8B{!JW zlqz-iW*U8`j|2mtTBVdaS*fHez@_nRZG@grn2k0mtxY%vfeb&63 z#IRHP-tDzItAU}%P&c{PZqEJU`8qCYB z_aZWCKWoU#7cXDGmkam$blbtB`V0zwZdL3-LOQQ%1>=)bihvvmu6!g0Z@eC2=bd>aTu2-_U$#}pw^&BgNwIMH107x^~mePJ&vreC77vrt4^A<~q zyZ5v}7+vi6!E6I!yqE5bTiw2WS3=Pau;#Puq*T4j@mx87Of!^zCu>1hy+)aBxj8s~ zFpr~cz#rljg?!1lvyLSe@Oaz&)6@ZdY0(ZFwB|8859KQGXTlpXiac_W*KfW-%liDO zoV>@MUI9OO6JE#5Q+Vt)rezd4p0!hyDvpU{?W&LoJG4I1Blzl1GGK}i5o&2ehT%Q! zYL_{48#=C6aEt;(pN3qEx&21dIj5j?X?thqD=0Q;+r85hCMf9ifMwpeN@t%+5rg8_ zitU?QelsLY{diVCGgb@l8O$PrYwU{}Dx_-rk9SXRzj$Z$sR#+PC*FIAf#-nrmQ&le zR1p6iZwHAQT03X;D`+pTHmm|Sl%cnK)e&pKjc%}OE6Mih{;Y49&-R5=Bszv?ibeK+ z3P41EFsYnq@hlpN0dF|2B}{DO;LSxJ)H4oZ7~~S>)IRAqTx_&=-fvquc7e8sXu8c? zg3<(d-Xu(Nk@3|Bg~i7Ewfc=_+3qst5}GM0U6^B2ig!7?jsm9=d? zpfMx;Y>`j>?V*&S7PTeaqr&<@+W`7ia3*88La2k#ESl$r$!aX$it*1j5Na7R#;58Z z63=r)?*5ayC9h${9*&YL+Ll#*I{j;)c%1y-$y!94|RdH>F$Lx*Kg0%wntFb}p=S zR=4YsCE~+zkrD`PJRVFL_1}C{$xmnr55W}QgQ+f5opal{lE)&gnKZZ;8_%rmPz)Q) zFiIHe@10cUPzXiA{(WlS8{RqW7bQUuDbsQtw!}| z3^!e-qo_!CEEVYRWjaPpC6S2c%%=kk7(5Dl$q)h8O- zW)aCJ$a9)Fjsm-#eRUqd;n%M{VP6#o?*WsEdK)h;oo;2}1+Fhc4F*GA;Hm`d%t)uI zUxf;L9Q_GaPW}@cIu{XJ-o8f02?DH$3~#B3^UCU^OGbzK$eeCaRw^#6SYp(q@Sf}JI1%|(1MrMH@u>+A~~C{aXh(_l!RBf6ds+1lj?S%=r%XE zD(`un1%KjmiL+9mnz%!|oFSGYnc3Q5=@g2AbcHVvxM69^6OaR1CB;uNLBwf$f%>JJ zRgUj5kfSr;T(_^fbqDUt0KHQ7IP=>Iw3SX@{(cdyEE{ECnBN> z1%0u)-ccKIv$*vrs`Oq%Hjz!!02T?$!HVUN7B)7}$$9j>xQIo&k;#-_Mj3B2)7}?h zuRp#)y3oj8R7uw#?G+qPGAP&PK#HaL&5~<^b6m9yEXGbnmz}4V=D*7%wGXY+ch1#B zBVY#~r`a@6{Z2VFpqcbBUBUUgP>4>Mr^X4-%w@nHCeOSJUk`z0v)X{ z?>oXU6F4nt%X%|IU{k5ghsk^2`)-xjcjgsmTZnx&HO%*tNanlcOpC)M#CYXb_DL90 znHtwKv3Cb0pY|DrZGR>0qpy6Y!HYQz3dIN7jcfdh`o&~Tlc*Zhrgj;gwjp5l5QToh zyRKX#D|Z;H;@p%!h3VW{IBli0ebj%1cRi$JS1hW5XHLHnhObn{sA z!_F9i8Gmsi*Io+kKcjBS(3NaFZ9W=%q#*8Del!d#uC@V~bWta$S*AoweDoQ3h5LN_ zdp>#5X|blF+nbX4i_UkKZaz2YPn6u-aUP$G3xf{`c z=($Iq_xldCYgARM76`|r#xopPxOsQRu#4)D=oZxZdb~aBXK8!adrrrQpU7f{X@GKe zb;AsYyqlnlxF~9$8&JY0QLLBK7pT`U$4nYK5z*2v=@I0NI!Y=C zXMp&Qf9&NU^Z&5-R>5&K$=a?Zi!5elW@ffzNft9RGqc5##mvmiOtzSrnaN^i*1!DC z?0;rQoH*C#;zaaCM@O&jT2-|&v$FDiikt~0c|iVLR#s;$QtJo&b7K8PVX@%YxKO0X zysnufpTVESQ{&N*uyANK2o$#sOwy762ugtDXvHwFRn$Vh_%7jT+D68~ko-Wo$1x*w z+LAN8?II;*A_{FtP$W5)gemK;_=CES7}rX{=0V|*v)OP55T4qGim~hcS>H6%msa9u zp@P3fcbzXGWGnwKS*3;`&_^&il~n3(&YY75^9yt6X!|To47W0TDDo*wf#YXj1Vt*e zc@=Ziy3#9ab-Cub5~qWswN zeeDT^O3!*`9ASdu_VV638)0SCS!RZmQxOoqo+4i@-+OOE|Mi1R^>N{oUppOq zeq0-vh-7gpy&chAQ@YYDzL*^;L@cK*&*A9$dZ$My z3140_RIIeJkcFzDs;a6YFV24bT$|_jAM2I0*gCQQlBDXfAy3u&mm_(; zL#`(r$jP-oB8o5vMHeheP3%?cuVjUV4Xw4cm6h2Vd)x?fJ<8Q{{r@xV1;GHQNTB~& zd521IBdwrKl>CR3Ex_Q*mAYbk)g4K|&#jEx;S{slA@wQTXcl1a4y_uso& zfUELD|7%)=>?|GF{ozHS-1YJ5MX`c*gdM-B8hQ+kvW-5~{5kluZy%G-(N{6BA~*f- zY#`8uQ#OAE{v66=!vC(^{J*+nLS-;k-x%t+8Iu2KYbXz2I*mKrp|qlnDEuzWKK&SNhx2|NqVWe{y7oBom2`)LFi;va?sEMvYqjuPI`- z2pGzylVeT3fkh5hCDldX^%v=|u)x8}PKm3<$CeAsy8oJpIfY<#Y-O_wPL>A15+w*N zQTcU#VgBG?e}5mS0Nyh?FfuhYt`6ngtN&kuG9&zcIEf@5E{S~1(a80~uiwR{(S(VPO3jglTC_pPlCbA3*_N$U<^tHIM zAIulSWZ3|3g!11{(OI0YH`uCihGCk5Ly&4*_7MY&Y|YZa;W z)fZM1IyxH7+PpH!;;4Qy5mwh`p$IG#)@}UF&D{Th;>Nsl`&G{W!m_T5p8ZIx>D;4{ z_u|`1zi|HrnF;dv$_iG2Z5IO^vpFzLZU4V)GF}V&T_xoiiO(yt*Vdf>A?i86QMGW9 zQ6{-ZjplC!A$5sGvNUe8V59A6e%I6jZDYtQD=R9gsVQlh8t`+{35EDyVGdxF8f^c1 z!_B*7n-icKgpqQoU9&i^2b6_H>3_o&LIgz-hlZgT_P%LAv9e!NKdtz`zWRHjK>m`O z0_3)oJUbPVp)x?Y^>l=GcJ{4647W*;)?M)}ko@o6O>J=u`y9&3Zp?u-4wI;YduzQ* zeA$|{aNcXC%bEJ5h;mE$7lDS%%J!AXeb1sV`X4r)l=9Kiv;C7fuXFP`{Dz2If!7sSidt%Blc`K! z@zKWEKM(J-Y{e*=BeP6W|0_16?E!+8`(69**=pDL!1YrL7EzG@;eSa{=Jc`qPaVUUtj$!+W(<(v*%c@JaBVj}Y4lJW zaa*XEd0JPhCSFDP)E78c5Et zV1cjCzA#XbrV))i2i!Y_uGsYu8;$_vJ&L(ZDVm}(_r7=8-;s)UPhQ17aoblmE_dJS zv;&2PYl%X(a{j-hz~TI+kMY(ZrNS(-mmj0Ln!ihGEUTZEJR!0ACjXOCY<5-3eLbX{L7?idj3qca ztyvkhqJd+>Ht5{i%0K`Zi_4Sm0{-x61jz)3*Vby0_5G2vx>O*T@c zozDIH+dZ7=m{oQt@sB!1H&hXB-yH~ zd2LW`Wg%J4Oy`XHZo`qdb0YMJ09SRX4g}!9eEb(+Gje?M)9W~`JO5dSeH||8w}wzb zRuBy6`{U`xlAM~{*zFVFlfrdfsQX1oh;tm0M-Ris5b&-+{6~+Z7lgN>XMpHeN{uHa zKrb-xi62Di4Hzz98TwUGkp_71zFeEL4gNs_12vt?qA~(Xb*M^0wi>QHn!cCQVYJ&7Wxj<}eR)z&V z9wZgz&f#Cl$lvo@x-%C<8H<@srOM?QJWda10-!R=JB`{FTXXU$mww4(g#-ruQ4~C# zJY)b}x$NJrx&j`Qjjd66-V=b9%yY0V-C-;|BH@E}x2>(czVc>W{Dl^g6p^E9;mzkW z1Fm|79V8OE>FinkZ6P%qo`?L<2A}<{Yvpz+IDR~* z&;K%KHY^N=J@#CQ1k-sPDn|iZiH(qR{MVb5&rbtS>9uo$USzX|a#NZZ;5e>^cvF(K z<>l2?YW=?#cUe7g!TvN4fXgp{gZnm^sB8=rMgbHgAo}taYZ0JB-nmDN^yP1&MfLXEPQFBc;zuKCIDvQN(2sWQQ%^{2;h4whom zMOp~wH%1d+A((*q4-@DwFE*?1@v2S;ewSvJ1I$# z9O91^P^gp(3P+HXBpV-t^{C8hD+L4mJO+~=->Q@`VCuSy>A(%d)5EyFT-6Hr)!G*> zt!Iy9HYibv#;(*5x?lGP{6;)s5`5^poXF;I`zWY*Ct0- zfO~|j2cnPxLjJ2#_SppugRcNcQ{La;=qd#LY@no8C>@HGWGf5pbU8CweDCI42M4X0 zw*+yPRy5}~II1JW`sA-xf8K=*Lzmg)&Z6~HWQLm53}{qkzD}_H9n94&S%e+K?t#8P zi|Bd)e$`(}x)ks$EpK3#=8fDEHE@tAWdL~8S|Av#vS{cRX#}phAxVyW%(WFEaI@hzA;n%2i9oi0*S18y(=~zW0H(ApMOQVD-a-{>12|lWtJAdT~Dk!S6747 zttX*-_`Qb#&;z4lgK0GVv%IB;GHLiXJvNjnu<3a%5bHKKSB9ZAH1mr9e|2?b<$0^8 zrza5+5f=%UHT{X#S1nZr=f=Bp6U1Pq3urQ*fC0pbS4|nBAKnr2t_Z@Yb##i+E(vP0 zc)tX#r45D`k-#h8SN*qfo-Wmmqs{cU2W(tm6@Pbmy6JUY=%FaHpiqM_6*EkL&lJ$1 z8C>BNU#SiH^Ev36VtKa~o^V@KmK9`x(Q{ZvmOAfb;NIL6(D{B^WJ1ythKlb;>repg z(A(9uDJuEafsBF*M4kahgZ=!kd_(WO=pl_d$}2R}^=;rq&wSP}k+T;jW*dB3W(u5z5shXw5E|Hc8E)n=@%;n=pBkJ2^sT0fePCNmtw*nInm ziF%pb2-)#s^BRIgLN5Kx&%v+=OFRJNkfNn>lZsz%6#;ngQjA6|RP-Sq>m0}2Y9d71 z^1dWO4V{ptZ)q!6g$Mja8#H-=JQjR+(|$v~?4LwU1|P zbTlB^op5BN59Ow~-uB(WQB5wZ?SAs~6PLyqO&VUkv1$BhLUQuS7Z08xw`=-opA>#3 z)f>q7?kK8Gih(dLGOJOPNu>-oX@dHn7zFeAAl$_&PbR!FYgDWmf@C~%u z5rarrPpNc%$L^kfpnPt&(((Y}VasNz+hpXeXaywx=>}sWy}4^H&HOsD&8@OQcsK{- zB&B2q<=&;jI)=0epT*|-blEZ3s%`t%$t}}oXq5o5`8~5%F?QP3R0;A+11;j{Nr7e-YYeOpKmbx3~5g%ws<^AHZ|{`d`X2GE0v6Cqj$J$ ztoE^8n|HXr>j`hPJH&u6WBYuA&!@9s5jG#^ko|(4H_5Fo@pUXg=$;ZTi(fTQS6m7w zqfK{gwo9e~`D6Le>!#7Hj?C$5Ab=ni-`;GlT0&XW2mr|yjW6R%x&t=PDkSHC$I2vt zf$uxuLZzpJ%drL?J^f3b8yd6JJpZ*CVRbK%LwsM^L9Q83&nEJ~n2x{pCRmrbBx+Tl zM3-0T8M(^IsX-Kj4u>AUsDImh-E9I$$wJucNn_WDS!e?r8k;TWprssR4me$=Yx*>aaQtenQlRn+9x!8>Z zSGSJ0s1Whn#Bo7%3HP9;qc$tHthxj1W$dQfx^{UAW`e6Rw_}2}B}Kx3s@RG;B*g2q zMSO|FYAEOl96UEWg)^|IP=RXUVK z;)TH5J$!{ip6#_S4lsx+c=f;a7z`jar37EiWY)MY{6p`9Mcw5dA0JO=^Ed}o!OaZ? zk7+a$G_;3POKP%31y-79IB6qcs%GV8yNw`ETwGdWlkJZK=;`q7TOTroBMxK8>H}R< zrnfRb0UMI#`RzxaetCGSRo5~4$o*5uWrJE}VvAsd7EO>)0p zI6P68h5jxN+b?5t@+7o*JpJxng-LU z=MP(_pp%TW>Z~^Fen;u3iE}z+JY{(O_MQp1^woXE@VE?ZX$(GTMucpL;;RG>$LT=U zQ|iG+&HPD1t)}YYRK01mX_ou_&7{Dlb+a~5art#|8BLnsG`%w+=JUc#k&`1Pa+9eL zflM?DwNMMy^_4Nya60O4^hJbtsCa3sl!6#60n;tV^{hNY`}=v;?{mQ+RaI5V+S}mk z@auty^xG~HS?lKceGJ`8RjFD#4YdsQj0IY&7)1rGF^7ZG>N55{M~53VmG@eT_u%DZ zEj!W9cduiM$VR$`(s85S#sORwvE(l?02bmWUqeRGAJ5NcU^J1QOH**s{5ymih#>g8 zA}AEnRJ63LAQBryas1!&)UO3azK-oJd&a z6B7HiwgCFmlyl!=2Hvx@>vXwDXX%eca#fYp@5r53HQknkFv7b}+YOo?mK zRMVquONm-no^QqN0TklG*RQaJfyhTlixW-23*C^tJ6l0su1Wr?aul6HSYz@Nt z128gL*QeH9f4sisdKZQ7dHg!ImNi{K9MN720I9w%MMKx;V;?JEq^J76Hfx~6bRzsop`6WA6 zAIWMOHF^#|UQEU5IZ^bChk&-e#;#gI5j1Ke4NJI%q%MV2#mGO-E?QKFVCGaloDm!*Eq2L|rzvQu1){z*hzseRXw=g~53 z#3FCHx-@_7If%)X@hZ6_S1d(RUCd%{Hnb5=U0TXyQ#L%Fcr}GJtdCrlv~}2In>f7R zZF9kBRE`R(8HlmYa#*c7t7J+a^{i~Avr4ZA5w@E#?^e2XcgD*z)z>6%3LBm0I{ zWRbT$aVl`cjh3>t=9D$Qw~ zZPELx|6nwUes`dc`=r8gm@FCp$6@Yk6eg{UF#a$_mgn&R&WE`5hSN<*06gy}2C*ny zuGfaymfc>x{P?~2^u)gGS7>doaW+7q>t?KsmJ}mnf2!rW?mWr&LMy++GRgn@J?HB_ z#)2hRR#*sLg`KiqgQjk#woc;HY{0h1GH~ig4TuNts)jTzkSgt?jMk#L-j%2s0P;e> z*&(Pa6Xnb5}9T7wIi)Ir!Rbi?_-(R@3JVE(i`s6gXmKa&A16BfAf4wm8}4kkf3-_YMMfPc_3W z5zpz+IHV$zF? z3p3yNR)V6io*A8u()_^hri*pGp5`&>BLNr#KsiEyAW0OuFVYfC7hX~_JFMu(!@e8Y zNmVwiX;wjeG1{6muRja!H_{qU`<4bN`2C7&!kh&q!?$~%*}!T7gZJ+7(_jd)-r{^K z1DPtKH9q{d0wPnxvYgqwy?)FxOG}HuTt;;0az=q_0Gsn2rbW!^$`_suz2lWwSPs}5 z@FDanOBd_mT-oq$OsVz2h;E34)tfAbd_t-8Z@@t{ty(4Mgh;W9JJb-CmgC8qA+m?p zrcUJ(YiySZirUg5={Pz;(Ju`PLGUEd+NGFeoU+51HCK2d1+L?bi`riEi6LjTgZL+v z%^Tqi9d0+{9Lh>cYxW0tth_S^z!^es51Wq50He)gk@@J)ALFWRNAnD)nb1jr2VCo| zoB(+L;SU|R?J<@~J;95Fc-j5YHevJOrP@tA$1x^>cTRA!&yet>Ww?AlZm$Lj9tLo1 z&w;gP1VLK$IYiLI4FG7fUp7vj$Y2T}eni_s(E%&GIhZ8rx)J*7al~4oR_lMmmc!0w zm{A0Wfwu2+xh=M+<%Pyva7kh?S@(Mej;?_QWyT|B7^Pm}`<~J&Qg56akmiiXuw)yI z=V1H#_uQ3jZ>>`gYaSqJ+D?rIJ%is+8R45sWg>(|GH4ss`eM_OAPm%(s6@akezl)Y z*7bJkLkdn#NRY{3yh_rvZGAcay>6=s1P=6_uVFwqXS>q}aM0}Dc`n)vQB92DFd4UU zgWvyF#9u0oe8=@#PwgE4@piX}?{QiypD)UV#UI*z-u}=5l=8L~H3iX=J8l%oWd7(! zQ{{cToy{79z_D@zuu86nxxq{av$sQM1DRX49q;#99_QXig&5kGcvye9e34dtX>{G-x-KCJUG8|jE?1Q0yGu2dqI!FI_;|fx zc)8b3hhGke=4UqOqZ4?$&7T#A+&RpTZavI{;n?(a^xn-31gKXZWXW2dJE1;BmJU}U z7@e0rH{TLv-e)8kejk_7w2n?TqZB#VwpcCK0n_AhT8!Y2O+$Qcvad!--q~?^b;t3? zG7mymS=i|ZMFFTdRDs;*+@K8)F+oIBrM?aAb?Wn#rW5w{P22Uik2MqB(G{cu@oMN9 z58K?rID8=?Xz}-KmK%rGwl-dTcOsz_OV6=PkNq*O%~TK5i3}=+4z&>Z0&DPcIi<+~ zXRPji6y|axy#7jPZAn4cSG=nzN}F84uw4q2lsv52IxerwN5kRbS!SkYH7U$h@*r=P z{Q?;%d*AHM*1SC7U=(JS`r2h;;xs3-P4cen(&#y+{v^aZm|iMIMj9FlCN46L2UxYh zprCIe-x_Sz6?pwd+`1|~U%!DiL3`EpibY|dq8h{#L<+>}%Gu%Dy&hlTpv@PE@7i&{ z=VBi90zJq!>{vLQ1%_K%GdOmmF^qp6Ab&9+@9uji-qo z{P{yhTKF^Qr_J>=Be=`beM=jDTb7YDG*!+HG>|;AWcM!-DVA{0dJjA@Aif- z-Ibk*Dljx;JY8gPF#Ef-bg@Q`jMYgJ6m9)}!cWk&h&ZsiI#xas6e9JB!6yX#{xb!u7(SmrwIO+IM44J9x#M2il-QGm>=)G|t4YwwEwZP71oA7-a7pA|;NGAtDJXpZ_$5lmeM%Bt$s`n&hw>NMj!t9Db|}X^BO#gl z)U~!Y^6hQuG}bWlc~CUWBe{Vb(Hv07X=y<}@Oqe?= zWqh+#noVY9I>M@SA=(f=drqcG)V zbLhKFWfAarj1_x+h`);RcLYaW&X49Bpr4hClM6#e;iC*08KW?M$ZB^y7&D)4d)hG` z1z=GV`IG$Ff8~Uo59hzf2t#dFSRbS|s(s!pPrJn9ZN2{_b~>HSfTw*te7x%lDa?e# zlGNsH%bMFIqGG(gJ{{D2HV0WAO}qGbVf z@>*$FO>k6ze=M{Cs^ChoWRZ&$m{9Ojoy#OYxOGq_b(+h%T*F*{i9i`ReL?Opk{6N! z&tugts&jNKD|gvvliY6?by;4sW5HIgqQFNl#8Kh{>oggzy2@iMBAoBmszQ?Uf?jdL z%1ELQWQ}r=^wsU*mU~i(j}7}oj_D=;JTiNpGJuP@rr!SJZll$bB68W*=LM(f{bg@A z8XliFgoUxi`z@2l1y{iPHCW)a`x>p>%=HL%+TLfoA49vZ+vRK_pjK0D8ZHN`NL+=Uft<{O?%Ube zSOvlRr5JEffpj{XZ4A2DEQ`(#IQX-wg>89y28?Yru`sD4pFJAW`T{X7QOF zxrcGr^ry7BU+?jG+_PFPw0XVU^SD?IvU@-MNd^kpyf4~avv~%GgWqXMT7mn(V9@DP z;Q)jHg0qK3FdZ6#0-v#J{6MIgnKcrwl+Q7gaaGh2&*~yhuMk&lzi19u#?ALru%|Z6 z#rf*eTcc2FO3w`EI}mh!`j8dKCh*yrOGqEkH>gzqjYODQDY@vg<*U=Y!OTQjklPd@ zNXSCtdf(Q+>U!RPOi5>at=u(^Vw-a#6)egf>!2_3_zd}Yw$$!3Ji{gyo#~4#M%*eU zCIU~e;&Ux+D1!QI2Ojw0_V)RdaQ@-mnAlQZfBA)A5}hHsBC0$&KHYV5y?Nbb)GY}1 zFwaotV%5i}I}c5)Pm5PfH>}G2H+v<8bW5&kbyWfnFWV2OZDi5N=n%C@+unUVUC$@w zXq!w~GSY+0@xd1+@6(Vq7Sm4cPAua~UI5EinJ)$(i&28hk^V8U4w+o>_&w(k9)Ux*^?efqWd22sp3BJS z54~2K!)nwzCNC`FRKG1c9-GzokE>l#18}PHZ+b!|&--x!KVoS>zj2RbV#W-QaNVq| z5)@`lOx=sy$xSWSKSlF7Egs4TNQPe?*bU=qEbd__WsS zC&+Zxl&-;ME77cc99bQmZQGuxYpd3=ldkwhyiLPwsYPB*Pi`wER`~husCx^>lpy4a zGy^YBMV%{lbFsPP@HGz2Ql(OTQVBHZ6XnCZmj|%9jOMmoMO`ciCJW*@*-bWzx)o78 z^rMIHz8L(Hzy0@UEy=Y-uLAexrp;25=X|r(EcIQ%TCj2I`Y)#px80N$nKOy7NkgRA z>9T5$3GJHG`%;eDAeZTxc~cnZrSlLzp6igq)PXR%>N21?r5$_nkIHgnT~gpTKEUD&Jgo*<$2Zqx7#v zuQW2~hszIgIbF(19;N<4(Qww>Cfxn5wDOt2leOG7`y8kP$YgVsS8;(?MnVZE%33Xw zI$S9Th? z_84_+IypzRail^6&IiRvBvdX{Fd9B0D!uXB^r7^pA`%kPl3$=mztM8BYxULc;AARG zusCjt|MwwJnGP9Glipq&KQ2s%ZWQ<#^vCBa-OjA!TGwp;B|kN<2=Pc9K;};$wLnV3 zQty6aaGn&Jb3PR3Z*ul6J``=Oqf-N6woO}3K}6E34k zbn(gGjQe1Q1btP0N29vgH31dGp^q(f z^*1%23EUsIv1Iw?E8K4qX;eHvUN%j`CSW?>;H;-NKi(h2<0)nnUpGJA2t2*SqVeDM z)7c(2T?w?IJsp5Z7(jbI?8d;x#^7;D5L|Fy-aSmv_r=XlPKcH`)Gtx>$`6PKoKM%x9i$I2gs`cQDSeX;jlBzDu6% zeSZYB;%vFz)D(CNXDFJO#N5D;X_BS}9_BRBawtxsBO!4uM)3A3<@bN2BDPq8m)TZ$aJ<&pl&gz1TyGi?yMH!*!F* zW~6e?$aiM{2oew=r|>aN3o*gtY)12v@cf9+>%>aLOCo}kp>Hvc0iTQKyPkRZsbckWu1I z#^0#_iHfm_fB(MR z=DgTLR*tFP`f;uqzK4ifzEyxa{k^OntYnGuE0KaD+kRWIcZ%swWU_tCOnfo2Y$B>+ z1$F3s{H20qt6>fV1ek%}E)FhljcYjixPs#B{V89A<%9Qt4St6YHd$0`bZUeB&3$S; z`V>j0*KuDIo}qq+&D#QrSnBsoyWT^hM@AU?*igY&)6ipV&mfZ$B}|zVeN6dGp2m}f zLX^1E4M(#LRj1Ocr#!p?LSw}N7>B(f`EiR;8y2SAD(txCfoL#(RF5V1nu{AauCuhVM)JVV8#`bL5FXMrnJr_CxeG@VHYy zZN<8c;H_SW$?|@(QTx3l0`z-s+~;-R7gh(Fpkd(pIU2nNHySr>WJZS`H+o&SB}p-Y zBy0APt2IWkfb(Cm13H-~crfZ?Fi*!I&)Xw8{Fk>p7aj%2pB<+6D|v>UP7lSmBXK|; zpW#*y=+q{{9Z%?X=S?L(DG~aXD9K4A9nHfeR}2Kv($i;fyRXfq+ZN1Gxl$>LI}> zbUYXPR9m2*2qJlER~j4gBQZT)(l_>N1^*zaT4ROp_ms=g6g zQk}ILMhz4wcI!2;9oJ1S*>aUJ^-Fyf>Q+5>1=GhNK|x=PN3&NZ2to0eWL2Hirzg|C zRR*)$$^<#}L(?TT^W}g zrwZEoqgZ7we|3^Yl9${V$v`iF^hb)yK(%zXr%A4fU#B6@r}z_^(>Fs=rboHRJm5bj z%cOVA`i(T#pn%Tvhf?K=5}NWp&C&S0!DQlOZFnv60jrD5w?9_cyV3mbZpR5!8veTP z*NX)255ig@xO;*T7OxWm?@JHoDeUd$vu5+YUx*a#zjs)_Gy%z>{cO6ej0XSLdS-eZ z9e$HrOiC`qJmv5t*^g6G*f$!;zzF$KHP$n)S(A(Qhxfj8a>Sy|``O~G=fhrvX4}oI zv^0N}_X-Y2*1@>g+YILe^$bSlFQ!8=M7Hfu=#JRt7g!{P-QE}LA2GJ&DwXed(LQ%a zqfS%EotKx{Z>Offc3XwW@-OGnyufhV-*4q1b>350 zu3DkdYI{jtt|mw_G%oas&nrW4`)!wDV`Tf66NFT%KxjE{~Cu zk=9csnl7>Mye#u=n=9Vg+39HiO_ZA!l45_YH1o5z*EW?miUm_HxGg0v7!`FVZ2n}1 zr5@tpA~H5eJ2)v{3|3|RmMuKFBJ^Ym|4X;&*)KY*R4%%7u}#;5%CnWmB8zoSmp&l; zn@DR0%Sm9Ph?AG^TyJyU?9OjVIZyR=q_DqBfWtZrust4qlB~7nO!{mYyz;%ZBw60w zzo5=+y#U$_ZI8+=n*q*z>ar<)Q1Ok)TrK_f%G5Dcxuirn6pHlM&f%)VzA3-M<}JV1 zuUtY4;=Q)O(dbN0XKE(0D#KM`o%XyPZ4Y<@&bz~nr+X@ttN5Ngb*Eb(uE6hDZhwI{ zlzU3Jwh$*rU!kX=sK(hCy1gB7lRO%imIlO3v*5(i032mW%5|isbdyY+~&zm&-8V2m;b#p1IUpnT{i3;-q5UOGS zh)5T;-Cqb?Kb=;m+`ODU(>+3zn!G(bQIH(Jc6bF3GD;~Hp?kbMpRctG97y4NU$9qd zDiEl-P*#2|MDL^jtS`NlG*%W*Trdigkb<&WYB<;}9OU_aR2lXllV2}rG4edMaAJM)YB$K&g*Y~zZ zA>jTeQ8fj*&S^MqwpfwCZR+Ay`R*7(Fkj)2dCRrz?XR8f3*xWZ%*E%XohP2m$n zX>Gc|`xU~MFU8)sy9D)HmjV}UyQJ(!eU}|E78|cmjt3LnU|w6@kVw7^pL-Z5;xaO> zUp7CY`u#qhA3A}8fNr}onLmAHWtw+%TfuqG3I3yt<3F&r4HLojSHUI>|Az&TeD!Ss zJ{3p)tj=B5gmnhoT5MjY&*t(9-_xVWrlmEuTUg#QX)Lk?-(rtw8BIYe4a^Y>Y>kkx zDC3qq(Nqrf86Bm7IEUn{ISR918Bah^nA7G}?NpHbTdgrnW~hw4eM7g^{(uzDiXril z#&(|}2>$R4%gkZ@)O2#t%3|`B19la_Hdj$RU2F81&g*Z%bBl{hQ;>6me{N}+z%=aR z{djc*wC8&H*cqR0=PtPrOGOg%NjyD&xtws@T@t3b?GY*(7#JE7arJh2c=NMBrA|o{ z{2EHbX(bbcfs;*2)`Jw1GL(X8fm9vJ?PVRv7mMzO|M?k|1rIgoVZ%qDcP!$KL8Bq2 zNTG!ihJ?ImzC=F11&uNcI@fBIJt>%RY-p^D0Sul(HY*2aCd12sk)2dmU?3nrkCEg1=JZs$ zGIs?w+BF|p5s0|I3W$EUcupSJVq-AhL= zb=aCp=g z%b~GIt0lGqz(hP>m561EgWyg!dCh4*k-*wQiiDmh^J+umZv6OU3Vjk9A3C0&Z=22? zYw?+RXql3BZ@@l3FF#+c-G!M++~wq|y!oSRdM|<1Goo1v8`5+}2nP*~$$JgTbCg1c ztgb@IDdG5RR4iY0(k7FqIXN>h^r}2~IUqk`xhyz3OVD!eTO)B>sRHE5HZ!p$sNYu* z>2y&(r;i4C5=cWXMN>b6$@gBu(r7_t9VU$QfC*C-<`WaDC#W^vl7w;(H1#E%qsN#|FEH8kRK4BQqUt zIGdE>9z;wMbzetTr#qlLALU~@+AJ|0K11Fo9S83=HdLri(zkoK6gFK5pvt-^vE1(% z-1@mbb#1$KSbZ9G82s6%#M`B|7yx-6Tjq##Q)XPV_>)@1c}!G}SmlW%AoM`;X#t#Ax_wlxVk{Na|&QIX|h)jhh2ejnU0hEx1&HHXtx8r7F6|kjt%~jeuPop%P z9>%2L;$B`R<5}x<#xw(PkO=s%Yc~Qu$E8?6M)i3(f`TYVC!0T0^iEP*x?>kWb{nq3 z;&77|DNCvlpRP9(aFD3WnX#C9Uu`1rA1+j`I!^M7&6CU&uC*~FtcU%f`4kThKL#;n zbb0Xk<)O3lm;J6NdRM*OCt=kMLW@k_lj%Yo_a`vU4hL)trv}DmQ0^IT5Ab0(`?8pB zeAy*$%|xq>`0iHf{d>2I;WmtU_z)bMjqu~#9UG%dIVFT)3+en5Vf$UQ+r!U4_)x?R zN!VjqIbS(#lwk@sknV1GG^NSK>9tPf7$DiFX!6kVp622m-DDmZIa~N6aq$tnQ@n*q ze|t?S#9_>i9YaDEo0`z8H<*ndv*EDY@Y?Bh_OpFh@uQR+ZaE!JZKwXN_G&$_OWQW= zYiqk((4>&!60R?O|%&CRf-EKTqzZ% zXvoq1B3!_inUs{6C?W+A;Pr)Qs!(ICS+F8)mg~q?lPSoGL%O0~zdR9t`j|Re;0_83 z@d#UMMMOD3?48N>x?f%xPoLb|XZ{Fd@~$|n)Ko)JK#*SE{fXu4F!7Tbd%I+` zPXkqimlVYyam4Ru0XRx%!pC-z2OfIreuTLOx#PDZK1l~MWIW(3<87h>#q{)aiU?27 zRAs2C)BElr4m?Zl#zG4$Y!@V7Iqo=>ILXi{*TL;BCD!NhTd^asdT=Ph*@uF;BLkA? z-8>HJOvhoNA4SK=Tp>y`B;+2G88p=9XvjhjtW;Fs@MR@leimdxFDI_Ruh9Nti=7e6+ zwE`n<_u;1-YW0vpHwgo~9I;VNUkwd3O8cOCafJ3)0KbwUImZFk>ttm4wN&Zuxb`yI z^N+)KdSP5p+K(>;?)i;YDxs3{iec1QzB1$0`D^Z{gozYwJhWi_)LU=6NxHX@W@{mA zwooKs6%l0w5rgvxjW96C%EqVuDj1yk z3~5!Le>M1ou3z*uG^5Uhnb6l--j1$rU6P6gUwPhQ=)K|2bGahynx02@fA=CFJ<5=s zN;ehlXg6SiHn-2XMSs=>B7KQ0(ZO^zksK*9{b%qnrFdydBkhlG-$6ayQd+5HtO8m0 z$u7VR3EHSo>C+M97nWO1Yd$8>P?-r*s(;E!FeUXyCKHp8pma9Zfpu#i1DebF1Nvu| zU6l@G;82z~~sCvq1)c4Hwx433vA=R1_)7Z?H0N~o%HE0mX-?tH(|RY#P#QiaR_Xlq zWg?MUG;!-hi)N&Xd}t5cm8w^yZkgtwpLvc}UE0iP`^{l++)Sx-!CfukTrN;VT195E zMSEAoTV?^Sd<8~6cCvi0kD;ZsgZKRpza7PmX*GN zj=TNzaCAD2+^S~ajF4!b=Uw19yQOLc84kI;!+l*=hi@T*arv*51j%cMk~CFurF~17 z{dFmFk-F4iDcQYKv~$~$y<~9FVDzboA@Y3=s5?tG@_hb*0f{rW{-3y26j-LC=DfWP zZ7^3`?0568u4qYKXC)a#*do0TGF_K&6Ysry`a;7r`9S27!5{wa-leGU-o;2A6;2_x zURqCiC_(tg;jf`;jGP<>{q;3P@v!+3KivBvx^E7t2tJ{ z>~dGYK5pMs}pQ;3o?phYoH~M;Yu~LmR2&ZVIC&_ES6pM(Kx37!e`zTw<`U72B({ zITlq4;x`73rXXJfN`sL?9b7A0?sF#&5XBjqFCxraB_*%$sfk_0bkTR!2E+}=Gv;I^ zk3*frDRC0wB?doXno#un1BGEGg4wOQ?j=%aR$WeyHY!;hNvQe~Ef?xxjCbvpv6MM` z8}d|(9bC6W8+KK$X65wW3_!v(m9sSJ!n80hk=Q(qJeFo0pl@|;-&IK!Fuov)LS%S) zG1obuLK1kMM%3f-#9khEZYr*DD)}Sd+U?ieAeN3Q6~_2dQbT>eZ^On7R_x>#7L1sv zv8DTsD3AO0a%V(AlyuL;gY6*ol{cO?!fLbX?hvU?C=T!I(O|s=Qiw{`v&-;xHJU&y zvhvoKn}`T2YCa|@&!Lf#LJ3bMQ>35*<%PUPEq-$>b+W<(CZYtry8lDmI|gRdaa;JX1ImG%6@NqxqZ;GIG2i|;+|75c6GiE zMfbwT*E@qiF@sR>V+4lVPAYgHe0pla^LMFW@x7k+%@e=p7(}fv)Zg9t` zOq&d98IOK4dsiS@9Twm7yr>s33chHAl&z3xwUx+?oO5aL&24__S$QDNGIe|mXu zFg?{p^HBho)RPy}0`m-OWJvM=Y0_}}^4LtA3JUToF`nwhG-KHztx z;adQ?MD#|Z+3f38Z%Y=iV7Jpmz6o`rGVs`U-7=B+(LOVebkD^0m ziL;6_TSd_cao$>7SPYi@eZt2$7;O-xvNGdPxLQ=cqQ2m`)2h^3P#f#;R^bcM2

1 zKI18gu&vo>R;|;VU|~8w-r$}vMql)No1%`AFQ1YEyPQ~vA`~e)Q&rr9VkJ8Of3*yb{+KEgg;( z=Im9m5>OI4f0(=R^sOQgc$%N1m>_wZY($?FS+PmKE!$-0&q+=ed`Uf}@cS)KzWwa7 zPe-I5ZRuE~#)~HH$h{Hx8BY0WZ@9e5JNnB2o;0eWNkRNhj;sgP@hq;JBO=jS>L5a0 zrwSvnmV??}s;FEV3;|h?B#oDOcsPv+BrQcePQLwdWf_UR4%wFw1f&>RTU9LOH3eKYixs zkK*S@mzVmn-Iv!iFr_#M-E6r&bqhyE$-?B=KP4o(hdlYSEoAnl}*I0h^k(j2@b2_|9k+ zLyDEoJ)a~p*M$C*a{mh0jfD7i$^Anzh_O&@HpXlinWkNr^;WI`6>f#8TO02=7w z1T>oQJ~R^?G=?-F@J|wun%R>@f=%~4TS69myXx`ra+=+*ms-B^{nN?XZno+Bnypz0SveVL-?c_?mmrCQDYvgz50I2~>A2@Smo z!r8;>R-Og<_?7R|z@N9-N$I7#XBuIrjkfwJp_l!|Ss15Pu?50R zpEgBxuch0Ibt7u1cNZ;KvYiJ+z@_u4U0kvbYk<&brTp)l$&d!4Hz_?>aB@iK&@~qYKntuR;^46ACs)=na*6}=u1`O z>&uOGm(tJ}Td9UdLqj|Eb@1)3+jW4fg_BiL*ST3194aKSWozNj;&o$?A0AKqon9FoLpDJeZ06Kg)wDBiB zGhb90LQ7&yTu^5aF30D*y17v2diO7f;SzRHF~pjjMK&}Gj+$j?E&3En%v8|B+At>T zN@8$?U+ZIu=fgKC9vaKq(#CzOKG3MZOuHK)Lu5UNK=wDdxq{gc^5@b8$7?Fqz|RH8 zw0Lgbr95DW862VEToH>iC(&E*PqtcL+eJ8Vk~|uhIQ#mkgxO7(X)N-0mBd89>(!AV zRF8r1cpB!b&E2F3<&mNcQ&zQnPH+=;mVkP|GB@apkaULwm^mM+snPEGQ%u}(xC5V~ zO5R#*7A@Dla-5ySuF5Z-OhDIuKZU6A~`t0lu&6Eb6(mcw6MFzS{Z5(TvQQ$^=A`ljz-b^{W^$m zs$V|G2O_GlJCcchMF|pU|H=dz4?i`0S{tUfD#%bH`m@CbB^n-y3zmC0xY!v;Ha(ng z;;u4;xbLq;wAg4Pn>pn11QJN!3PDjJk_iq^d{JI=Y$6|JaXmvDHNauPOyCiRxFoGh`7N z-#fn1y}F)~wOw1K3Y+EA`txr}sYFHlj};T6mpKPeu*q}QD!mk(lNXT>T5|kq>{&^?wr*N=}=Gh5Ua$TL+ zG3RPIojX!oPSBOgXZcq3*vxLnj`1Zg7S5@8d2m(S8q3d@-DQumzW@+nKp*DtD54Yc zo+I|3Mqesf#X&YMG}U$xMIHsPL-!m`fGylR>rZ60-%Hs%~Z&$!CR z(+-G`ma^8C!``TajCv@$X{oX@cVwJ4PkB?wl);qXq=7B|npr}#Qq!e#!eyE(@25Wr z9F)WLDhg{3s%3eDG$Mg&T@z|#6nWu<1`(Yke7_i3P8TR^yjpG9epag0>a<(3LqXlF z1(@GFYTH$iYWM!M?fe!ToIKnSSZ(UDqmB~i%NHiaM4Hk}o9n~j^mBziEveCL+Vg%s z?4)Uae-Kd~&)B!xKelO0Ehw6TC;7TKihxM`Pv(rR(!@z9VqkybCYRHWJQYPlKU6l~ zuZUo@C{#BS&(Ny${x&k^q^d+?QUXI(4_WIt$K20UQc%#$0WtI^qM~>xJW(I!K~$9<5Q!CJ3%q0dMLDuOF~upzx^ff7=8B?Y6+DbO}-uRMOq zNeJGxT%m)@)zIBs4+`63HfNWZLioCU%uR>I5oQcUc$p30JaEqYe1d~D$m`&dkU*}d zrmUGChcL-Zkr@h)%Op?OiMSA$%48io8pdX4((ih-W-ep3B@>WyRn zyikhMO7?*vhSrgV7EqA3@+~lbv80q5F7_j|q&;v!K+?c?^ZYfUU^ab}*R}QM@g!B( z`_k{vvr`Kg9!ni^Ei7a6aREs~q1>9U6>1zdB@Px^LT0Mp_BJY3MLx{H&1B1m5GfSe zAYu=T%6x-ZBh;-!nGiyasVU(e8U;3)=1G{x_#=~b>cpxjst9hM zxWJ!xiF{P&-E_qWhAs^H46(bF^KJiWr(pyyW`#6uSCHK=Kr?`}@8t1lL%Jrz zbYX}E1BW)e{kWl@gDA{rkh=9|v=b5s&#fdBuNfJ^YUy1`tdQ0xp^XqA)%+UcEjkcH z&0Qc=Rb4l7xYD1eTeZ1^;#WZ#cyyVQ@IRM>ugh}H@cyh5M!9dWZmv4Fe{(3|*%R6e zl0}qI5$DMeXFvJTPEN2U@A`9wKOYglfe?c-#H8{dsG8xcZ#77~>ae;P6lc5p`E$Jg zI~u#mAw@EngYhN5Y6#l=SrZ^UEE0dCYJgBycTHf{-Ef_3>2DBfso!aH#&J$7A8$}$SAJ1_0}0*U!F-kv*>(167b~Br zn}E|l5?}L`5jk6dJC^)=3X)V(rSa-|G9EiAE_>X!ZS!EP2#f6eThHUCsAP|+K_C9* z%UHsGHo+H6Fw}GkfP^-`x-6kQ zVct)8U9U$%oqEq|B6w(IS%-6idXSs=B*)*HFtnD(!49$OC-<@(X*=I*{=6u0Uw4$X zm1!=H34?nc9AN8H+iY|NRbOcKj^+pIP3ewh)D%yy>|^hXQbXD^S75Q=iP|>Dy-E92 zkfdmrD`Dr*fZd?ATF)m5F#8(cyd| zG>fKf*gudgB=JkYgT~@x;J|#y0xs53{da7)!RMqb9wx6#!_OxxRHa3Z1Dgu~b-LLd z^03Cbvam(Vz=>R#oS*gkwAVFWHX|r@iwr0gow&EJ;h%{n6+pXrj(|v3-nH)W4 zzcw^9XsV6!Qn8nU2_e=lR)?nk2`9_Y!H2K)z&aZbSSq1*7X?*mN=mlb z!*=BImXcsnS#9&p;T0YlviX_adSd18V zX;p)y9L99-MMMCZ!5S(;KY(QLQ0Uv6XXd1=o6@@VzMuUhc!0g^F5T5FUC66O`>ENR z+Ygqv_IBl3Z91bf*y9I+fjwH`oKaAv%SzWCfSoq}qbh3^t8_j`niA3GEnbFo-L!TZ z*s`T1=BH^~Q0RH$-1jlr8s0Hq-$ofw4^ek`79$R^XEwLL*5vm22dlp zo@DWPpp8oE`x~H$FqvI;0leZZ6xmEyXf@lTne4V}l#jQg*}HsFSb$Mr*H1b23Xt;^j3lH)QEUtlgYAy>8yb%0E2Ty5xSP>WRri?$=&4pz z>}IF^9KV<7t3Xx*!Q)L|yhtW@BCp_Az8a4f@9@4T^VHb=2QlnryXvW!0b7tPkW95} z2!V50a*+qB_)F~M0QD;&qD0;;Jwj1GX};seyjj7EYOm=MK+m9ZDlQ_WE5)3h8UOgX_3=B` z?ayL`Nu^T=WQO%|oq^k?2exvTIZLxygnYgdX!-IPV`p+CMV5{N!aJ^SNGJ~0m=v@G z#E4fOxkbB}x7Y{^uGSHAp+xuXYVU{h2*S$mEuW`vHa}YqaQd1SOY={+{Q3Y4i2~0S z9>ZagMe@t?b2tJEvD6P2XpHSa>5BNmwaYB@G9te>FFM2gMG)K>$daEf$7?t7VZ;XQ z@ciKUs_rx~5Kcu$4Ysgn3l0F6ra6%siU)Htj!U@{)-2HB3;U5dv0%La7AAN!zrIk; z-k5gtb`t_mtT`#~0TXnB4bCB*MuX=C6v$|5I3uW&$@y{qvr`|jOo@f_{dGP#7p3>? zhv$*`b}K(e%OB9WQ>K}6yAFEy$D6Qx%JQZ4gk!-s z52yelN#xuW;qub%II>OKf+SkV5a^EkN?~6*r z&MVvD5r#Bqjg5^_8QI~?)I&@#mznmfffkNPbbKz;895~OFPadZ{#mpM_q&Kb)c{N} zeOHA&-H3uKjJq_M$KYqTkgZ3jwF(2ok0B^Qer=~aq|$Mda{va-c` zORklLl@}kr$fL1LIbxipPn7iX62Mu=q*usIKULL%zN5W*VTQp1_*Inm4rx{ zE$1koN_Czo-mjfq%vVhT_Dj9JWbt!bO=W_R51CM>Z_saokufHS(8X6`-+*ypEw#h} z2z)3XNFw$G@LTlJ7%3?!=xd;H5*j;oO4XEg2~VSIsuOvdZ7B_fvN(9DEZ%H=+$%Dn zoRr+rKAM2s+L8V++44C0h*FYJ>H4kQ@m$692t6-{-Omsy5H&7}_&8C*wa+0}2u#Yg zRI$X}rWc_ml`A&uJ_f z1ZEa~CWN%ULZ?1+Y? zEqnZVe@ZN{O06M4?I&&W19D3#^xB3t7aQ$CwakmcelL}J6Ip*idbi#lTVJ29y!IYe zIc+vKfKEh>9v%Sv)byeMp7#n*z#F1b-0Si9V^Z%OfKw=ZJdX;Fd_jOkd_O#^ttayP z6DIh0K9A0fGLnGg-0mC%7M(N@A|YVyJf$|;PS98+*AM@l##+RX4H|5$Jd>h_1*^DI zNX&~=cE346uVxZ*D@SMV3X${0RRv}*G!p=i(YPxdiihH<=^PAwSKFEaGI9p>d-|iL zD-&abvfZ^Vu5!QZ|HHu`0zF}$8wEg03@3OekLu7mzu?lprldA*;e}l+d~OKKxfe7B zn!xBMI#&@s4wq;uPf=!YFqgQWjK_AI zto|`k_iv}Z74tEx9Q_{xms4%fxkh&0+83HnsPQm{@8O#_Aos zex}_xfK>A|wS*vyK?!a|6iYZZH%(j^9-cn_Szsb#NaE&BKQVbC;J8iNzG#63xi7c; z>(-Gjo%n4#jF89K{iHb6^=iZA?b*T6ZnF!rX0kNb0}ImP7Sr+|UJ z6|Rbo{-dL0*I3UQ$HI+67BNTYDLyJM*{`8}OY(c%Aq2!@uk<6)3@ zj{_*pc*~t#Rp*TU^qKo-{7^)^wN#0YPs%z*FQQ#Y*P!yy2hixPI%zTO%ilW%W-IbfP@{8fo`& z;X_!)xJW*pDh1zT5u81G|8#pfw4ZNw6Mw7AIIe4`2G~wvu2(-`gHbmdE9B4Uc|{%B za6h4=mE6uHua07pB~76tA!5>g^)!D*t5Xpr_Ij={q3v03ej?FAkBaAWd=}ClLKzMu zvx544^CIxpm+f@KXwX9jeV?!UlY{=#CKnVZ6&4o~?0xc+;tQ^AI$w$pmwK({tO!e1 zrSXWYim8z!8+Qq6;#Qsh@w|}*EqBq`_djqqUzF7x9VtXK%iYz=p9P#%9k`PM(H0Uw z6Y4r)nL);iINS2W(a`3=m|*pbCz3J{_&rn5!ASxF8Y0oZlO=L6ho%3)ermKd_6F&_ug8*IWCn(k|Tp{%l0=(S;l{EPcfC1zHw zzeiApN_$N{*f}x=(eGz+w|aw<*zDK5U}Q{|8qv-hBMIk|4c%_J zTOxlxh9P1BhpmMJtMt^&l#~diwN3Yhx?`pgNrY$p{xr5c2ey zPILr2dXdB<-fUWPAjx!2U%7z6fspH0d_&2(iia?xhmtB6c%o)4!j^q3q^5pRnPbe7QXLYYyP(fu{~_pZ0w z-1W_rgZSs)fSSqe>6YHFzDovtj=mU$`*&o%zM5~cCt35V9nm#Vz8Cm#?8p88}ajGcx z8;S9-EI*Mp>d%M8R$(<8W|*OUwQ6y|8%=)znU9sV`3onJux2V|2g3<{wy+mOLJj=0Q{HGp)`-D2E_5OSyKHkUSEDp05 zcBC~RMIv>$nwQ0v>_@1zxOc%VG}OmMj`Lz0$F18oL=fYUB!7c9$Y&^FX;?bKFdNL} z+pIH?9E~MBfvwjJocXp$lCP2PJbj|0wk;LwoRj22^UalFVEb+UyYKTMzMXyS^`DOe z#{JKsF}PG3a5vQqsIkm&(M{5s#;s?~O!;PhuP3i5^oq);lg#iU-$A^m3Dbka zSYy}MP56n{U1nLFCareMUuKQHhVWatZ`-6S18~#R+=KHaNsJj9A|kTOWK>*C*WJiw zKpMnmEe;gt;>8ln4RSK$4O_@@c7L(zi+EX(cg1#${z6F^VKovlPxM0e?(RLk4YbJ@ z9w>}w$g<~WqdDL-nWLST$72B&1#3e_uc%QdUT>qtJ1;bDhWL|3Kp+RJ@!RX;K3VnV zFnf&3?*d^`J8FLI#xe5gFc6mTWhQmU7Q3TbL`X55L(#8Ek3ZmagohZY$#+etjHb@nVDONl%S{f`4e^SUL}tPdFGrH-z2#o~ zXe~DR#Hgdoqm-u6}Vw*kZsO}2g5E?E~j5$%#7GwIFy(`F&-@B77B=hHRemL!;4 zGep8DBD)MOgP)H!Svs<_xdiEq&dR@%9Sd4K&F&yrRna)8PpD4XVhQVf^( zLuMo3NRv$xLv5$f8<+q6>YSf87e3YSM1e;uXOsTh{*AM2A%-fZ0xChL3gK`8PCBj$>HG3 z>9VB>g+!F<)W$-hP7%F04COIOyQ7fWv_&JBp3 z2335r?ku2i69VNwjwnmWMf(zo*3u(aNG6>5JA?racV4|XMD61sl!&JWr^+_2pyF)8 zx?d6`M@tfHtuF>2UI}Lg151;3nL>Fu#ZvSEJh2apK7;y#8d7nNtP&)>HjBcno{2$C zb^UM;~2-!+t1XF z9iLr+3y$;vUd-F?zz9ev=m(q3`ERaqH;kw^E&S%0tMQf*iOCE+STN2grNN7zo)#PN zY>N{rZIl11w3TGJU)E=oQ5@YrZqw;{0i)511qwP15!E3vOQbS3!FEl7Ho@?XCn#$s zL|cL=`O7}!d^!*Rytj@(1xGP`z6zWw5w?(;M>*F|{2d=$0K}+tDVb*({wnLCm0b20 znw>?_%8D(;kFqSvawXP0c^H}raAGx=A4dNhd6fDWdDLeJkPf+YJ}~JsoE7&6r@te$ zwW&g2?@|`kL@9qWr@Xky&@ijGQ}Q9xGE}>cXcTL$#!|qUEKT=&&Ry-UiX;3|ta+sN zBuX$#g3cK;_md*^C!BEAGaQRp)g3vEx>ThI4JN6K<3rM+o_IzCHZ@`UE{=0NE`m$Kl{lJoY;)WNM(k#{}w<<}xnK>2=L!S}$rQrQ`dnuC#6| z)zW3-Fp=VpHG{)=H~uo@BZCuE?%piO7_%j=@H zN%qr>>GYK)z_(YD6-MsTCK4L@c&|N#*;sQygRU|!o!=+K#Wi@GAwh{78fiPIQK0)D zP3gJ5?Neik?`kg8rim!+!0H^y5O-b0k_M3(dsKmtrAIDTGAaI!B;nDGqG;9Pc#)XG zQC&g1$dKjFPl&3ni3&}FH|dYr3(Q>-o$-mcXwt|JPf5#r7k7qstjsr3y+R@K0mOpXg9}uTQQ@I1zDduSOY1LX&d)?crdpTTa`JsW%{7-NDn%daFgWQzP_5n4B``N zM6Qf3Sx^UE=3M0b%}TY{*Yr{`*<5 zW|p%5pfAg0w2f0 z5dbeU)WpPOI#ckec~y6EY7B?VevbcXlQ!Fw8oQlN?w9#KspfKIr7f{05TwWvR9zQ5 zQUw{}rHHP>ma97mkXY#R8xfc~(=yeKIS87pGw^#KwKEYYQ05uMTorFC+Vgwapi}ot z;C?6%o7fMAx-L(t&LGJ#(gzWuaN>DLs%bi)8G=2WEN075uNRq}MB1%X{R*CmBrUC+ zKM>felt|5_iZ-AFmno1t!vactlhM7uXM|klL*_Awh)&CNl#Jo{mnrWEawC6y9(I<( z>Cc)JFF~NB;lgU=+g4Z`6U2^J9dlKbi^I;RVZndSsNMA*N0R_opdEqqVO$xLT7qW` z6r(Z;7^auk4_m2lejb6XkExoP6^u*HwPqLwL18rlh_DTWvAMmngIMlgR9+|}w67V1 zWVxtk$ihakMcPWcDL}^=rJSU|o>+9+OQk(jTU8F4Cq$@SaJN^j6{R19z})PL!(7@% z#eSmYRC_R1829>i*=UG&hrVovfVo6kl$LzhdVOKq>CmBaF?y<>%)ELXdthvwzU% zDMSuKRh@b0ntIWPPm=2iVzMta>sMOc+~mP9K1C-?BSvpV!O`V>z7&VzTGCfs;g8`P zDy11_zOz5L5_G>`yGf)|HN=)nxnxJM!6Cw1X?F^9wb6DGhpAytSQK;9OIjoAC^{!w z{|oncMRQw_lsfLb+Kxy6gx28}CF;m0n7k#>iQu}P)wceZNF!=%Yh0$Jf~EcYwcd`2 z5@)eN^D5`p^Wiqg5~T|%W^HdTF?8bjx>uZ%alTQh{6(7C^-`M0B?O?jOq`}8}HUph(I)>Gv8`pkY0BBVT^{_Du;G(wuD z5#;j`c_Vhr!R_9(9^nZg$pC~88R6SRF*Z?hny)XVP2;Jtphwo9a-y;(qTmV z16g=M4%q$l#pGPSSf8(y3kd;rkTCI&KP}@_)g9Isz*l2*V=cykUG%Bo2fNGQbPR&4 zch@H;@}RH+PszjXvgR@_Own18M7&CjBt?V|baj2P>E0h0wJ6J^`GGR4_VLkgQXO^_&oz^a`pJS*cxicr(!BVKl`hC>!X}d}4zwIvo+5&tXBE}2GD*YQoSN!RT zfv$(aR>BT9S!1TVhUf{TcgJZc#T-d))=zfh`l%OqO8)>0yH+9_E-m&ARBOE=@neMg zrla=w1#Hv2bkFn$Sj1z?joB^Uf1mRV>54gqLK94dd_$s>C8g$54<_fODDu?@8AU!V z=$sSkq?W##3|E@Nl=}0-4tB8Du`5&dSGeWTA%8kP?)l%;X?GbB4FDt}qS1X~>(Iq$ z?Rfrd%@E{I2x2k|dzk>BZ(^V-_B;l|LYe?peo zK&&7N)N3?ebydWxC29VvmR8plXj+|TDN>SW#2Vha0&Z0po8TaErtWcq1;0%5> z9Rs*2cO6m|`cRp}|HQmH#SJg-E=F>%{I;FH)0fhQkIRR#EZU&8c-*1db%|+GZE^o= z$$;4P9`^b!ksKC9$hd4EXqenu?7hi!+VL+qO9oyynsm9g8W^~rOtNIRp%G&%k}JKToNYIj`cwnrB2={z?;7gaM*ODE2`u zPtR=zmj<}1RID-RX*%Y5v?MjIifT;u+i-SQ{RNz?^8ZDe#88i&0dQQ>zEn+J2j2tqi|6Kl*p>%l{~p+UqX;l#mi|?iDpqRb z$nDGi;v5?r_D(?+|3n>eD5CZBHM`14x4{?+)#3(VzdRI&>V#_+2Q>VMc8%D8)M(h` zAX;rROT*+Lm9lb@x}V~T7Vdm^ojSf(a?5CExjefJFedyM=YG|~A*<@>^M>~Edpt(z zP(76+T`O~6L@*8r23eGcr^aP;f}JJYPi}3C^g1;t=`}U~GL@Gn0Q0Fk=^( zpn1t&7*_%qFQshDlqn38X1&dhr;c^2!pum;Ck#%!Zzr?yK(sQbzKi>NHLBqGv)XV0hsQeqpHq;R~pVKBZ3%78M;BFT>#HId63jmcR)e3^9E07ekB+rTk-3stw zoWENR=synn*QIrwVA2LDYTC&vZ31qdZp@IV}CMN)%+#Dw`F9ap}X+YLE&V!S^dqr28lO+$_P$=(T z!MJlr%?fYb#3`{HsD)*C{xwcyv4^;+f1%SMz-;Fp{|=X*j{NU(EUU=FO(I5@e6Ec7 z4v0=xxd2JtZkB&f1h4_f)06`CVGh_w-T;QRd2AZCzxgSx8@lXktwA}IfJ^OsnHQd7 zRmY5a&C4U&KYLS+b3`U%>5HQ!M3!aXV)e^RTkR*J-90E(WWn3NPiH)O$p33}X23)H zesGLz)d!XRf3L8O(Z5z$Mn=Zu%XpTc=Fh&&xBtTq{+fk43aAmBuCV zYJo#v{_B5#4f&rx13F^;nIX?SVe#Mp_dq~{geW1RsWo8H&<+0QhX4NhQy$=*s?I2_ z?^*w!Uh?q|dn7{9`$_|$12YCcHi;0zrk;};K*YiW;dASlBWgC!2{tAF; z`uxwv1Uh0>s$CD5VE@yP;BNX=U1}>t(F}R(vuTRly`UV z@H3d4aPSsi1&=Eh?k8Z4Wcz8us&}ma_ewAEfjxuIW{r;$;}8-3AmLhjZO>-n^Xn0I zwbhlkC6X863ub4$Uml&e#&>#1mim<(j=Rxis`|}E-L5eG|6F__IkK-RDhYb_HHsRd zI_@ciW0w7=@1(eukl#u5y4ths@Wl|w^ zHSD}ShV0|3*DU{Sa(SV*0Xr3^ruzrj#~%5Qw+@TQ3)#go728Kg*b8-_Q}9udK|x>_ zQ)k9&X@M>c8nsHra5I<(M@PI@mqnL8XD=ME$a@d3Gh)5Jrzo}6 zywzdG!t3o^)aR-5Yg=-leDVu+6G9Qwu(iI|ecJU9DTEJ?F>;*Q&o)EP$P)_La8tEy zj<02gv;sFtgoe2up`s{0Ji^v!Nr8;=jPaQ{#BQ=(-~JL%xQ7oa5u1_lP`dy)DA zQS|-&9pG%vH(RaBXRw_B%pyp$T|nGPH~MH-_4^-_;@zRVgx!R|fq{humgXE@rq7?_ z`b?OahyJqy&+;g97hK)hRarXn{qV3BH~@9RRhn;Akw zE;S`5Juz{#&TDVPU_KtTzdv>7WP#;VX;HOiD|?C9Vtpa>8IBhgkE|O5KY8W8cRA6b zM3s==pSihE#8YT>s+t3w-^ji_J&lJ9u`x02J`=$eDrF|+H3{d@)b7a`Ll0!3y}f$= zf5e(&Vd1?6v^Y}N^FN^_DC9`n5-tXQyt}c(-SvSZS82C0kdYYzFik)!cz9)DQLX#U zJ}Ei*2D3(I3IJo5EHT&ie*9*gKpVf;T%8=xgB^hDOs;Hd7~O-;+0 z8d_zcG9Hs}I{*;I{ppvRwtw4^o{GvBEprVQ3q?bu_$)ozkx73n zyO}I*pyfgD`{Pf$_5&9d$Gb}rRj4lOj{4}aKW3=^_lGWzJAs$|M73%y zg9p?>!-qLu3_l7fisV5o(DuD&n2q24d95Rc7cD_~=Fkn!4&&4(Hly*STKMlFC`cbq zZIeu>R)D5}SuZImscEQav7VW}>%d&5R!|5tm)=7kSv#12x!A6L^x5;ad3a|e`mL^x z=@V;^1P+a>T1KgS*`5o6noOa1Ow~S5f+GL!;Naj(GsGOIoW&pWQwl)--s7_Uh|Z9( z+-^K+a-35VfekC2-9AQne&~jzM7)0*?E5arFm}7n86}zzaB>oY5SE=T?OL0 z?X{@faIG=HMv8_R8Bma~EgQqGy?wdENMVMg9kej~w+(Oe3JFbn-*T&KG#N}Xoc9Uv zcOsmxf*VC6DKsMrYnhc;K|w2Y0c{=`4@8E6@D0znmM2}X< zM@L#Gc=7~YFn9NhxXA;(9`+O{2D-dbk@_+31jiv?6O-TO=7cS=@f)qK3`Q)|2lnJ* zmnEY>iW+_kGH{!V=?HZAl4bMet$sfgU#x@x?YPY+MNCjyNK|Gm>aEq)*8SDRw{Erc z&g-p7W3E;z&&*Hu0iE`t17voUDQUQ@5F+!2wZ6Ep8%=nY80UHq$wb7JUl$iN4B-L zww9L{WxAP2-9bS`WeJHIw9m}U?3xru`sovNVUx$*A&+j5Yyu!Ste?bTwG#b2H6@!u z9EiD#Dr8MU#yd1$Xb^(m93 zIT90+$+C~MYh5>x> zsMXH58hQdg_v<)P(YmDba+pT-X#i9n2img1LX3$Yoh`zSn}Upw9jf3_eP6N8knnJd(^8&>swL7mp>dV6F$l z$<-*z@b{-oqdErmpoLo*TmdpVYy2rCS?kz>6WN zne)7!R=eq5MRTUx`H$TA^l!Dm=K_{h#`9k3rstn1O4^l^ zpFk&;tv~LU_*XLA7g%-RAl`wcVD)&3 zT%(lw6~U$}T$yPJwuK9Zh_`e!o8zOja1wn^z+YhWy;e8v39}PEGv=4xw*Bfiw};DL znJ%8^sM1)^H`i*Yh3V*n=2-VdWd4j%O!BNbhPMVf20kw5YpBS&N=j%}Z}!2 ze)b1JKVAGBRiouIsl?F$j)_SB7jJJFR7duG4dd?auE8O=hu{}TAb4;O79_Yk!6mqB zaCf(hySuwffB+#+XJ#@pzxjWC>#e#)-KwVg_K`m4?6ucki+3BS_}&bXD3uJuqv;P0 zf(+V64Z|eoLGCczI&MF_01vCHj1SwL5Ir2v|v^gqFBab8Nc6RMAa$u+#fU z5g56gf}_ZlMgS|Y3*K%>UTS3I!H|Fir`w%?PeL66WL1~OR6K;6#}ZaB;L#9Wj4Vzt zO-eV$G3L#}Pe6OR4e&i6hSHbpfzR9r-hRNK&F?hAAk=T6LD1>M<@x*mVlbR!^u*AZ zEP;}ZY?xFnB+AQT&p{Z-KNA(dg5`~)*J9i=+|%zSUph>Xcj#3trzC%w&JwT8U?jrD z6mfi$$-}{trlpb2?Rvi8dEw}$_S%IyG7utv_(Nb|sMYnp`^CVeE$ zXl``hVZ3Fe+RcXgtdB4GrnmcADJ~dF*~1ZIve3uz6_&EEm>c?Gs`cX9@K1+*8eV5J zQ~H{gnij5_bxV@5H21Odr?LIBH203X+QGS3oU1M8@ZOq}K`7rJ9A_Wq*WG@0ls_F+ zX8UUha)U1)(wjFfy3rFl9-p_4%YSG+cM8qq7QMR2f2|-V+x4|6K1K(V2tfsMTKU3@ z@!dBLxESjjY04w7`yTQsUK#Y|PxCr=>E|B%fa2TW*cjl^l(?`kVUssO5I;swbp%+9 zEc?MCmpAPZ1^L9@ZH2K(;p7(-#3m^U&8cK>pCxuY(Y}beOp{@Qp$`iLB=-az*C{_g z-xP{+e8i!xADxZ>YmO-6B@mXl&Z?@j=(RtsIE?=|we^+{{mN&B#V!CpEW;?*Yv3+I zxyb+2ZE(Us>JO+PR7_fF=ofYo+cZLAKDMm?i0qReHCJAt6U@}bOS^n`0T`EP1+Lp1kbq^i_BX&_&#JWLG1y8=|zb{HD_ELR_P_Hz-uv>UmuG1h%Re& zPJHXxD!RGaS}EE|=lxSzg_%OBp{}J>K_kmeh0$fy8V zN7K3q6v<%F+MZF1KVms-7YN#QZAHDeE05b7<@>P0Kt`A5>@c3^W!myF*C{4V+sV=r zS;v8x#}n-FO3kfi3SL#K0gLiT?B(WH4*A52r)z4Z)r+zUbW&vq45Rp;vHF8kQMyl$ zC1x33hEi@s<7s1BjwL4uBg~8hzW6LeI!@0B>6>Tsv$a8AaiDI>blgwtLa3Y!#olN4 z>Uw=QG2OC5Dl_E8E_ieO#ikQ5_sU)g$(tV|^BUBonE-s!q97y)QE(7()jzdAm=5vU(R53+k=>BZWqMH_)D!W{#8R>KjXsqwzkm)uk>qP> zslhF5MB`-_3IgJ$Jd7UaXG};;4B+D{Lf`q}!+x4b6nDL*NPObJeO&sNn3UM7C2iyv zC6>3iU&0ApKPE%iwC@GAxUKG)kXK~aoZTm@2(JdEPx~%pXL&w`-QnQ5T7i>VSlG-p z?YiK$WcLJ)XPQD~LuEwV5W>QS7py)`#zrZ}#U(W^APY=vhwr{qGX*=_HXeDCxm|Sy zXSO~QC!&*sFPKPRWJciS)r9W0I>v?fXRgOZW}VvC;n4Ityu!qAqimp_PKqen{1`~U zv^hfrMzZjUhbt0O82TnSaU=M!O-DyZBbx6rzxfSMmzXZ8X-QFEPk1?IRq_4+)UJGB zx)n7w2{Vmp1e*|4Fd!Y0T!bhyl+d<3ACDIl$Vg2xY3+RceG3l6xZEd_aynvU@G=Ze z6CemZl?d7v0-@l&@k{2%5AW2~r!|M@r+*TB`&@F;KhVrgqyL6sF=oXv ze@a0Pii*F|6spTZ;GB@T4vaO+r}kbMd{jLmv5{}u)hXnrF`6o(du8Wn_QHdml`#S# z#OK%N1`RSkZ2c}ZBzdMb-WS^56=Ab}pi5YQi;IPcIz3Q;iK)bHyw^e7Sb61h)I77! zrNw*D31YtzuY`c$EV=oMFh;znfzh!vAp_>VvipzUo^zDfUaLgoqqbHTEcW)iQjPyQJ&>%-pWHN|}DBwYjRaW^hjxAk+s}s?wr6X&xC$6y2K1TbTfJ?vU zeYiQ~u$mpHfpBe8By31Oi|M=vKS^~j7<-bM^5oo|7@I40a`wf*xQsPwq7^Xb$c*#v zlN8`{Dt^&bQ}L|8^6cmi($Y{hyne5yZ)u$iU7253S6iE=Y(Bck&dbj~*?qcD%QHdLw=t!K;;I1n{)w@i_;Ux{Kf(ZPz z*Ed7=6aMY)m9>2>E9*}c!_Q9x(|4lxFvTvX4+`gO@136c3)7u%i-gpDh7X&R${r;H z^7K0zM}9QL4j>kby5F7f6!sMyj61^XvosJwbAJa5ZY47(sZDax1 z!C=i}2QI|II$4>&Y!9k=IuZmiVM5tKA7}EtLc_|LR8-F3H z2xpMsLXCA1Y6A9vWWoWg0u-fuqv&nGw$TitOtWl;sp4Fn`2s5H5tSF$K@i5>$rO&U zq~rtK&Mv3q#GG&1mfCBR)eKfmJqz@ag0vgYXCZx+C@=!P_*0W%deVr-uyWH>@jI zTdOD-KZdc#y{m_6R#x}N1nx>zgs!+N=g#;$x`(Ut^j&n&U9qvCo~}kWq}C#mgE~i! zCCbWl+fwROlV*aos0p!;FamIntc;-_#pJN`ot~e*GYk1^2?9J9GJeuT;*juZiouzP1<()qE-h24@rWM0Oa*;d;_=?GL54 zNiXe4Dh#TgHt8*yUc3lKBYf%jv3>RtkGRGBaOhO<*0`Xsx*eG`DBk33ens|@l9!%o zjkFZ0ldT4Z!PU+uvVGf3!E-ykFSXtyL6lCmBq_5FH|2ka!3zYel zjT_eUqW=$Sgr6w`5}1{U7sI@{FBMSq5G(x{Vo=ob?DQc)4+RqeiSO-0HD`MBs#siL zM>O-$6yV_T%%vCjdQTk|*w}R26^i=4AYHAgBxbaS?*t^B0woV6FVL+2f zujnoZTgCJ6d$>aZV_hQ*S&$E-2k63RFAsC1G1!*$%fT*Bi{%?BBbu%*)5{BB=Vm~-0eo1J%+JjotnCE^8N`CoVi*3g5UUcAO zMBE7mo8I>45{*eC?@6Z#+3sCM&6jI+xQ+&u38{YsWM?~DR~ESnKVH}EXn{~lS}O-( z1ROS}qFi-89D{7KS}lb4!?A;0=Vl$XZg;M)yNXxq>KYei467a3)iYVB*0gIqo^#L% zF`&It6pr*_XO(ITshGkL80u08hU+CTSVp2}f z6tx#1sdOV(^@e^McM*PEQP!vTWiDh2{Z8ECS(G)U%%C{y>-ZRna0G6FXqngeO#PJ{ zZPOw#bb>WV7FqlzB9r+A_?R{EV+LF4u7bD)?K6EL!j2T!_~Zc8uO-lYxNzg=luUIH zxgF=luXb;o{6>Xp@WQuR#wsyn87$1!ApY47ug_o}=~j^@IlT`CB<|Q;9rQxwWoR$c z3wJP~8M_@OpDDdm#e1&~KLLKmP;s^paowyXixC#^^XKIONo7uz)yQ{$j;0t)x37<@ zU2@ta+*n&$i2{m1(5iI$fzp%;^r zRSHSI#HCBhp`_eMN01KwXsa#tQx3^CZ#C3LKIS-_@Jx9Y$T6?ryQA*3N6D8oY({Q@J;23IYWh8Bp)~g|=!gMDh zNqy``U6%IKP(=L|CU9Oq_6NXie4gDcu1^O*BFT7IXlO;aVzHIJo{uDR$&R=c1L5#8 z;L0^Z^6EBI|Cwyis)o+z%}cCA?ze3G-Mr^1SS+|B%kwb>Ax zP<=Z!w!yuXCqI*&R#L~W-35mlii!e?YN8)zMCX&3?pP81DAL-Ow>`lZwKKAz^E*LCqFYGKe#ahI1VA(Nebb8{4%<7SdvG7I!K@SS~Ye){_LcHYnBEK zga#pb=P}CBPC$OHJN_^&@AW%!v(Ye)J*h|n4!Zi!k91;TK#o-P$;QqEHgKEAnFIgBtPc7S*PdIi@hj23TB0bgp}^5u*h%*-9KVi zF_?!02wp>*4UuzumO>YrEz^Ax(hrhuXHFjWpL>Q~4f*=IM$;eb5ZxCm$p|V>zD(on zGfvKzDlPeAF5Eeqa{eq&ZvW)v`l{sb6&b1KK{O7~)=M4NCO(iq=TNN@})N|WU0a$yE5iT-$rY)h3u;9?Q~ zLMyDj1xS$M*T^uwrtS`Zf1|nu4axO7&c`!rkx!C))GG)MN3fN;_F#otPyltEA<_6| zY1Mw1C8-Zy`ARmCI`%w>*hXwP8+S4vv@F3g-6y?I9kLIILH!W`J?6W@q~foWGJ#E{u;sVI0$E9wQA%wUWY`Kj}<%d{KKe| zXo9B>OwL}XiUf~yW%K(NFS?IQl&*9?Z^&M+)F=t>?(k=vQav?VKW)H_OA4`Zv?e4y z7eN;_esf%qSC{I0O$50IkB|2ScW0lotx>Suov0`&WvLl8wH}=`b{qo=dwagXVsb&M zwYgx9|Dw~@up>t5c7iGJj7Zq6mXP3=YycM-7Pm|TahsE)^5wR4hbNGtPBRRZuTRYK z9|kesSSHB4Te&@5#x$}j98*`ph_&bmLiOM@uFVan%XpQX*I%s#B(`6!#ui`}!cT!_ zjVNfL#*;X+Eu6^aXjBsaK#8raSPzOdFjKR!^I9U1a#qH#nDyX0J% zLIw%#A1OIb%v1jCQbvyo7{)ZDQM25iP?<-z?K$i!^PBRE@)t?>b3`#*3Gy&$vH!#b zATdLrpoAh$7?~C2We0~qHJR40RN3a13`)XJMjV5Z;9QJRRqZ@2rIjmfzXyLh%#QUd zh-EgcvmAe}hQzmi@L5w zZ?<)UjWC#l^L<>Aj|>q_uxr`wH+y6ir*P|aH2ny6cneVHF>X3ni>%cQPN=o!Za8m- zJ7JYPrM{rVf+8rXhvz``E5CebyG8hTui)s_?6BXoww@rkRHwPhg7~H4lAi3!VQyJO zq_`VYvVqdPp|UcRgj9=3VR$%dnFKfYx7{zQ{ELJ_ zrU7&I1X|M&lkBYwlZRNyyr{(8hTW%g=7(R42SU%Do(61?al605J`^CZzlg;;M6Vq- zIiZrTk51YQ8=2#J5fkqOCFp~hYaLRx8!T%PMsoc{3A^$-h1to;4655}`;;n83PJFC zsjwA`*jwfh*LDh9sS`&IDh^zs8EZFJ;hx6n1Ql;$m2wiGAPx>@^#X&Z^3&qYt-E?9 zjo0+?PZ?7GZbbmLjFK!XONIVnk zwGp-+(Nu-GC!~FkcsRx&_GcJ9^No0EW~Ps~6qY+6sZc~jEMG;Mzk`tu>dp-cH6*?_ zZ%2ps-VE|&KywU;MDQOL33cAU7Z6b_k(b-!9}_G==715`>R|R22E2UYLs$yXbU)3i znv2%xhMht)WmPsn`bB@@R*-v)|6`{GHrqcw2J${_R71l|$$rKWgoY-__4sGSUPdee zd)QU4bmT29rQANaX~`|j3Dic)A_XOw0rHWEj0+7uNvH!o1PB%P;^Jc>jKC2x3xl8G6fGpJ!*+e(rj_yl7j-!#XvJLb&sjR& zvf{*nP-`$bvPd~MUo+OrrMMVo&KYf~9$5eNtQe*+7{2DOZsAU#u8Tou$6Sc=w)*Sk0UHy4N)_&E4OR`Wtz^fuVtw7=MTN@AmxxNss)x*LT=h_djRMKYuaYpkn>9&4^6! zjEszOOQnh<|EKRle6$f>Q?UEuMe?EX#|8(C(F!1z`7+x?qt8;S+v9UPk zbL+A&@0pwa_y4-EC#bzHN*^rb1mKomiqvN6dZ=puYwRdK0Bw~z%||ups|_+oFc^ye zpB4o`LzbZsqN$6E{MDU*_pVq4Ry59u>dFhyzwY)&8-OS#PoM8S?DyYq+~57zCMTv` zscu)o{>Lc)YitqVfqvcmh`aOm%KH0vZeBpjcIoz&b^o8yBLR9ds7Ey#{qNiV^<|<2 z=oxq12BiNptv*r!v*-W2#&oWh#oXiu}5={QUf!oaTZ85hgBS4h|74U&k&p*)CQw6}

    bsc9tnby(O4`W%y;pYZ!&`WB-k&1z z!Za}Vt4@vqz3t)uo(8XlA&s>^DOgKs$(gzpPx^xJPYx)rLoPQyhlR<@$qfvTSy@|_ z6cIX4LUWf2c51wl zk5|{6f~Fb`q_0`qK@kWvUGG`ifyexvU2uKex&$Bt)h&Bp{ld9 z3JM80uOXcoSX)j!`Ap(*Pzp;hXWJ{wrvcyz{oT9zy88MObF0$jYp${3VKM=8{R`0< zCn=A1nCY=tMlnvaX^Yf5`*B;Mj?mM1T|Cg1V$-H&7mp48Daw6;bB3zM?-wPD7ZJum z4~>|W;NWx(mE`x#+XP+ylbW^Z%6EvpCC2$ zkC0t()0S3QRY69;WH~G?Y}EZ-7R)Mbr)}t5c2&`#m}_6iroVEW%DrbFncLvh%v7t< z{@VS$3*F(B0~t?aR?DvI75p?qbY-qFWD4$nQDpU_8s$g14zuL;k;-ong(u2S1y$ot z^qpcJ0`}Tk+V88%^YZQk0s_u7RCNCuuY(4dk!uMR2M0+LZUeX4G=JiB4Ipz8CXYTd zGeb+$yKib{YGT4S4oAp?6@-6DYXVI~&{FBD``xvh*}#T}HiLi) z95#>W`NPA*&!>G+fQ)**H!8T7WaQy}T|fyyNXyH%9TBbHkJLL-(mXCC@k1Fy5=yG8 zuiEa`v;hlk;G{i0J*65cZ2D;Y4a|MaJ3q@!vF1XS zO&!*-<6x>JFKcsmzcaGMI|4nMVBG=)oa&mYJpdomg}rkiFSnnJ>+q}MYA-P{@i5a1 zF0QF3Jmcb^B@f$4`**CC5j(~-&gVDImvYnCyNGCMoQcUpEp*q*1K z_r8eXG7gMTw0nYEn?heA{+To>akQmmE0(j{pOEnbw`O{Iokdh}Nr^2$YMd=mU;o&H ziY{_bA?bojOAmJ%F!DIp&@M?hAW4Jgig-O(^s85YF&Xfz(8*kPFuT`(oH+wj$OK4e zIQ`0|Ccn5SS+gjRm`mCP`=HA#5(8!k`H8yKqeSGaZ8uYbXN^W9NgR2v`*TcXO$#rPvRD$C zHDdvtUEK3Krm>ei-1F1u3k$C+SUyBeQdrGMWaXEY0g+KtQ+|DWdwn@z4xGW_d_Wie zEqwICQX%H=^(hEq{H08_^B&u(-uu{$3~(_`EBu|1GYL&EOA>t^E_nrD5c6qJHq`&h z;ZRuubq z=9EgHK0O|sEzgvS9Fo)nZn)6TGPv$Zrz9i_G=2hiA1KBbFC^nM6$Z1#^U}*`Mp3td zmDUrj6;%dU;)0X}GX{czR8m2zO%79(i`(r1Vrfy4I3BA`^G%fZ6E)|RFFaZ0Qb!U& zJb?uqI+>93+52TENMo{!2|C3zbLo`$o0reecOBlBefW0h$WBd)wk*~p&;e95>rdaW z5i>u{X@<;x{){95fzSH>9-ujt>2*8<-0RV?dR?2zS_AmUUnpQ(qA{=O_tzh%0uN~Y z2e z3l^3Vg6PCk4;O^@bu7zHJTG}7$v95(nx;odL^Ck(*dC2Xg8>HFHL$j3kiYsB~`6**2L<1{|A$_Tb0O^pVG$eAxw}%ew(tYQl6b;{l8S za%FaIZd~e{NlMI2@8|ExY^@U(aqkam0Y*Sq2{Y0A7q9Y)@FCF*f%hT(m)a|eY@Hxn zmFd`(MYpl*Y2EyU@4dv`-d+2h@Jq5nIeB@)`-fq`u^Z6TqRLe!6!}ogN=PW;25{sf z$wk)^{J`-sEdfz3>}mdA;=Q@+5s7)nCWuEN^Tvc>81HccTYyd+e)g_C_+~k__Sz zI6Cx`oMJ@T$J4e%D-p0D_9ZTZt^qMPwb^$Ok#>k#w9K9-V z;US!{*dgG;b^k%xFo0TAblv0Ikd2@|RHJx*Zq=T)Tvk$KSI?$?HfbTEvF85o-pRz= zXdkQ@of(u)_vi*WQt-M~EIYfCtu!nrNolC4$eU@swL9F}&3vaiAHH$wVkQ-A=&+I$ z6F&I9#bCLBB~w*(_QLjVEk{Ysc4MX&(P$Z;=!Dl}TNa{X&CQv^NDWBD2-=Q0lI^5@ zKh`uUFA9CZ&TD@PF8lJOju|y6!Sry!2iJKCK|INfavJ~WPCTTLF^xljA0|+;8u1&F z)YWVl-a6OR^!FrcK8M38eIUiGM?iwH`;txsvv$Zm}rOf=i7$!_Qx}e=>kmImLTE+ zM>al>eERDU8<#J*PwFSz=)73ECyQ+O{^RBGfA3g(j8qWtWRL_~HEyc~--sKyUdJCg zp>!m3+MOUem_`pD&yjbc71}#+cHB2juX{LO3L+;|ePr17d}GeM;{|4{A)Fc{URBhr z8M+G5fD3HzFo(^RFjGwBb9gW&lXnqfAD6M?uxs_t==d0Th-@-Spw>^w?=dlAL|^sL ziFr)cDA1M3{aopIl+(vl-g+X>&#!6C9DH{0)QwG|TW8usY}?`e4Oee)c+@mH&g=u| zBJi~mz|lKsO2E{?_DTSIPb)9zxi=|mH`wI&PUcGI_L@_jVA;N%A$yonHJP5y*c2T_uRG=YUYc0 z8v~f5i$3p!O&2O|!8*6R;G6yyRhg&%vvJHoPrgviF35fH63Fey<+-!4jT04-=h34u zNtO|+55DT@Phl;+b_F(*Ni-s13()C$MSC&3~M^!o(#crdh zFr)e5sASz`8?~OK6_}G6RY9Jxny-RfgEVq?ISCaYHv4sP>KWb6k}mH7Q1QUI<535H ziO;GgTFF&XRnv91vz4UYkC~B$jjbX_G&(dr+yEB!obj^LsfShT?||F49FyHsWg8q$ zz;B3v>@*$KU@(1)E~=^iR@HON;+j*S24P=1v-I)fZVF}(T;``Ek^pj|a9GL3(k*|J z%o8?7VStgg=Soc=+BDCFaBn}Jh?Q+d~=DJZ8{CsuOP~OJO+kf=1jTiv~ z&>Eir=E-;p|LGK)HyHxp-s!)<^E`n(wLHj)d#WPNCmmXhq~JIQLPhVYwE+%O&{C zP#phJtgmVXIi}q;BETs80LX4K9s>&E-0~f7nooTO4Y@(q*39pEd{kj5k=R*SB4|P1 zhX!UX%bSAu0~EPsSlfYUZBb+cA|)=SydChyoVMAcI_{|j*H#S+DTr==VtP@mA^6f; zpS60z{EcMc1u!ry156bM_5(QAk($bqt*9h!QWG~w$c1OMyAFvI;bPmL=# z{f&}3ztnFw^{ZWYU8OiFU%v2LKE$u2G4s3t$;tIDwJ!E^)|!p4<+MGPI+ z6k&E|CO);p0X512UE~SJ0sn1{Y-p%P-Tojknm?JV-hw%R$u48}aW7%) z%}YG#lxpOGqunGXTq^Da$Wie>YDb5o(hP*y0``P3ZGrW{U{}Q|g+73_9|4`pYinA0 z%+dhph_-{%SxM3Q!Ie%74X}o#JhgQ!!%Oizsga}24{&%lj08!E2`N^*yL4ht2FOT% zBb<(6K92?ae67rD*-8GdB2z2&GsZ|4Xlm6qdI{2fSHcz_=jYiEg_ZDvcf(^$K~vOh zNbKA!i$`nf5RSjK08lM!PBb_MPmI&Vd^mD?U6^Fnq0qp+oO5xsvP%4yjWvp?f>s~J zH6?cenpIW>Gxg~#MfZ54Eh>h!rTpd~WkFeX=X;G!Vhwf z`@x3?aM`l6(9J8>BqkpgbWDjmAK>z zak+PC^ow1+bjXGE*6V#BFChJ56pop_W5V<*fifR-d$a{s0C|#5OA~4I1_r}2Ri(4& zL!=%Etf6t4!3bVQit@10TlSxMl6yCgDaQpr@$#I=VP;V|lEqSSb2ss>lWx4Sj4e5-Cu9)#(%Rw?DqK*bsOgv9(K!A`tT{J6R_cpRvH=A zt}=xEZhoKEq%gb{Ue%L!C2*qB4jFe9DR`E}WD8Wc{RA@bBXgL&sl84RI#Pm?%MtVk zQgq}ZKq9|)qIF7jK(o*;Dx}6S_cZj{irs)tsUZti(|p*$f!!txE*OPKhI(zwwJ?Nq z$wP^;ED!yP6$^AI_a|S00%}suC)6R|1lMF2_lxQ*t}3^AXY+&9Eedl*sUbIhePq^F zU|=7)S2f_`s-#}AJkly+bEi66XGQn+oibR*_hm5i z{GjIJci-nuU)M#HqwgPgmplXe5}+a?@+$-7)waa7v_5!w(a*i^N!5ivT{SnDCHcyd3*Zs=!*7skWDbEc z&Vg#yyA=m?CU1f9*z~O=DH(inq-d2SsvvdO8zeMxk+iO_2dYS)@Wi1U_*SXIe=2T6 zc86YG)|G3>IX0CduJ3h1;TA^LA4ybkK0)^lr0>0X@k9MLgxHA=sXIZH;B>Y$e8f1S zdoA=v1N%uM@$n1qnwh9c!7jP5*Nv=$845X%%U-RlC9jqEbJ_d444_IgC6!=qAbQNG z5Hv74T7A}sw%M(V%6uf1GBpKV_vzaS)dBPK{dPePx~ALBEH6GCEiLbM}v;>ERxd$>8+vV3rTd(U>Kna?P)L%x>jB^k27fnSM z5Ff~VzMhX~Rn9gZom2VIWnmP#G;B=yA(M_bc!af8}IW&E>Ek zkMakFbse`oK<-Ag_29?Xp|6a!*uw5kMq<|iKFK1U&F4ocX*r9{Kn=>`sO@k93(W1= z>Toej2y4N0nNCZ>4~TX5b78#wUj~;kN!$#Yr6zz^x?<_VChc?*hY?`Nz8T)obMHe! z%5T5(i+A;mfr^TH%NX#hm>1KxH_de~nRnF@`$uq6_g=Nh@WtnLakzL+jhS*BM!O(u zdKuF-dqT?@1dfrf^Bdg6spFg>u=&Au;QNL+AF|wqUhOe^^&VQ4Xpz6Oc~4mE+w_Nc zEeFVE4+ zNN^s{X(}MM$3_3y*P2l?Jh2%m_D7-pi+YbdokmKsvWq}CmXeZU9gNvSR)HqcT$zNb zQ;m!Jd0-Rgy)INxhaTv!#Cg+L!1N=1W(yLc;Q>n^nk4PymCeN}Ge^m;p z1!ng|_ZBd#M5hM@Js<1+6!3owV3M!$Wc|Myxj+08D*&p)3*X$r850t`rgr7EUV6<5 z%$3bxdc`yuxd%iARspO<^=Qa^k56g|1ja0av_!!QlaL~l-X~VW(O6GricMxi-!^v$ zifqk|y9g3?3`6=hvHx`-0r?O!NK5ZG*$JMn(rj<~`a@9LxtxE<{8sIwpqQ*wp|V%P zI1?h)-PoDFYC8lGlM${aPBLaZC*VK^4-zUTi`8~|aU8DB`uH9l)+Jx$(^Wvjn)dxO z{IJlt%&A8`GPkP1%<(+y52~1%s>C0)!iUSitX#i3cg|1E)bt@=#>9s4NAV4qk`TYt z`dVB(#2N`9`f0k}u)nl+Q2j9i6VaTTPATIFpcYa!kh!052*AykX`5k?nmUDc zM(QB>E-h&l8WHrOK}}7=0ok|9e%6%U_-qha{h)Lv6Am`ZqgbZvn*!fT#)tAMhB zKp~IK$^awQNQQ8#K?!}P2PZ%TmFXqJ?ZV-hot=dVsYNH@5?SDoM)}wyBQI|N2Q@vq zI3M!%nGV@$!WU;)IVXkI<#o8+@1!UX1F0PIjp?!jLH3=yDCoWMX!5vGNZ6q5_yWQiNp53jQ;W(dM_KyRdm)lTLWmtzdXEdDSVd_jUlZjlR3YK}R?@U^!+##3w})e3{+Ym^9WI8;dnpNyu+c z1FaH)?iA|80&;6{6!cN9>*<0g0QnJFeXuYfqp879NhzZRHAPx57LACYuSF{Vlf)M@ zN4!X?HPhp;EwAC6LRdzZq+|?~ZfOgoT}(b)K)>iEimiSC3aj)p_53bAQM^;0n2A0# z6LdN=vSIC5==@|_>$|*bnqRspfdo}}(o{pr%|zOF7eG(?W%jMdfTx)8+)Q<)Cbe7Q zrx6W66<3Unk@)JB$(!H%tQZUw_J*dp_aNk21(4XfpY^3du<8;yPHn3%_dUg^shM&; z>H6%P=$Fp*0*Z)&2se=f>qXbimVpHF0dZ|J3^ZAqDU~h2C+U1A>^^0KpQeBq5mE9J-r{ zF>L@~%1wwW1s#vbB~=&$w@&Y#_Z~ zY@ZeN^_mogqTgdBv`oYym%10_g9Q?Vcp{1I))rR;O+5290Cs8BVr>qGV5; zMgJ=FlDSk#d8Au9>n|oOuV|6}#*%)0nNWZV4Ml9?PyV9HUFxcTNG5|NM3 zK6mZ_*bv-eU%_I6~c- zfr@|CM*L=Hp}|9QF)F=v3t;gpzl z^?xq(Vn*YHH_Fom|M!`*S^%O%i~O7K|Ghx5^^2*pLL1lfKLhq_%ya;v#wzXCr~h8S zumkS(ivw=(ubJ{kV-8p$ot;q`yl=3~!$R$vwty+4t)1HtTb%qqLvSMx-1)~tH!}DL5aRzl90k^|C+k){q9!*oGSt#uo|>NCS}hl`7w`LL&G=))ZV&+C z2w4}Af|PcQzx5Grc!7Lz^Iw^UKZjH-JTL1RAreHvlYsV$^gj)nln`qZo^)VQ#r|)5 z5MqIGj?RkzKfIT}h&%GLum5@7{+u0tG{B(IF#N-i{7q*%pad=m&nt8PRZscn=>HmL z3Sczh5akB{o?gG_vD>c>7_WP$@PDqz039gqr_A}^2M!?vF8u${l#R%Y;;(f7>Bk?T zVgOY@M+e|bVf!W})t9TT;_l1Ie59uX1Jo68AJ}PBhGs7oC%m_WF7tneI!h1R+|sh7 zsHmj0I5Z;6p=oPtyQSQceRg6Ypsu$!pmgMAL(6di)p6`iqMOv5YD~j_ewq~9yrjIS z>}x6Db0#M*D+LdBs^G0?%FoYkugWPdxDiv;RSX*5J!(8S;0G3&|1%7}&th3sRr%G` zblkMbibC2j^`GoK%u!xUo!U34#Oc_mYidfWs#>e5xp{XoFfNdnS06DzwaNLs#fvv< z7;e+5|A&BtzzumkMrL~V4PXKDslRvAs-CWZ_~9oml14*IM?+6P%l)!|huhh#*Wzo1 zn9^b#&Ix06^|5?be*P?1?)-6hp=)TUteo79Kz0SDTxV^)2e{87Vhf=y$o@|*S5_ec z%pA>LsqLa-*S7ndZTNm)X{JpzYZ4M&Sk!)(@zwEt3^ij(eY*%PBjdfSXP3K#6170w zC-Z2(jj%IrV;b0~r}+XLO3I0qm1=jrw-$@Ob8}8*SfcC=%Y7vUeow7_kWG|~$ker% zB#M?VAxZ0?%F4>d;>SKpH-Qo&$K1ZGz2$775cKqk_nKaasb@5!@ZmPAjtWgK4w5i3vN-e=hjks4 zl^r8ziF3Te#jClP1`1fcMq#0^n>9d>RxgoA>yX<~Rq*rVH<*3p?>;e$xwU9{P2We< z*4)^4zMlg`R{$Sv>*)f8eIQNBJ^$X{6m6tff_{1Fh&WY)CJbaLOb`~1I*0^1?VvwjSBR1TtL5g4V0qH-jHae}`2~4=q&c$-O_t};ko+j* ztJ)eM=vdLpV^9CR{n0&6ptgg7iAi2YrgC|dQX(jvOjv;b`$m#|3nc?1qh`AYKWnze zNv{X(%F0SwW~QhS7)7w^^n1&BySAz-_s5M+Kq{#QHbUJGm6#_h5OlaOI5?QhZtzM= zD}@6Yo{Hbf60ooPd5{~$V6)O_2W0VTNo}FP_W&QD(QKUTDO2Fn3*BTHGx4$aGq4E# zBn~5$20b9;1EJ+ygU^p0KOZ~39nY)$D3=u%h(0FPm}&|Y4OJUQ?h9IwQup+170gJL z^CoCq&&8Jk|2dW1GJTpX9dh+5@3(WX);#-B2%;UP(g<%AMrrg(<6-4RO3TU`@}n-=F?10EgB3AG>S@Qzx>M4m0Da>wpwHBEo9XA zL<&|^{dAf`OI}j}tsG5&RgHv9%oR?~W!U{T7hUx7zGh|f$`r{piYA>iKVMBNv}RjxjtB(X409jQ2HRy zd=;Kf$iZX3GoWqT_U#v@WH?Bp+ho5Bi~nAXNT^C>$@g-*ze1~S2+ti%RDCM1_n>TT zZSCZAYgy42M3hIdpVI>X&bysqLhvt+Cv!NK|RfD`#; zUx3+ITBvwlZYX^+rhSX^dHd~><$S4TTujU*kSq9xtalG}{@rE$_I^8R-Cz(&5FByF#yUxCnlE>By_8_EM#C-B#m^PwAn=6pn}SG?(uGWmqnnmx)`Wx zU$t*#DF8^6tvSV&UjtGfIpX?NLVlb$=Rvve2<2+K79U_v$5_34P4)dqGV8n+s(_L` zbEXqUON%9x&q_w*bY{&{vQ~XHjW?uRf*F-mApVT+q5uA~NM0ct_oqQxAf~R~`87F5 zJ5h$XsiMi6T3dy*TPH6J3{DoR>PuWKVHV`s+TQ8vj;Xs{vj^fh7Eor~CdM-|GTty4 z21oYitmFrh!#@`Aa4DQ@UWcax-Ic5U!FOQyFc{wB~j zub5m-28*^DAHi5nK2@v+bHv6orUiAugr8xOK^PqfNc8B|*-V33mWw{QX}!Q(+T{px zJDyhY_aDb?Q#FN&-`;1VhH6a+o0Rk+H*1x~&yJKrBMwJc=;(+CQ;D(_4&!oj5ARfCo+`)p%ePFij~M znN>lLO(`tw&|1OEfEJ!VK-I%yvmZv#2k^II!92@-IFfMIQ<3Lt^K^VL7@9}p=%uG? zJ~-I@FgGb-j!LNT?O1sO>yz4FM@6?_h=$iTOE-}1E&7B4|Bt=5ipp!*qIGczZovue z?jBr%J0!TfySqbhcXxMpcY+0X2=1D2XZBuM+wRkSxa~Zgr?f`+`KMLQsxiLayY1d3 z5yf=6hR&wxdiAP@?s+iXB;DGrf^`?i{M5#UYjCa(Jr7O%FYfWR+;pBn=S5`?xyq|- z^{ps}?O%$0K>2EUeDpGrUtlN*bwomr$M{;|%g5D8`b+pH!FTQ}vrD#wwt*Z!x3}^< zL#@FY$R#CVMaiInF)B0FTRRE%HHSg0a8%hY|I;V|_CNFaIj2Vg`7 z0kr7YC_|PPf}kH~GBzzOJY0Bp1yJ0HK{?C`iob17`}}g4AfA(VcgCp(X{^6J8;PaJ zu$oV;~hoITY>dn=Pm&9x9aU1Vy2P3Jx@!-3Lo&*1Yku44%;9Wm<9BrRh| zc?^(@f;c2Y68-*6B^B|q++p4yPaofl=y|Wiwui%NNl!E zXLI0r*S0Y9WI9Xc!}8R?qZ~70R&mq$vWkKNpQ*03dt0Mnj@Jj5qKjEZ<*R{an48f3 zhXk9Qqi8ditGxEIGOcCQk z#!hp`F+116^y=l@rD!UDI81udg9(;tLpyX<)QI^Gpe3~VH02UX8kIclxzA`bV(VLU zNTc)g4kr@U?{_2NH^WsI6U*w!+=!I!Sj`9c!kIFn=9kWx-ICeA-){vTR{&@1(}o@` zkrmJKAZzL-X3&UBD3oXDyj2X(adz~Z>mOAB2$E}dTIG7L7iZ0~s;Z(LAHZ2k!uI8? zdAG8ZDqnjxiZ(Am#uJNC7X&p~uepf(+Y)_wIw-uoIA|yzL}5BqqSl`&CKUwyb4zS$ zayLH4`nL&t=Bt-`OuEm{O#M8QxHGr6-*!z1SezXX58mpD(1;PRpRT(%zw2lYwcut6 z#i62oO=YznY6v@!U5~}}-kq%MF1YP!hfBu9NR&xqG~UQCV6spPFZ|Sc2e^Vbt;Yla z#i22pa;W`78J+`EF}^2$k|(;tmskk}cY%*uo!fAF~rvXDzy%x}Axdw?NSBR{`})CPhzm?M*88gkQ(@l658fvHew;dLBdc z6g~0rjDvALk#H8%HJV;Od=DT`MUNUDk>-g)Vb1^(ZBK!Dg}pfX<1bgsI!wru-eoGD zcQFx3n4RKF*b9z092zqvhnX$};7Ztz;j;ZEI|>$&i1>iX^v@XYb_mISUq#OzleK!J z12KDQ5V?%)oUS8*q&c^z{bY?fC^K>TXD)@T#~wt#2_eG?>Kg*d*BN1?ZBo!V5C~99 zM?#vkUw|K%0rMV!@UZ}%r656ZOd|n8`g|6Fig?1B%bxZZUHZm^0!*a$0gl3d1`eSz z!~{YExW!^;A{};&ZF)1{P-lM>=1?@9$rpuj7vlhzTwzEhi%M9_J#vmeDNkdaCf%tQ zw*mD0Cerz=UYAjLF+E%C&w^!ECDu#RsRoISkP_6oeogIMUzZ&ZOZNaS2Vy`&S|GUg zar4|u)B{dxjD}j{F}=4+q%McjB*1CCojUWy+8wO)F3kxy+x6NelGCAi6Dr`amd^w3 z?KQa=7q_x`B6DYkOO2SQ@eMIX;!=}8yeAfR#wtivPo1Qn6FAwYUoRK+lyF+w0nkXs zXYc2|Q^`RR?*MSrKU$FF$&?(1Z~_PlRR|}FLNRvS&uv%7OaP4+A%+x>!U<|BFo-!d zgq>j%Cm8{e{YG^IUpqSU$$b^s+nG&XBN8l=-3q5Lrvl4$l**A1ao%oNK&B1CZbDYh zmppR%Y_&Q4jA?>ODSz|r*Mpj-ohV3hh1kmc!c+f;Uzbxn7pzGuRIzl}aBy&fx0n4$ z*&gSuL*!kvk|SFSogNQ>!DgHb(7rP6BI7Y+q(pZt$l!ImF3mF79g5Pf!r>A0o@2(u zNd8=5U*8gc#z0+KUR|sKU0sTwovEeDGm@A1r#J*`R1WDZCQ@Si*Q#1hE8yR!F!AHD zIq!YBNj_cEYB3v+_V;t+jU_Dex3dwjUM!`U%Hy#HqU8^;jrCVkkdi7i+wD%;WLBg3 z&EcD`bx&i0zf`T_t0}7fx_>1DBr5omBy_cUM1noDFU4~>bE9zgb_%Dm{YaV0;cpEg5Vs#R7jc@ zghplOMqmnC6n*#Xv@j0+LZjt76^xZSY|)YQ1BYBn-c-vA{N@xIal|MbETP5{KP$1^ z3hs}nLZAQ*cwcONXsjW?tmyxqrMGs3(*E#%mScon!*P;pFekCkLEUBW9vYmaSQ^h{ z$90RS(23&%kN&EOhk8SLA$ChG>T!B)XKKLa&W`4O;({5~XuvfzdU68Yo2`oMBCPR` zd=|b%D8!LnV=GZ(6GZHcX&Fy|vCz|%Z+!K0+k83qS%2s;`1$Vs^1ZFtQt|dLis1vIe(U25iR&20awQe0ITCC4eCp?`|ZEXFJ5G z=^h@5tetNauV>TOYM-Acq&Z_vUfrP7Q|8<)f*_kI2&OiIpv2a=>RUZulxargBZm+n zV|}0eXxW83umSB@1(D3pq+xtkUXCgicr zY-)0Mt71_7Raf5@_2pU0-OVuw=_2q(WuyqD=%LFB_+s_x6RHa@m{nv$Lv;UfD@s~F z+QnOdXNYuTZFMJ4@Ase-QKQ%5`wte0S!o%La~;^(JTH5oZd^fwh2#91gL}epAMS_Z zlZQc29poWIO^gaz)&tbq0)MWar>fU%2@C%6P49A;JlA@`q8bZF@OU`$xy+>qLBf62 zQS%0{Kd`tv=_eF}JD9IwC`w3N)L>_@DVz8E$vVG2zTJGx(1Re+`$pL5h9~wut!Qyd z{3FroPrGF%=N|R^50O~**olC#oNusUfS@~!nd$^1{GkRKImvgZN_ACLG5Gnn2{oQh zvnJbsuu#X3Ri7Kn%F4nue=}@aR`;iZI^!jr+z%7TGMn{&1~qc6c+Sjd#(!j17 zhj=m?lh%C9Re@TRZbYcV=F_UZFUWVKmyQcVO1M$RpjQI}P2;xuB|>3+%a@Eh_$y9M zLk?e=Ow@lCsr@dAdaRGmYrvxg2#*xxT@^Smgm za$dL6?<1|no*PMw+yoKnog=DSwXR{38I)~#9X%ivfO5RwbUc$UK0Z(H`W;IA-CTMi z@9~(yC20VuhHJ3g^T5Pf_h1BO{u2B{H_q`Vyxf5=#*PcYO_g(6$!x z49-{NHaikEWbxh=^PS2c8Rro`m_%t>8aI^NA_wcGkF{ImoM9|hp9niU>kiaKX>yhN zu33zn!iz-dPBCVm0;;dIp| z@B=w78i#%fVxvBYxq>8ix}N1qTl}Q2hIqoQ>_fDsAqTL<9n|xE+hN>Q?siyDbZcE< zDU$j1ypO425$-^EK7nKhsih6eb_(3Dr!9325Z zGLa~&Y+of$0j$gdov{&R~Sl?5j3^LN9S+0i(j`9k2?D6L=Ln1h&S@iSK7Pt z!h?>!JHg9hapcw1xGrV&fny{B#%?f;@7#~fPp~$)ZkEF??fAhUheTT?Gzg4c6SlVf zt9OT^06_Z9^)48=&D@6*Xh0|3?O+SLEI0Qm$_108QX=S}U_Nd*Ab+sz|PD<$<`+$9lu3v7(`(c)1MxWn=Vg zm1g{@!kWyYcXcUC5X_oHnA}9BEdS)q$##EBI}CfyS*GqSJeE~lMJNsQ*S_bA3G9& zvSKcaS4fwjCFVn-S=4eoJwgHePHk>OiAZTg`&CrpEzj-dW%;f+t*`D`0$}&{4Kd(H zWMm|9m@MU0joa7bTAdI_#`%D&PL9tvgmEb3Tn7%}z*HoQ51>b_#x8{w=!mD1oSfwy&(KR{Zd^r9P-({{N6;#{AWh`D7 zH9so3>an1t1pO0K8dWBw&0NetR`Xs4^H=&W&FAXvzeZmCAeF2GixjGL1GPC@*I~h4 zw_^2JA%2rqH$UY?Xx&HO!!!*r!!F|@n}whYYG9x2G0$Iy7Koz&roQrXj5zZvB`E#( zsE)XG(MNYnI@6ZIBswWF%>W#&`XzTWay0PNK+lmd9s_VP-pnIdEyWWLvh`iy0d~NG zs5J)dyOD2%G-HfE63N6SY3NZENHPlQqq}%;c*-IR4E27?D+3Z`d8n7VoqTH5b=}gT z30y(OA2tDvcj_4gRpx@la+?1c_UJc_70F7aI8=)tKd|4hKiE{p9C+AT>{)%F@x7XU zsL=n@9bJRDtU^x~i^5_y54?N#7&_eJ*J_f@|23WSq^Pz*5@aPQCdKyG3I!WGTl_@I z{<#mO?$4$A{4HrxQzs`nB=C|?4(yL|2DZyH{^Rugn|poHuv^P3#gp|iMn%P5=cnr{55ogoc{i9uXE4M$if+QHY}U`?=YUl*MEkolJ1|!51Fx@C zVVhiIhCt2ux9$deqSgJ*Jn(&9P<;uGBC&z@eSLN<*t@7|v&Tn6PjV({6p`6l65cvw zP|5cOYw8E_r|{TDb_zqG*6VE;8lh~&qyE~&OmKdAi_-4g6xd}y+L3sUxuj=XM8NWK zS7c+V$u|mW$H{avhC3tZ+d|oRpUkHeb&{S9pHhK64s=+H=`67$;mSq(1-~I9X#B(y z;Z>2dGfsS*oQXb=0zG`;b>Z=Q&qVbTkPR7&i_A@Af6SJlP3fExiykuJ@n8fEzTjYM zW>h984>m)AZboJWCc^!iv{O#Z`Q3opBG$jUu*m!nrydQ}Qra%k)(30z^~`jBhlM&F z*KxCpe%i$oYIXS$M}ZC9+zTf(h5$Y$q_p9q?j(H!WGivqE=EWSK%6Q)ofOOwrX z`JGILn8+W(#A!Q6SR(KAM%_lq9EW$)T;sJ4hq7q9uLb2ZkloBnM4Y~>WUQq83FLHoHm<&>nuo82gYYQ!w^#! zoECNIn1cjYyPQj@IXZ`Y-UQAwCl)3XA6DcTJ zJ4kAz2^)c(l^b6sC{7hd1Z}`>d(i*kXa(iHy_Y$8$b*2~&0$Pky;|c0R2z;h*1Kx?6QthZ-J58|9{D@XxV8Vy-m2aGIZuaxRio<2%z&)07JTQxX zW?nXVpEsMDLX2(V)@n3&nK+v=P`?V11f#o&E&}#tDs|u>J~jOGMVuM6z5wBcxGu5` z0m3Nswoo3P`)4oOYp>pp43)aDBC1?@cu2xItr1|4Pg~MIBOe?_Tq)^mX*@(oVnDUt zv^9as8QEc^&yzl`R7yH6ATL5#Ph}eycJY~$m60zK zxPYZ*V|w&l)JZVavAV~(l`6Bp+R`GKUZ_hRllRWtC+s%-__LL|b462E@6D&_xjCR} zUqOS0I<9pi-ugET(iaGVgc~$y1bcO!aSFe=0Xbz4aZKG`)!}I9S8=XDaO-zv`~yQ>^a7 z$H|@aIrbgti^NDJJnh(n^3jdimFT(Upyx)(0;?HWY{DUpaF%M8jVWu28oG2t(i($| zu*sFJkEvg$HsNvUdm~b>Dm>nMY8U=(>l3OE#;aREPQf()wUQ8Pb5BKS%HCBJ>Mb!> zlGbNT7)GPbn(XoVnY9q$`*to&f{F{tZ>n9W@pE7=`Sx*l_xDiHVm4BTu*n%s~1H&RL92&Tpvq@9X@Q(ohb(s04+pLhrvD^+{19Ph zusI1CvWPNeqE0C``-E=s$q`)rwIX)cs{$cZX)bhjN_-aBw>Euc{|k;1K?QZcTy`Hj z<;Rfp4hg9S%y;bY_k#ZIG=DR3YOoNUK7nsw&xgbqasI#FaL{S~?cf0#44Dezxj54mb&{tztR^*1P{QrLfZ}P|1kA` zKiO?kfO$|mM54}L_3Xd@z_xfScTI>h`Zc9sCDT-$ z3W&lJ{WpoGAOb*w;q=Bx%7vA5Ll~Ht`O&=ux-{|sK|B3%gKEK?>;c?;7Aq!hA60SNk!~r=oVQ$^{*-JuZgaUI0r-I zCA?$oKYsk1SK|k`#ATBsmH)ZZw-~?;?INzW|9b@fb>niT0g~+=+a2S7cu6ndAZn^d z$~FJC%fy^KL^EncD(6V#YdjI1p{3aRI0ATU| zhg0cUSA+b2ITJ*`q{T%*I0qOqM>hdjT&jJ$=G9ddz_kTFK2~Xj&S(N7$9;RR=|2X@ ztpymM(Me2XqwE1Pn^LfTIbE4rT3MS~0(BpmPoE~nq$46u7{DojdFK%`-}1lC*-JF1 zsOen54MWS#taQ}zu(d6B^ZR%Gw6ydeKXA${ejFK3Oimq9n9BoK|ChT;)qi|7Fde|K zaD^pOA8JiHTwrzYc9V?QH&CPvvXkXvQPytfQQ&UT|>}ck$Z<9BsCEDq)IqF~= z8XE!Mq-*Szg_wtb9S?F6gt_?%HgZ~Q7HR<^{B{7a zcw#FE1TR?Wqz7R$TYqy@G8cU^jlhNkmyrme(^aI#*KHb5v{4Cgw3VN24Gj&tW!dmr zNBr~}|F#QP0-bWaiS5Chavj0n6${5%+|p)@`;U&?dtvd z8NdB-z$$*oWRvQ2zIousWCE0yf-MvjDnV12nh6^PI}KT9s2g`T8^iAOTstZrdBK4! zFVO_U(q0MvqiT=WtKm)0o5{SqyuH0WB{{x3ijPmQb+*IN_yBQoY&Yn z-yZlAXtVQtY4`9~sbj0>GKNQi1Hwj<6A zlkeg(IR@#!ax<$i-k$(0mv)ftXY+T{x>_Y#c9JW_2gj;W_oDCZgl;qfL01w)rFaXBDskMs#c(M>~<5^7Z?7 z*AMZ@>6t(&`1Y%+fsev*&TB`}!GV^?{Z;u?m^54HNbY1i5Om@VbSPFa46m=)vcRu~ zd;rW!^+nkPTnswy1aQsIO%M!#`uX$1XL@?Nd6&nchGYJQW9HlPx1KkR?DE#syjBV( zwsc%QVuywRg7w*S{&Lzs_yCum(TA!H9nDX2s@mH36~{{c#|8G!y4KG?>*U3B!Tr8R zcrT^CLr*-+$iyhiz7OC=u8J4f8nY`l)HOe=>MZqG0g)1BmpOj(*vhS0Hi~BRL^~)%jb^c0bh9!H-ESTic4&nibS- z2mdqCQyHX}nhM=F^ye8-416{S)8LBrk9^R2N(>F49~IK-0f2Le{F^S3&M_`8B_T<{ z^+C}5MyJQyb#nI%Of)XXInWUscmT2@EH}G;0`96ruRu@i?bPJs%?mx<<7eQ7(gpOp zS9ym4aH;DuhyfzISx%9?xqOQO+$utD%g8M+uq0!mQvvAS;vUl}-PMU2vs_GWOIX%?{WX*NE z)Zc7!+zcsgUv0q-V5$eO#E7G8XI(+{!>%BXoRw}RNX)$AOJE%d&v92OqsT*`gNh2E zz#xN?D5h#1j(Jvo42RSU4<<-?C zaGs2aYF996_!E|%-s*^F%t`%;0_`X7;-R$RqRr!+DkNI+s1{uvv(R)fBi8toG zp0WUCjn&ekK{vjkRa(Lv z%%*25@eIXwi+YJnM%LmQH0CSAYefXf{J^?`QFKWzBQHXpSw?*Q+oM?w3=Bh5t|E)2 zD!_Wo+04w$@s=O=n{6Ual%!&WDapL4^JijbZTnHOOKGg4n>bTp0Yg_diM|ze6z8T` z{#bCPGKVsuV9zo8FxIp|ELzfesb!5Bnke1~G)hz`$by`1vj=4q0s?eZ+_7?L@|dKg z0_XF*zAPAO3!38`TZ5$9rqZwOXD#$_)wHaZl79KHQw+ubA~82s+(Np#{tGW082 z2+1taT< zPj~d`yNDL92h4b=R}hhqP*Cw#nPs9U=7!fXBF{ZNsV*y46Vk}~ot|0-wmTjZtB#Qw z)W#=D;*5pVuw+MnC84K}4$txV`SNf$Z%Vj54qmFjYG`cSM-ayLBf*CAY!Dp9*?hN5 z%?XBcH9b8YZWzd-O#imCpe=y5)l+&E=)o%ZV!}CiW;eF@*XL4-^WUBN%8c zB?E_GCHJvG2=tsB~xUXSA z`08!>y@LYr#*j&8&dsTzB;r>!qzvl#*-&|tWV7uL^5E1j2}X z#$9*|B_wjo70^=g>__0}f!&8CdcFXHi-A#-G+nAx?2oj-^dT`UT{r~M3rL51xjiQ7 z8!8J%`$iA-h|y2oI!0kOF57LNR{DE>`8~}v2tw?Q4iyxaihq4jR<8AGb=zQTWi#~+ zV^Y;qA@JBNJe$gUk7ewcG59>u`}oc$CxaO=2HF}Rwh^%O4enEm48u=31Q%ksJ>wh> z=vG?XJTA1?$)i5vbsf?kk(RVOn;c-GBS{?NqoD;w@*ZXOK@&wyb~WNXYF#d@h&d#@ zYnsJf@}4NtIofDs`SR{le1%TM_~7aNYhPZG6ZZ8@aA!>sqB*JgaSFT5Mu%>%KM>Z8 z>Di#A2~ zv_fxs0B8}RlQP6~#z!B1{g_U`Z7_llYPJR_=K{`;F;u+fzXb;Xegp?I3P?PNH{iL% zZib?8fe`l-iUL|?TR>{PEa_iH1@?)IjI21z{Rg0z=E5KtFB@E`H}Scd+~3g^K0v~xfx*I9B2B3~fYZ+C zg05(q+xrLkcTg50$SmeS{ynf3fqXj1Xt7SI%4!APO1l$LaWPtrrr_h`jz=-n!!Mug zv%|zA!y`-5`wUoe+LkJ4?{-_eGa)&#S7Nufy?HlnvWpaWrz9D)yMn{YBdM;hBkJsL(9m`O4w$so5a)}57Yu8L+(LzyMz`H~(d4Qu z-?6%Gb!FADA(#9KA`Tc1SF79RM3&1C)v&Nqx`lFmad6XZHF!#sVbk~T8^~K6RM7At zZ00iOw2w^l2)#2oe0|{4oV$^LCjcx|D23HX3=yDDVC6*y}+||by*&@MyqK@ljZa)+ipOX>d1V2D8aYcR@3nkW%(WPoi;>|3Ghph z@yfkAmht!eK)}OtaQZF44+MkDW%J`p)u4Sou~78o!kXyB(EByYK)F+^45Y>VOezEa zmaqT!dp%v)mm<^6_Cx3u@1m5{xU-3Iv6123`O4Ylc0l~ggK+5-3B5Cd#oEA0ztJp%THB|a?|?cdxbCG8jN(aTs5x@~HrxP&C^R5|Y=lYVdH`~m z3d|Koat+wXOq8Sh?SB23RmaeA2_}6UFUKD{nxX0cW$p~{CU^n57|o-BgMu45>W`?i z@D2eEiXVPT)c=P6kwmZaqX5&tJD_q^W*pr=rRzSw~u03Ho}$JyXBRbY&w z;@X2!Lzr(*qvw--S#ajgFxnFkJM&x5AHxlilP~>PFsj+-?OHLy_!jT=gD4Q0*V9)z zlg&wwZZ8P|^*4d^>ELp^|0~Zq=WE8t*GupcoY5XMb#+b6&j(kIl99s&ryc$s9bgiK zy;cR7?>+ZeY#XWnHBZ7qxFWKTvqbwck%S7 z(2SuaCD(L#{Pg}cm$Bxb@T+q;3fsE-BsiT7jD2IizJ06W;ytiZ5ETVX7}!%Pb4{b` z71%P(Qxdt2t~rTA`-E2v7+Gjc=9A7&^Idzx&jz_t^vGxoV)O*4f z{%8dlUz2k97auGjM*}}x?}lsuayORN*rr{#fUw@ot&f`ik#>m?H(XSMnsA;vdxVm9 zZmAG&e&V^|0a6J@S453H(a0o%#tI6t=z3k0`qBvd-yYQ49I04@^uETE%Lu{$Ofc%* z=v zLG|d4kBVLk`D8WJqK(ku$x|{`?fn*xqyOOI*fem&R<1O9Yu7Lqg zC;~1wNc32L&ui()k+D>^!{OTryv5V)F@tSlgAlPRf!j-B9*$~83WkqVL|L`Hbr=>J zOe6IkAK)s<_^Uuiyp_HpGW1=U)~a=TGDa2 z+8LzTa6dG#cYT_mUnCWD&~VgBm(b2Ii4p1ma0$jON?KpQ`_igRviE=BDV!|RdG^i zAcwNY4#>9P(O^i{Z1Q!U4Qqf3&fy^9Fm$4UK3 z9tuK=fHHV$A);%Mgw4nlFf89(C240jJC&r^+2|;b^v^{gsA$eQQO`{@Q&9Y>M)6`c zi;Yda8zoQnxCg_jjj;i_AW&7Ucm@f?;Cz>bDs^i$t+ z*=qj6Mfc)-#+|pg44%gjl~R~oy_XudLm}b^uKMHw% z=hNlAJwtTnY_#WLoR6kha5%tH5LSxR({$^5j0h3^81P1N+!+X~;ddppw7uRNb%iNR zrKAv{)@Mb~7H1Lt$kpJ_m6w6)`>jU&P=&XTd*LZ8OFwTeYT;!pkg@b?4vgXP$hZ$2K}DCU30h zr&W@+3kyxICiafUD#%fJE z$)Uk%%FEmxFZ?f<$hm(Ug&6>1DYfd&I&UTF@2(TLlBga0Lj;K0Sq_bB)?i9vGBqH@ zws3_;y`)JooR*iqCsn`B*4!doiV7J%*P(u?Zr$Oi&_Q3rDzt5g!L|~rRP1H9OaC?^ zM%cs9y*-8KgUN@Ele)dNN;Fm>%NiP4x~w**X|CL-FS5N&%QL_8xc6Np%DG+MJAW^IDd%4*LVtTzIDqzkUa=P>Z1XGhI|7fIfZ^HNr?2 zP+n42Q&s!2S5_BZNobqNgyWawiY*PFg1z~AxuUjNKgpat5ud$4MQ2v8V#M zc%!rZE7vO{!5R>C+>4cxt%PAUK%k#dCy8|<1k5CTDb`x3MlCxDmaVKQp^c=BuFHDh z)org!kpfUfx9vM|H-MMN$Q~+`F0S~v#@sWT#d6A`&)08#Lqt8wVryQs8S%Q*w^?)e z68>^kW5nDYwzrTqgbuKh#O8`0XrRzgQ6(f+hBT0US$ShVZCjq=3g#$$V*GLk1Wc2q zP&`FdVv-cXLU8UA;!vaLz(Cyq^(VBRB4B|tVU$=T_?lx<^zPYiWx)UblkYr;C6eaA zpg%-25Etc3J6MvY7Jz~RvAZJmsWxgD)n=c#uW~TT{jbdiCD<7hsGC4>p$WZq8#ggC zp6vd;@+BC1ca7owLxhAYHBJ1>Jr18fjI8kdj|B@34D|Q?7%TL7;$P{fW?`i0phzD! z8jKE4TO(Qro;hf8OvTtCEc=#YCpPw;KGBsgS{lam%id~mJ=J{D?hZ#CA!5)T{)*pQ zhEO}H{9OP{@@ARb=`oZDB9}<|=v8!FF3mggRM#&64Yw=3T|FtJWgVje+RkGQ{2737 zGn&4ojE!5S9CbiSK^5GdthQsu>g&31`Py9mpz)4RLEhw|iba7*bq>%3ly<)1UvXLH zq$G#ARH&Ej&YfOtsJ!sHomD59*b?|KmNBz_{>)LIo0mkQ%(kSe1%REr@t))qHCVM= zr7cuDaSb+TMN?fKepKSk$YqU8Gb=h2jTGOpUueI5v~j!Ykx~4|1V4O=l7+^nBM!;e z!$r+gR&a%HZg>(>sDOdWAOXu*!9%w^| zW5ddLnqCl&+Zymjb;_xPS>NYuHGPIQ(>%>bMNNw z)O1!x$6l?mmcJcWR>9Vw)2W5ZNPNc*cU>le!Z`qQS^ggS8^DkbW+Bt!7_#skNF(C1 zM@j|yxdHz3$HdjZ7Vx)gJ#C?ZAKpg&VU~yNBJ@s zk`VfS#daxe10GOhBq3QR5e!|rcM{wWy@iIEnWnP(WxE(^L(Y3(5hnqU>7YAt4B0y8){ zF`i8UAlk;L8tD&^&-13YNcvNtnn^;bNrraDVa?feQqz`iP6nH{dGQ;O=ozbR$w+i0 zh^L6g=6BR3-G(5pcw*L0&UM=zj>co#VH9!zT17h)h8ED(VQD5=+>_-=ozHp_>^HZH z?P|JmpCIC1?%MYJ>q+Sv!cXjKXH7EM@fRs_ZBcjhee8xDzwgYLbvm+Ndngk``jjTv zp@Ed_*av!*i}y=(8;blLQpC>&rAM=T8Or~}A%>JSZlx4l?ttQa!q zBi3=a%p%FI`2g`ZArz{88mdmB^^fyx@$vD1baZg62dD)Iz_f~J6qyzk6d)@@m5E)5 zlD|f!R^eGfxdOUXVaH7Kee}f#KuJw9jd*>kC;_7Q{@o5Z>7mKT7i?VR8P+Cj!+AI3 z>>Vs#K&h=RC!y6M;PZS;f&Ob7$p?fCKK@ti`&n(1_5wvShHP2f`*dS&Q$rODxJ|cJ z?1O$NH{WF%vNC;xJTHs2YI}^os^9Vu-yy(w^)!S&w3J4|+1|3zA`zY9z1PW>X5m+1 z$_lIK!eEkTa`>EE*Hk{Q_Cxnj*Zd-D{o^msgEh#AE99|dD3SrblLn|578VD&k4|k) z3&gck5%^^5EecxVf}*P{Nhn(!NKZuX;7+apN3Gi1TgQ|**3eGks_tLzDOjXG0Pjfw z-yZ535=4|9!4R9XG>a-f%v65#ZtQdB3>#Tx@7RMR(5JnWi0@pog$U!%pyC^npUei?>|e9t5}ljwXuw%!{xfQ2$+#L=zjnbA6f?} z5-xx&ft;vsp^c|3f+1dj;n~is=a152k;qQytJoblQ4Hp|uzGY0BL;;0sK5q)-GZqJ zfw@@)1!;M|YXwlN?z>(*y0$=A67N+lDY04(bhnX3_B;zx zMInW6(aXOFg#R~YRm8CE!mu7vTiQ~xvg`}n4h9Ua>?{3WIdpqKsK8`|u!aT^ zy=gn+rCBRJH@@>`Go2wUAy-SqcRwyno&q2&AiGo~4#TVEyL_Yb+2@XgjJlT(#`QnK1QSqAp5L&}biDn%<3=`#r0ZeB1CVkB+uZ>^-bQWYQ*D4gJkXb8 z%3V)}^MrEBI#Ajt=mYa0I<e)4UF<`A-En0L{v{Q^Auy+so|T&BLX#@N)PoOI}*$?~~Z z5Vznrw-dH?e;ASAkh|jry6&8j&$?@u3SXqwMy)f^aFfg*%WQFvVFD2)zuOW)aN-~{ zKF8+D2`)>Px4f(0nNPK8BdE3(WhJByM4%uth6gMQl)^&^<_wwF6Y2`clHil23rW>^ zqfs?xV6+mXuzWj|1rdw&q0A8XCjGokJWzH8vie72N~$~W_IbwHGmNW4Yi5vP^o&G} zuq*Z4^a1l~2X zVHf&aokcfld1zGyKC1sh{0v7MzDvlZ-61hyYs@__OL8*)aAs68#et@h2f6{-7|_Ht z95(<2?(pgL1^(70?|Y6g_TKKDkz{k+aQM*tamQo^%(x>iW2YQZnFHu&1K9+BN?pH8%&ZuGM6k>NrbT<>9c!4=Sxmbq8URjK|g z?tufD5rV0~w$s#<7Syw|^44>7EI7?y-fP3kR#jDL8|ND=+%E}_iZZV&Yf_Of)m?4V zvD)~4FMf&RBsDNDeuUH4UqgEdm}HWk7Ml`!jvE2C2fxEPJg-N!xglBgdyqBJU5FL* zs@i@~-^)7AiB=Q4|Gkj<-Nbtl1A!XpOX=G|g&{g|tynsfxkD6!bnBc#6;`bt7#>Vn zx;n!ZfUD{0WIZE*v&z8)o5W=wl5!L0XG4$H`LNoH#)Cdg=>0JO_UX><0E#|!C++VS z4bW}P2_Tf$QrdM!)A+i)w@|m~g@!|bhaQ;D#LJsd10_Sb;T%z-R&%z&)7Vi-+*3}j z=Xl8{Pqh%OM zjML=@oiEk@aUj1xh>r?d=#PKhWRd@;kp5Sr3b<$nBst182Ae)so&R+PIWUeAKoib+ zWVI?v_3v5bPlqcy3TQn=d%ZQ@{rdy{k5m8uesY8b!cCuX*Eu1~aVe_i{`U`m|9#cI zU(66SGfdO8dR66HnOu$4ra+_if8F5A0AEGHk2$7)uVw%3xjDlH?$g=JFY}YX zo`rw^=l_1tFXn!o*DCXWp3?>!2y?TIthe-Uipk>(1@=}QF~AM;24C>}uTcU#?7CMi zd~Yt6q&Q3Nh5k|fQicC@w}BCnNGSj1%SmZTMMXtfRaw)kzYMp3fWMGXK!2YQ5P0{V z>xX)!)trkk+<)KwaDj5+p50PPTI$NGQa@ySJYTOxh*?6l;*pV&mgSOYmM_YsTcDmM z4JH3`3V81z#@4f5U-P!Ma)6fl<>gGw))u)^OW6+`yoi0H^PRamAkA(daGAWE8(o66 zQ?}4wW>{A0??UgNVbIen?t7dAJo7}kM3P~4F+Jd{gruC5q@29W#JmLs6)i2S4Wy*( z>_0%9n!*6fNRjjw8FRx`;VoYZ>V{78a=@?fU&jVC;XjS@0gM-H?JV>ZcqY{Fwk|vz z9gWK_uH!-TQiEdey}L=HRtP*OH~9}^ZO{T4|ZW!OV0z;I$MUnbNC3TT45b<%kyU@1sn-Lx8iQ&lHoBc2qWU zVk<`;H4Dboy=@u`lts2mY%b)(flMEnnw%b;d>|Hu_=TXK)I7dWZqnScY|351-dPRj zr6eD77d*Io1py5m5*Qd`U`iCm4rH7?7nZlvpc_`ER=FUL}@EiT<>qVv4%!kR1>n z78Vw6UKoGCgi8XfXZe6KYfDRu&g%|l3Xbd7%`ux#;0KcZN=i!IPY1XgJQ?24$LsAb zHVz0TP_il*@wS>oL>=hdGlK*OJ%!2tm{B; z+W_n2uGj1F^nw>cj5zvLWpb~F;SWBZK&IZq$!T)s$+#O^ zE7-xNuLD_=cW3J)tvdOemBKX9cS z|GAQ+s)ouVHR@&6b7ha3c!HRvw^y*tk9}oh)Y@NxmfG#nhSAMp=Bmr-`DU|PdO8&~ z-Pq`;A4BZ}^}b)cv2EK{gT`!Zn~fW%v28nPY}+;)Hnwf!?C<|~ z&diy~t4v;G_B_wN@9SD?eU>L0@ZjD7=fEHLL)^)TZ^m;f%gnVL@%Hzr7{?p zJosdPcez@1e~$AHNgfjlAaWe5|3>el9SSq=kj5F`PHuaba`8hRhyT1bgwL@X3j-6F zB|B&s`6F{uc(nMs`8()a3NI%onE4Jz9ilzdjZL|)PDQ%jOl?v;W)I6sm)jYm<8ow6 zOkK^g`I-K;BkGh8xrp_2^B4JdyAm}LWaT&RHfK%{9IFoI&)X-1_AP!<&dO&AOP(SGfFuoR?vVE&zBrPXE4zY5k= zF3bGjhb6=zzgd1)8Av()-*AN_02Z0@-spU=8Wgm=-`%xTxrK*HWcB3phEwmPF>c`+h+JUcDN$I~+7R&m+O5Sj5_Yh? zJ-84)%PNI|3VIHw`i*L)BfSw@0G=NXp? z)pDo90y(@eslqnt0EGw=2{8UbaYh$^@KBn4G%%b$si~nR=o6?U(0ix=66q5}+;qc4 zSs5)uj+RME*o00Tfh0d&fpxK(QFE?T99c~fWi86I0Bw$}Odc;rz6Wm<8==l$8AfHC zInP!TfnPqzGICi#^lyl~`GqnqPr{W(NX0^!=>Qv9$id;}i~3Qe_FUrhl{Pv zcO&YrLqTYRs@>kwKlTeB_;z zF#8J_rkk|)5iB{Ahq<|OO;#o*`3FhdS*!qDF&&VigDX9Q;Gpt#&5+o@-Vh9s95=MH z$w**WZgLA7v*hFYNaza-$jDFFTrZ{x1WbGVg@C?2s1*cLsnF2Sg6K~fXXzhQV82>2 zSH(f3jz|`HhKs@wM$%=*lK7};$>hp{f*{&Gi%$`8*~A*9^agczuB{?xb=B2FYOk4? zagXfR-K8d74|At1NTOLwhtNz}q$bG`sq-o&Gg2Zr2Wi5x`91FKt*8*glL;UuDI+4? zdnbhTvK$xrNX6#TB03A}K*0y03aXS~%X9q+lGvu4SARD0T-8fTcNJfd;fCS7Mo&>O z+6ssJ*_?_C;Z0~vSe46_chO-2TeByL_KQE20+b;P&MT`Ky zbvZm^w&*^@(gbCFD{qCD#hXQ8(yiQ4n6shUnkaT?{Isz^*3LnprJF7%Y0LkcDE*gW z{5nsKT@c{EcK3{9Grz~Xo#3YPxRxBVY+h2jqj#8?dmIhA>AK%1y8f}`{|k+H+yh{L zCeQhJ&96K;ez*%t>DLf{vqZpF91;doo9x-nLSgfPUOOwQMWx{Pu#{1sYpb&@JG>}X52Z6b9h-e{+o z?K3;rzws=p2|#4Ct1Mmx@cSf!O6{G%3Ov<<3^-RrFjWY`kH)O{;}ZON|I~Cn;@t?a zv$L1M_|MDpUI1*+CiL-ygam+y>$6y{79?*(xC+>8+T3n-0mRb4Ynn{>NR)Qm2<|g* zW0=n3NsJ(g03eLK0*!!A28|91>K5S3)abMzZLI=TZxOYP@Cnz0=nA>+K>a-%VQLCB^{4+!^3w;Wd(8n1$u16Xrc1(AL?*t(^2g*7Y9Vn8#vj!KVPj0 zaw>z)w|WE-c>o{1c!&aO0c_fMs=sDU7%CNX=15eTXPB~ykBrWdSt@;K+RT3rauq9ELETs|5iE znOr944J5EsJ`80s8^;KG5&1k4{37)|T`2pHs=z6!HA~*?Pi=tz0(SNEDZvO~4ZO4OdZzHCqmJk#S(1WvBeRC-1fqU@|sjPvt z9YaF+4QpHloI>#01u05o1Nx)d!6d4bszcn2h;Ry0e&=uIv+Nqv?+4&gch|dZJU}N@ zvY@S`Y!t88eq0!gHBvkePX7v~_P3wt_QkoSfT^67b?H5(c+rA`i*D9lZyo&HTC-Sb zvB}7&n-UOxeNb~pR+`B-F3PSwzg^3wXXz1SN z$aOKAx%#UlHpZG(R#r#T*=^F;mZrH@ijZzp@*-?T0x*&8ED(9Y63gj^WjQ$RD`u(j zf);;s0|LNUVS_1xYLIXYt4Klzu3@J{(UzcPg3Q7>pXum9fQ}K5(F7Gi%gW$SrMSKS zB)xxXJZ8XDd)0PCBySY^7fRkp{nj0zJqo9H!u_Yh;B);F1G&#dp;+Dl!L=cYiP8Au z*$+!?p9vzyBJ9=^3iL~Z?T=TS=J0za%vb4fJ(GG2%3s#wY-5DT3HQeJNg|=GlW%24 zmIKw7Dud7K?Jo4v_C7g4t*E|RzfCx75W*7}yirN5T+OhfG^XQQCDgwKA=-4;GY$m) z|9B672T0i{T6y)E%p{o$F(wX{L~HZR&+}P*KfU&gb~>X_rT{N=sm0#ByW{{Ya-^oT zG+YmS_q)8wGuo|dAlX9yss@wG$sFX8G?E!>ECge=fcN)Q8m1PV+iXlJZJ+C4amw_S zwj#3w|94gu5?<=9Q`W~5MlYA&3k2^)Z?1*WPyxHp)rGT7R=_erl$^xd&O_ocapvo4 zp4og4%DF#2CFf!;{2ugNIs9SOTBtnOENT zF7o;UJaaVpvz;HGsmz^^eSny(QM1u#H8p&7$mFxvHDlm)G4PPtJhuU1;R-&?4O}+N zwW*Y>M(^h)*&`+wE_i*@dNd2$p&DF2D(ntf&M8J@z(E3GSm%9xGZZ$t&tM9j6~u2}kJR7O?v!>7}h8 zW5o#*cspkgn+Nct7I@r3-%v$`WFcuZQ3~;{a9cpYQy8?s3I+~fazHn&QP+Tvb2n}C z_d!W*F^AR2Sfm=N#qfBFTHJgvJh%4R9Y7!A1$bq>pwR!$R~j6Sb{B?_qbLkQ{`1lU zXB_ASfZ4|4xV{m%9VJf-jl`V6iTO6z3VB?J$$i|q@oahHWITB{BN%&kvFUjQX@7Gv zmGMo+cz753Kj5Q`r%{eqO>WI!#%>UtaK65uO7N?0<(BYrhSX@cFEQCbPT-cm4||FW zf>pH+9rkG#IFq1cyKQhn&@*KYdqt#;D1N^ae?3E&tQP|yS5(lNt6188y~D^sqO^6k zUmt)mna^gq769VT($of2-V$zCJ49N3tD!!e$_(w3z?aP=z!RtPaBKDG(qYA1Ox=ba zzS3p!H~49Olvk+pZ@N?Uute8@cB%!XR<9TuBKUx^R)6g*?|++B_n`F8dAn$$7R+yc z<`4qCF8InTi zS7r;7mfW)euTXzM%B z=9{2As*uHvYDDTOfq98>@|K%wDnk8Ck?64_j!+y;F<{n8hIiSzhcre(O-UKN8+AJt z-=e~z2KD+JoF*t$+!||NtQ5Dyy~!S* zSZpxKZ4?R0tju>SE> zm2MlwU$kJhoF0-aGU)udbm`9;dGY((1g;T%8e>7@(#39&N~2=BjsJ$+e`=;G` zhxnJOWLw|e@dKI^Ju=yRO+>s|RE@ONO&taLIQ;;HPEyu`Bak zg3?MlqTH#WTqeTJcd0ah4iFl^(&^_|lp6h?SM9baeQ!u<| z=ZgA$U5P8_N8LU&V7B<6S!lIj1j6s1bgtMqcVpQeF?> z#bvWZk0lFA#7zhrW3}HF3RzH;&WqPPP^0TrDLxfGwLr2u`9~v;l%pUOiFP^!E)jzw ziHc<{)xeiff6X-4o$P1{_@p^dm{C-*T1Qb4w*#SJCvJ4}AV4EzL`QuAhsCyCsW{QT zLz$6Z?vYNU9a0fU`b+!nM*(*ZzfG~#CHGSduL!LKuL=nk(k*>uVhdi{OS?3;V>XEDsq&rOrQ!HA;)F`6fK`F0JP7 zVUGn;LtdGRrD}%ui;Vn^U*)Tf!{b}?THoM`>bd?fJ`dPU==+v8_nTlgc0@{8DjNN_ z|7L>Nf5NsLSuUhM#9=G_f}M}yI~XhoCj-weAj7(8m}I`jD+;CVr1pFT5|qCe!Up8I zf3bBeUQJK>m*o8*BKD^dOQ5Ch%po84lvTx;sc$;toDW^wxZh;LVbOV|0 zh5Zr5=l(hip718(&1Kh=6RWS_FI4_31HpDAEqw2Xf#WRCA79op;F`2UlIugJYVyv` z+&LtAv|7P*YTluP?+f6$ z-#|i#WJ&7d5u8hy)KWs%QwDKx)<7=^ECx-G1&pI4g)l5m#c$$T+tN^k{QfsK0mt=<_Ok3mWOHe1rDcnU(rX{nj}$wf+hMr?DQ6fariTnZhhBB zF~pST)a{mtx?=wc!sbE>@af_FFvMhaQ};zfjQs3nb~{*jv47BPDb&rwMs9Ho|CN|5 zfI~7B3MXOR6%U+TdHjY1?%V#zry@NbuVTkPSQmzdlZV$|!P0KxZE zxKT3;n=jD8Hl89i#4dALY!4$K&%IE~r|otLZo`zpZtW+xiu9`vXQ;NC2#FBif_6EZ z<)A8i5(Z|q3_?&>A#-R#ENC{C?IN5^P-RFxW6Sil%P-N4TG8)AC(7=+H!#JS7xNvw z=rwF?PWYCcLzQXG-%8G%FtV4I)S@_#{uLJ5Pu8X zTF67-EU>}-Ag_Sv^4Y(KCveN()`Qt5-A`&nFp`?L?ITLJ7%9c;2Qg<)nPukv(OwuV zgy#=}t_sR(NJ-iNp%o8l>ldw<(qx5j6Fv<_h^Uxsz#8&zKeIYr!XN4vWh*GUN)gPr z00hgI`-+loS$mR0#8(Fs5kUx!pfP`ONe=8^{tj^wMC~SMF!*D|dCgC@^F+SrI}9bict`i9uPJlQbXOh+eUEN z4Tho0ejnnB1V{X0x1yF!>pj<^+9H#s#mGk_g30@a$%F3o!7WZ|tl057sL2wZMlwU4 z$kX$7$?M1hu07GN!p4KR4=mHF_EobsdqazOx79BP?{p{5+%yPKF`HvS8aydT|q$E&5LgYJSdf{3WOdCCMC+L~{i8;~84 z`HAvdT&&gDSPylxu1QsF7ae(kkTm-7|@w%7lm9{ z$ROgdeIh}FLLr^pNOm;SW+Mc#mRFknw}lR&912L47H9x3tv$kSC04N4Z9f>RG|vzV zlz>7mHm!KqwS*hVmis2v_5-CMLXqL&G*&tto_uZ=Raj~=uw9l_Z)A=9sMfs`gE}M7 zG?CiVUGWyCjj%ZmqlhC_&HnZR1_a3Mal*>|At zEm!*e+T%&RfL1*B1vPhOKTZJa!Z~EA5KGz94KI-ghz@Qh=QMe247YNR<~|gMJow{t z*}}sk#@jPvuxuTlmRlJ&)*?FrZqt!yA_-#aC?-0(&*#XMNt$fe`)8t!suc-cGMc;U zjT6;>UDnv-xP}3@x7D}>(1_SJ)|>4GoiA#qD|RbM?hnY2YZm&=d2M(Mi+ z8u8p&7?$plygrMx7yn{r7VYPoy@B|hZrRwRYB`9w)UhIqxL%@SLnq38T%m0_s1>)O zu(dw_TYwc(ry5n7W884RdOMbumL6y9$0v?wUnYeb6b#M>lxEwbWel)Mx)sy(k&dZJ zd3r0aTU@tOl&CVEG2W;fVRi<4-PsJ(jnR_2Qf$?P6q`k;LEPcPrsRZx46_kyj?eA@zUuio7q2 z1TJSXl$&lc;LF=blFIip-ZY@9FK7iz$zbj#$xBJaKXVWr(6Z=BewwST zG0Zhj$$q>}tS$1Vf2vwFY*iR~+dYT)E3;W)HBs1=6Y8y|y;aiwHU!Dc{}y%{PWE>f z<}d$s1>nHd>T-2*Yi%+j!9NWO4#vPtuID8tKL?{Fa-qMgrX)DK9=IBEZzUncZaciqf*&^fMyhnHFIs4+(-GtzFFE&J=~DlffW zZwu|O4}>$K!(;B<=zgCAnk7>vj|%ow$`4V$izhT9uAkigU7FupEmq1;RH)&r;9UDU ztN^Bi6oM+z?4Lm_lAvXr+YKZenwT@)Cq|f z!g@I>*2IAxqK|{|L}7r9B`{XcEkoHPR4nc^!kyNv!v%|iieoXJOWtcuvgkk23T|M<3EL={Qj(aGLBUD^31Jdu`wsfFeOqHsUbp# z!Sy2LDJ_qzY&MTmv7)ff;H{41=@UJpm*(Iz1RQI!T?~rGy^3Og%WVQ=DD}L`&3E}s zPC2IZdPM4c`;WyXp}Q<&PVtR5vqK_v5o|4QJgn&`<&ynAIi1APO2N85`Gx%>&62|4 zxG_5vk^Zcm-TaJvu*j_ErZ)LGrU?eE0N<39^MWnAwL&HtflyDI39e%*mjz`Vt-61! z-xD7L&Rd#2*ck}|&fHRM(c*tv-98HKv~(S_4P#?2?EgePU#>R@8k%l@eH>bWLh)kw zQ9rb z%lwU!fhf}Y^WzY+;vch48`4Y;U(fEYu{_@|$1k6*Ial*3gGR@`{A3$1F30EX0_x}; z+YCmNhX^tPbo8OOmGbd&z&|Uw7HnZqukHWirnL-xBo-GI2-|s0{YNXQbrb*Pv-6j9 z3?eo!mlESz7`}1}@yU6kF-$Eokvd9RHcY3(Xf5HOVpfctzP!B7^`V^+!r!3H1Dyfh zcRS1|FhEL$qECj?AWftUj@p3b8c*WejZ{uC zGh`4k37b(fLMpD@3Sv<>hC@u<5l|ts{AS|KjQH$ZN*E9-8tzfFb-ySj(DAZ!D}ro~ zDym#etBg9aL(xe=`aQi2=7OVgOhduMsWzG}H=wd{&@Df$`*xf@j`-dlR((d^udv7* z`#`oYcxD@Hh;ZaxXMHFE`QMOyabN)Ztrzb04tJm50Z_z*ei%rB5Y5dLHm0IGemj<> z@KbzT`MYfY!6-cB?MfS}CdyB0sQbNb%-%8oG0{iK?FuopEfRc<`hB$9Si3mUBokzY z$n=WcFcT4~$7{wRpTn|xx*1vtq%CMi3otJ1aw$)L!o+1;FxPP%5Ssz-P&EsvB7hMb z`h+B=FRtuZXz2w;^;@NLO3~vz-7BAfn7k=hWt34BCA$-hq!1A=YNMu}8rvu_Z&j-;c%{sHI?&Fu&g4KFueehP}T{3w=uG32RHdEy2M@6l(#zgZQ z-KP;db-NWsot?wTvxpe@Om52CN%(-b`=>PqvW}#WtC#U_6TI$mzk~1babC{o2V%Ha z^MkvWSALq*dJex`)(Di3$zQ4J?ZLsr%i|KbKMtwWGW+J&Krxex<#lxKuRzmdeW9>^ z+#k!>RqEE2-Y1Wg|F-cbXeTQOcteb8zdrq&SHLinITUvRK9fUVYBjQS8@pAi+n(Fp z3SzBCYSibQIH2(EJ~-A>+gF$E@?PCy%iyzXVBd30??WtLsl8&Y(BAV3qRN={h8)#> z{zMuWqIJ8tXhz(%^ZA~=njj({XuLa6oQ?NTL9d&aa_2kK_cYeuX1r+p1YM(Ut7)=F zK#;VM>_M);JA#SX6`SoDanchZ6iG%>(-^knK6T*oR)4#ekfV|^jRnG#aX*VNg6@&QU+60KL%ePPTY6g%|}H?H0O9|cd*R~6n;<9OYHCM8Bg;)XzE{F z=%HJ5^}t0$HDci{d~io^V#$LPElb&vsxAK0zcFvF<*Cx{0ysL}o(rwRWI}>NFv+0w z9l}CjMqrg`#+i1nN?Ac}J~+pFkO|q~*ETHfu^+J9MHqPk_6v&S_%RLjGMbbn$EFrc zqDCeBQJG!0C%;E`u1_7rlszabQWc~FfQVMpS&GiDKIAcHX(k7jvII>A6!7p5Nxe8G zwiRxoX%LOPlF(2gzBE${tTx=l(o_x)StQt>`Th8?Wc_r)#nQXPo`8r;LWeLK@5~7& zC6v4PAH`wW8}yDuZxVLUkON*yh!l*WD==$&8G?xcLLVV8(}AQH$%P;Z4F!VV-G>`7 z(W_v|y{wx9ssMese<~QQ;e@PW&t8X7yD`iYQ7=!aG}(jIZ)&$L`oQ*_4%Efy55M7& zs}o!rgi3J4ylHbl>v{deGn-JTVSgAXb6xOgpSUcy)=fZ!8Vxnf6d%=;~}J zkw+LYBzgkkw4a1{t;ec7_)Cltj)s4*$E;t#VT-h-%gvNxq-*>r+ob#J=s{h}C?0IIA3I-dp$k)~w)et@wY* z2=X4BZ=QS}&Zh}(qJK9P%P#aEDz;rrw-o>V)$RDwzj|Ar+d3#8gu}>GzFxa}H^8NI zWPs|u_(wNI>dhHdLbwaIIc253+i?upC^VM->pXkc^UDH}(`5Ji4h{od>1t#1?Fi3o z%R4z*fccuCFH~F4XX*27hgHKIqY=Z-gqE^un_2eJ2oI*95Aj=xXO;aoSN5A2^=f_X z%v8P1WF_aVsslSg?1h5gB@q!B#`c?vbzKh!!7q!|x#JHN z<(%&rzefsOY~og}t*`l0y6-Jsc$xbdNfWCz&nrt;JF6XsVlUH)hdw?8Nxc)!&MRB` zJcLm#P$7;j_W+<$MTP;w2OYBBCya;X{@8v3&o|h2$MGRKq$=>|Af z0p}XlLn~EN2wz7y(x~V%&jTw6irFB%E)I@U3|DBmkK3~qK#B>Zo@f4*kCNG9v%%^z zP9a|eZmlYQA*xws_|_V0Ta39huqtrckQVu%2S>5>zI}r#Y-kAcy!8u!Gty4j;{tkJ z(heKAF~qjG?6-XsP)%HNyF*(?Iu2OR>rIxJ)a|%os9T52_?fh2(_OE`K!?sTF*JRD zia5`WqRRjuWd7EpF{|ZmzkT=6dV<;Gf2j#)Oui*U=l})hV!{OZw9`G{aFS> zqOX>vkgMUfD51eyd;FjT_pOBtiI57gz+h7kG2+^0F~o4@!lj+Ux>*K)};MHL>(L^YX*VX&&9980W{6>taAI9?q3g~!v(qZ$nj4_RC zr3b^fL_4y+v9SZjI1qHh!I?2(Vfi+A-3*{BjSq1yvI0FJb_A`RrUl=Q+R_xEZx2I| zx(BS`9~qPoeEPrBq@+`l>UXQwx5lva>IKQ}I>9(GNl8T;XO0dSHiL8=*k;@mC{>4l z+x}5UCRJJrh|@s`g~i}WM#Pb|_8QJMD`yJ*H~Dt$8!lMfWFf zFZ`-iU$n2|I{h)+5e6hl;KMNPPCZYK&#crza^IJXd80OUv!y>h6<*$Ib|v65Jsh7M zN+6yt?Z)ghqcbb0Y&~cBxkiG6gB5tjVjxU5J|l*lWG=thojFQnp2FCtkqLRGF>$rB zn+N#M66cPH-~?aYG$fBVA6J}c%UZ4<@##{hkfqPj+%L_HOsIsH%YR?pdp(ICB)Atb z3qpO1=-xe~iu%2+E$jfRro+{4?t{kp(t2%K0`(+iBpO-%Cj~gR<2)vEy8oO8bUdLQ z#LWOgFO&u8fW(ii_la$P(ez2Z>}EmTU(dpZ?%x-B6-7usWWap zRu|=IXLj&}yko|-G+(`O%YIN0qajaWr z$lIP?)lGYk9R+rRig%2$k0#<8yOKne=U*Q+iVMA$69CeSdyH*ARQr4if6=|tl-7S7?W0_kwN_FYY;2Gme3XbuLHp{ zeS#Vt0M?D%lrceIm3& zdk<1B4bDR8MEwN$lN4T~3?CzwAv%EjWD;xY2q$Rt@gHk1SX;>OS$-dRT^Hdaorl&* z1yuqq7zO7Tg%A3*k3fk}q|`WqWST>pG!b8%{e?ktK+?`tTPZFtLpqAk5EfbtGHGF~6C=7O4H3Txc?2P}22w*5)ln7$ zaL4NoDr)M$JQl8A)*#86DwLjf~dywo*{UnI^8)3V$+gj724p&mw zt=nj=QeSq*p77w>@aI)+ZtkzHy~|g-Ww%4u)sbg;RO*k;2^{Rr-X#*jxi8HoE#YD zCWB?>BcBhTIF1`ld-A_sPLy7s1+5&}_#1i*+lv($o~Z3y>6p!o&DSqtHRpK`aD1Hy zrOYum9i@h1o%rB;BDwDZNOTJvuM4hQE7TdoUnV}{Oa3ss1MoQH)77f`w$O^Ds&c(9 zhQp9G!&sNpm<*!tHVB_H8Iv3{>otsYn$f|1iht|5awMpB@9&K5x;>evM{)uT>hAN_ z=rtH=zpkT8BWw14)0E$QiykxsR-dq?o zTrcqPxsvLm-C?;gyZ~O7GRqfg;a8f*L2f8@2aC%NHY&LaLm+wakZkrskb0eYV~rV_ z89z8BLV!f2&vqL!!bWq1HjG2=@EKM+#%hKynHIt;gR3?jAdwKXeA0QEIm|gG)}WF% zyQfpwO%So?W&PY69?A=kc+AB$QTEra%QPdzq_!cq4zLoiqUjsw<7!iTGoXhT>6=%` zIWXU4Nq)B$3v5UM$958_+iLWOq^{+FBcdtFN|79(B;G6PdQQv>doH?hcO2vJ>Pdcu zyS`weVVEABHj8xsWNWyCgNw|nea7OE{Id z4eZ0Uaz2{6Mp7W$_W3ZPHG(vrwOrwKNWHYUsy&n7zHdsHsC>IizSO2FhpN-vPQrW? zu*XjQG$vq+aL339`b$i`UG3`mvQ|L;1yRR$a)*I_sDC_L$7woS1al~M!$d|$P@l$P zc$o>(_amFqB)RO2rK*v@6$zdD&hFFpFy6af0aq+kKs=C-ys=wFk3`8v}tBkD!A zEgo>c+tiq$+$ziAzg(<`TdSA!YTt}1lKu;6|A&&Ip038XkvVIDgaF^?*C+Ai+sH!Z zojtr!y_B6`psmY4ez}kGp|=@GkJe@X%h6s2REFr!hlHXoV^dTb6n~?sIf|9ROy1cs z>hjdC)6ei4T_b!6-ecKG;sAgxapdXFpfL6awNh`x8*i zj?uyB1&R@i_kfiNXM0EU`~zrSb?m*It$=m$AojOz^<4$Dv>kSRsmx)_StRoA^=2)kDh+X9!}j8CP<5Tz~55 zwBM=p!(yi@>geOyv~`B95=~w-3RV$w_vsljFevRk4gaq2Ft>VmoLY*tIyWuWa>W0R zdICY2M5d%d=sF22oZaElPyITNRxS$e!$OZkoAavTV|ftf7(p^)$0NXMvu;|U|3fhF ztlnzxRNe?R{zx|fs=VP-WPo8Njh?1ZLHhU@wy9_+*;;+swvPDY%?FW2?aJTO%xv_U zRTwtAYIix1by9KYh4>H3dIPgS*J%_-N9*DPmK!rT&t&g3@p`YmPq=>FTV zHK8Y2i(U}&#BaRsR{00j9Fk}a91Qs?&I}wga24c&n%6)4u9)@YM9|6^yq*VgPRk=8 z8}iCfN3xkvy(^mn79v5L@!5@EarYNY3)SrO>=w_2>|$|WYk=%jBQAuSqqw+O?O#9Y z)S6l6InpIpvA3_ilfwsDG5kKgG4o-#6YreV?r%*WS?UjeZQtyz;7@<}8EF5ThJ9aT zo}1JtRSHQ)&{VqM9lS@o#tXo=_JSX$-12(%%Y!IxEyH8S309HDApsT9%pAEFnH|6o z?hitP50a0D^=I%A)o&yf${5fLJJ83BC!ZYDswrg5Jg3baMnfxQTa`sEp|CV}Am5V> zgF+xz8UJ1*y;^O~e8L8nz}&?0a1ZnwqDPReM&ZC_3{$_bfhx;qt7V2{25C45YjXTE zM+Rb55?VQI3-mv z+g7q2)tzLz#^kvL7Ha?}k58uEJ#+$$d2|-x4GqdpAic>ff`(X)wsV75Rm9Lbf%+W<+hnt_62IKW%0+% zk`PbwNC6j`J%e4Vl`8rOh{SuJ_qH-?IR)QMux%JioybgUsOUF#^M{S;Pc6C6Cos`< z&X~vvB8Mai&7O`ewheMik)%&|;WiuagGO&3{4Dx7K_k*i4@;}-x!LWAU2SVKnw}wWxpd-2JTbrme)f__j$0oy*FaGB#}kuj5%~M&m&H$dHww9T$od>6bll+n;fc zpkXg5r{pu#e-$RIJwntL0XWDZdi=*lwkzi6E8|N3gO3T}|L{X(-?pKoJUDkyFemx3 z_sR(kCJ{KIBK)lGR=gz~Tp-3PMTcoa&83_<3j(&3Th(XM1BRG_bWheKDLxowL=T_e zT#FIy&Ft}bJ{Fg;oUAD9jOvQ^l#0+7ZAEJ#pBV?EE4rOvEManxJq*uigqs=E=%8QD ze6ddyUH)BY=XEewYFBN58q00qZ2r(jkA!U$n&xofJo8bxA3N4{jI>`L?3RDOLwXyx z;3Awl$54wQHS4KgcX3c3;+#}U&eB0n3-$^*VAqy8CGj+j#)c)2ee@p&7E0|xuuS(T z++VAs4Fs7m&{u!j~Z7U`T5^5En0U zW}2Oi&m*-N-8oLg+hRMEa|aeCLq*)^CcAho55{}e){K<@mmSV?b%uZldO^7h{H8sa zMPVgKUsJd789I5}Mys2zKbyudySqHE&a}CtqidY9-?tuUbWDv!bu-!%B}M?P^^;dYN08 zf9E4L``Qmtx8dtm6sXk7N^B?G^A|}Sb zM#eT%w_SXk%E2Xu(r;7h+=8GA(2=vG1(g>?;_XcAnB1+N4*UrM&+Y@gWIKD(S!(&8 zNzDo7x`29}qf13eDt%1_*BB8E?4rU8f8v(_f`n%O-~bb;0F-Sv+XdSXAZyfFwOXnk z4KiR6V9WLmI+^n}nh>w-7A4s%mb-A-f!SGJxP@{p%Qt69YQ z$vOe0ZMNVCyHa|yWP#U9)jmoFW(%XO`7ibi-qk0k9>XPi!&s^7K$igTgQKOx1I>BD zDwC}~IDhkHHDm+gU!qj0Y$@R~V)>k06)Mq{{tnNB433&AiJ6vme`+;iyXuU--5};O zwKKJ97>IRlMx*nEc*Da2XlN_jHgX!A8gluzjLG|2*pb0?XHUs-loW2Zk5#gt!ortY ze^5Uc@cj>KsnZ@(Nb}D1aad%m1T#{ zuA1b@DrR3jRQ+I-m(^CtpdqRnN&8tBWQRx8^HJ^?vD0IF#^Kg;Vao~X$X`_D(41=Q`x~;3rexefMBeYw=#LUgC8qH`^F2~NC>aCSGu#3KmX*&^4)k3i0 zV)PfoES2|Lto4{@;;$EQO*~q`-<3DaGtwPD2;jHqN3sb)o7%t`B>oZQ#(m^O;9p<0 zHlH!-z`m=0%;LQ35oUD|*6V2aiMGx_#W2AwN0~DJZSX*BQ+F=tzZ22{qF+Sx*?KRU z+w|~?*LycEJr9!}j1V4lc&S!kEAUB&fDk^id0xBCz;3gxvcE$EwVw& ze$`3ABZ4vb+SCDY*RKN=u}{OCrN=X-$4X|;FSVIY ztrW{g`jYk;iAHLY+0{8TEP6JzZ>2I^6;_>2I0^yMdQ3B&cDAXx57QOt$eYIsIY@gC zUSDd9)vE9ip5IG@!C^((ify51Zf7>jXlbnu%hmn|$gsr%SR%HMOJzH@1fWLV`hRRY zMKuUFSw5`Vfc~%u7uV6@-PzY+g5&j^-ql{$2u8>?$|IPzeG!b@sFvPshqwm zWMynEmwJ{$R9%MR>$)hgfmUEq@U+&x;@@F_pp;gsRAc0 z^cY#n_kGX1uIZuoeCd%z<&PV0to%QG#RV*=OG$G{N13hC|Hw5KR1h}hOSH?5WiPWZ zL&hKYvI9?kkD96HV_`{EW&V5OpSSt-8bnVFS$f{{kUJwecnNG_8%T8OOyy~JYH~iV z_E+7rsq4oP(de!tj|fUV?d7*6MK%E%x7%BM@IOPN1;5i!HPg8yzC}KTG5;r%t+NZ( zAvD-J34;a(kj=EEwPq*ZnJ4q#2_4i!8ljied&STeCtIg#OlA&|wO^lbteKffNE*L9F`+vj5!EMIMfYCcSV ziWZ+kD5v0dIfT$`T8_0Ppdlri%r%LJ~i zLj2E@kDP?sD`kd)<&SjkuYQ?Ln*zIi)oMN15vRJ#Bsqc&v+A_xxB-cwO)vA~v~VnB zj0C7WJXWg9=FmvuS5KEy2|zgX_}Mw5wwqJi<8}V$%4vuysrxahsW$E;l_1F&dg6Hb zxr3O#`>Q4yzUOZ8y>Yh#7b36D)v=}$=5&tS=2WWAw|!<+*%%qVyrID~Hd_F3B~fXu zthF`+D;eWzY~{#6N8y*Pk^g@EhQWE&6XvFi$wSEEC9?jV*Gtag+-k;``4t~XL3zDE zPP)_;B1}-i-y+yBLt(Mnr{?PRYWnJ5E*%vezqXpZDs(pVJg;a{FyZN{F+m}PGn`?v zgYHceWXHXFxS*M0$4t{y*%I%f2E3oL0j2uu4Twj4O*2jp?(b(_8iJH~PMa((D_I|j zZh=l0y=6$lt(5rqM!keOADCgKErQk$=M;PI>$DpXa30%AcfP;U)C_#AgFl?9*A3;* z{5}Q!m$U0MOvI|~IBP5Gnn5?;1?lb9@PSYzML{k#qp6&A9Aqgdm<;hsLW6Q5>_@22#j zea#Ww{pFnJKMir6XSuODj6IZ^+iRO8uLu!s6}WFJm_L-JgVGE`5(cuIIIBArO%-cN z6)^9@2@{3k`k?dd|Oqoo!X9@n(7YFdc4=v!l%XSw5^}| zhkGM{0}lIc&C}t$X}<^6=x#c@Hz89{+p#X|>zz9@##Q>C-X%W3FFHh9otbwT&C5tZ zXQe2xs7XS!)ZzDWmsz9`^+!?I>IlH2+H0=`BL~=p>1rs$k9R^XGN#bqubQrb1cb5J~b` zXj~gEl#Oo1jfjF27DTM3>3WH5dkNgcV9frJFY``Wdi6KyE$p>ElmqN5(s>ko$IIla zlihGiu&$h%0e<@=Z!CyLjOjo0eV`OJf$S5J>D+Kq{&eSxw>p7Rm9@dZ47t7PG(Ud1 zmeU>_+hkETvnPFZoNO{z(1dGglu42N{0CArNrt9eV-#C?Gl@`T&@<_vN&`I8DMRT5 zg3U_D_#xJQEuJ5V3~4U1&i3*$iK#@A;=H}tEVlZ?=h>%)LiX^k#O+KLlB(bMM zafUxF{zuyLmuv=ncH%2$Y#6M<^WSg#AKTmBLGWtUKFrD7HiQLVwB@K+$NxDe{wFu7 zk?{4_J6IiBXoh?Fq$_$I2VukVe>hYAM1DX|OQbw(%RpK!Y-wb!(Ugc(aP>bf=6}B2 zA5IiNzCmeNP`!orHL|rOY{zvAu+#X@#_%7~ObQYBQYpmWCpmRwwB>bmG<y8||2J$-oEd@_mOg+^GqM;+IsMPdy}m;l z0fZx%0SVLpQ>WJ!413)s>9A)#^|y=qpTFRufI4@#TK;pB{@3!7dWI_dS2G;|CQSVj-5k-M?ZTn zat-CBFFxrg%!zwvn-D(yeaIpH5&xn_1F&l&yuWL-L5aVOTJcu4beU1>LhkZ^P(T+5 z2tnW6mh$|=Y`%`BxtJg`dxcL&W6q|MQ5va6RNYmyz){Wp2Dgu0&PiOVUW=4#1;`a& zqxHa_VF5?Vd!h5wkiAI4vICWD9nCIq8BOmM+GaUyynXPd8VPl%UReL%zXA3SAQ7d6 zX=)j_PMtZ3a2*hr;>!}K_^d{|*o-f!_8&SeKGKhaeO9|=|0$hEi910xjb;DZ9i8!^ zn2k->0!7kpxXJ9b6&2xxA?Ro;j04WPiz#8)8siao36Ud_+A_n#;fH15x1dBGJ_5XP z!AJI{f7=Sx7;ZAdi8o^;q5>CBNAs7obXa*+I4%dt){)#Ns&KdpYlaw7)OomBk3%^* zI$}SW@N-9i_&miZr3)M0OD>T88_&(wLjtpdRe*t~G9G@-y(l2hv?$@<11Vfc28Pmj z6My?&=&mK?x`IHatP*v{_M?33Vf!}GEVL=<6Y}H*4L}S8+pVhXNa#1HXPYrKT^_Y` zfX^rAVaa<-RnSgJxtu)9<{cX_d(1V@jhM%`6b#x7*;_ zdol|;i2-M9i(R|n5EmctSn-mM2XgQ%CfOW9eES%Cs_6_Ulmo(Vt9D#!F!Wev0 z-y+5tFPoV|=qK2=jFKCjD-^;vZrDwp2-_aFNO1s^d`UmM*n-1#AuhYMed+TmEVZ98 ze0vG4rX`XboZ?>_CnoLuG(_A<9Ttm;z-1|GjA%?QaWL$$e^K-6j8qp~xrqrNT_c?X zH}G>(i+K{po?=Nu-#P2I?ZyXuW8}8;e2B6dZwbH!tB;nyr51gu)bO;Oz#OYBZq7EA z`b4C(?&ni0veLgv>vf``sr&+#!Ev(Lr-GPn|7UVv-~0*VvakzA0z2^>Vo`X9N+nH z-KW1NdVI4q1m=TlPT^=xi?q=M4&B^TT6w3mu&@E0h^z?|Tx}dh7Qs4)_YcbOujIo| zih=tdk10G(-LjwOmuEOGLa+RFzN;E(x9(qFMa>pSBHC)7bz^p3-hlb$CN=l9iNki! z_+o)|d3e1EgNfgn_WvN@1ECxrBLO#yZ|s zSK}O;!KPgRsrZ$wAQ_E!4P<#~IGg)I<+Qz_D0ov_*ZB{%nVi^GW1nc{Do-A$jH{EGI|o-WE2#}fvmlt z@AD&|YqR|^_5}dPw4L>R=PcGe2;JSh@_M>ncqMKi^8MZ~nBe_Y3FvYGS1xA45AA}(al}xH*VRtX+0Xjk!r7&zrQvv@ z^T)d%tu=k3?1w=F?}FgONv(uV^yRdm;7fj)YwFrF?6J$Mq7xaNp(( z$W~am)0prsVNY$-fjKweYu7#GgY8;pq>@@M%t|i5QG)WaxRtzEAuI_(XaW;VOJS8q z2X#<{u;`i88US*L@?wuy&UM@w5P?4T+xz*m;H!sY`Y*(=wD7lkKNk2m12#PLm6OCA&2Sd12BRe?C0lcHM8E+14hU$^*1`Erep$X0P}O(F^3WW z2@V{t^lUCbLhy4lYL4SG&34+ohiSuT?l-glEWb7Vb=RJemF_)T!; zh*`t$-<~piny{9_h)8;v9zranJj$dzPMF3faJeIC5rHroK3|PwG&JTe_C^<~bX(j` z4XR0%M3Hv^ukySc)3-*q({-^BM3LlgZ}(FJFv8KR6F`i_9`Jg45|X`8sA6eD&4gwQ zxia%+vjoO8f;^-$F=kSW*;y>HjS%r^un_3Xy`ZC&me`KpR|n3kj)PnD=9TmaHLcqS zi^Sl1!Io_1EjAcx(X>M2=t`mqx5q2Dz=gctY>O`_`Ssmyxh2Mfgl>1e1H6D9>mKRH z3ku;5^~6<}<1*yUno_4r_A>!?63Xvy^_;d+aU!}!ia&jFN3P1CWuu&9v}#f4ix2UO zg`Kz8I`-LphwRtGN3ivfF=cAKKcYTIglE!5G2}WmMHlt{nRFvaExUtd?KPz)B2tu z#r{Y9)9aa<62chZxg0rE9kUq^94}_M&6D%05yqJ>>&tm)ApG1|LFypqCu zjAc-I@o~9HCdQ;34Gsr&^x+=4-A^CiSUsH6OIuu0Il1<&h#%|nP_IGbjXJC{U~)e* z4vuFDQotIAG#k;}D%JTFza>!(X_YSawvHvc?a=GQl9sXFHOkVuP~d`-gTiM|=Ys_& zL-2R>s723(fF?h`IO@CUYJrEtL;gg{7=a)A1tYYyx~LmUNHa4DXt_^?c||il>cZ)I zk-WSFcj)&6gcyWu#~{;?U6jQJw|6xgDUHE6v34K(GNd-bjfp-x*XIQ{uE|3PLw+ug z6}snVp)np|F~yyw<&_CwL4;ypWaP8En92?bp^(!QS)5dRuM&)t>g^0w$pvkySfV5_ z|FPja*P22`i~V8;QYcmN?MX5;7g553p@H*&1V4FuL*=K-5YePW9Ps(-cvfQeHo-Lun|Odma}yb|z=bUS8zD`r(-JCP3|WH#6?a^n9nDGdo~k*_)J43B`1HonwGm;(z34ieR+-| z>HuOc@r~)IhGe}Ik3!1y3AxenU*u85h5-p;*G*GR9ay-PY#bHIAYDyIz2#IsKnY7c z-RubjV4mpw>R~Eah(?as@0F#y3y*DJAG%4kjsxN>k zsVoi0j%-o4vpa#O$`nQ?69yw7@;(zmy*t$Gir9%ia5eNkhmIKphlC-PIhVjp$a_bg zCypa80ibsL@1yB=hFfK2gJkq1F#Y@<0rt;?(dhQ}c4K2B8kuA*>KjSfm^GhPXcGiY zQ%Z8Oxf9iMRm*y>G=iM1szC+9=M4@jrR=dgC6?}*+3px5Kv&&=?haJN8IxrsK+Ngp zXbHf8fjCP^Pqbmqw@WZ7aKaINMqi$9yK;PfTfLIaFuwrL%=oAILqkldKfY&0%3T7Q z_DyV$v!yyy*jnZ{F3XKpJ~u9#H2Mf{Mv!Mq%iGu>_yG{+^-<_AL63VGgRWJR z)pWu@a-x14IT5Gdf62TCo0a#Y0c^L8E|Rz`ZAI_3G8Dw?60Vj{D|RVKa4vP2j?INz@vb><1tJ=a~mq|d`=@<0zp zgAN>xeAM#r1jDC8O=|Y#SJ4hu;C(-n4>%fYJUR}WMHD#`6H^P{Nz@`J9OG-y^D9Gf z2tX>P(Uzov{B2*cD@J6J*9;yUvzo1Thtsu-Ok$rcZc-|12K5fYNm`Qm1Iq*803wkj?E*N`Kg|7$MK{sn z2#oJmeSA)-h91RG1cH3X21=mHe#Hts8d^Ab5t`g`v= z-Is^myg@_n>+xG!x(d`3Hz;tGCNHHa0hICVhaX<8J5lA6kjWZzDEv3Q4bJOYmwN+M z(0p5&S2fwbR_l zq5EMac`}hsP#kI(!8&cG`fG=k%Ak8w@a}0pnrq#N1mhyOJP);QJcA44GP%Y(SPcid zGC|%dJI|txu)8F4u_{MO-FT6AF;RvFGPen@^^%EP2$wx>{ z^&`7^Igj-^gFcwJ74z@zyg}9=N>74bDdnW_JJ1RA&prf8$ za8wwMPlxIWmutwjX|N`DS4M?^Mm8CyD^`j8b{T~xik3C$UlstGRXB)s$kyH~}xl49Yv_ z{M*T4`OE4_6}2-txsIT)orRMAYEt5kB5#(z-D+5TlwXPh8aDw1dpJKe_y$D6>{jlR zTP@ZCs6YxcHN}2W-LE7DO}Y|oxy9|~sw^?FYEO&-Nl>9jo$z~#Sam@hlUrm;rI9<; z@4TcAI;tqX2!{4JjKGJVKg^AD~(O2mBFe-2whvDegrO85nl1Zk9h+VNVCe>Az zabqOjOq9pDS3Ihy7*=nHmhVtTj}9MDoC88dLZN0Dh~ae{;L-XreP<7P$D~v$m)RDFkD$IvmPJpPV5>i(IDZ}rAIDbHU&Km1^J zf}Jch6F6OH(*=U(G(xz{BWgk#Ar$zDU}O;MTj9c zPrRo7r{ux}L+4~aP&aeG_MkpFmZ;77zR6fi=QRTe3TmKfM1h*C`D^N{V-+uJm1I@U zEaKrU1v<(W_LV!srAIH^3t(iWcoT#>C2Qt<8C3{>=KK4=3(NS&_HCvZ(xNBg?B~7wbEY@mHv-uDsxq=!?p84wp^~ z28hvW2~{YaaKFYz26yPH7NOBK0S>PdGNo^b941?nKzjb=w6FgTB`52WQjA@O5ic%D z_~BasilE@nx)Q^Cqn-qc(f2#(-nW4g4TkBrXNgSTu1^N(!FEOg8C*kA#k1pSB4AvN z_vW%)cReMW=(_pw;OYe@wzdAU-_U;Q>p58cEpr(~o#mt1ci#<>uT5koD(CCH+iPC8 zn5jusdJS98QaL$%r=JZ1$VCzuxSl_~uuXhpx9=~%=5sq{ps6ao;+dE4fp?h|iDB@7 z9`0R8{pfjFb=!4YV92Cp`6gVIt43u(R zPz{QGthnsCFM0XlwWB4!u%5eB0ja5Bm!>GYn^0)EsrP!Gm zu2PrLp>c+Z2)4e!xsfXd!$km`dv_y<0wA4{@0P^GVOvtcaE=Hsb+Ns{n=01ZjxelB z3Ej}xdVV#Ceca42Mk0v4kW#(=Fy<1H$WXau#8J2=@YfRLiHLQmxZ#a2P8y0R``mX- zyI5*@@$^)IY3fjF6!CuHJ((yGFi3ep^rgxKv2eZUG|)E%s0-DrL7A)s*gI&H?HkWG zEz%Ljt0c3?C97W?EFf-n0AebKAxqrb%iU@*zgRTaI@G?-3@(KF7Tdw>=>(^F_|0$M z&v5ht%F1MRP38z+h(E0T@Dyi}@Uwyq!~%V~O-W-!KFbu1t_=W=d*q0v=q zfZlk7kB*1O>LQ4R-q!O7#KKmrz;im5$i}A1Ww-vk*EYTj;)|AtMc$7LL^h;2<`>i6 zQY2DZ8$t_Y@bZQQ=Xq2-j`t?rEgIn4pW}Ll{rdT&p9?qY_wLcxQ{O8Z3+9sCsTOu* zU|xjH1XL?BH+S;#8PK7P+6&!o<0Ile?I#uiS_JAO`nDg~Lga(MAglK}Pl@35(dfI+ z7iFh(f6M#xdb)g) z9siOB##sm=h)6NNsXWB3z6uKtd}LVgwo^-H`R+rkhcx`FSZ3CX(R^Z%*z}n=no(MQ zt(5oYyR0EX8CLU4$7=L}EwKWblc4Y(G^nC@9K}86-`YGOG`5H(;#8e{S8-F|WqmJn zo^gS{FWxNq`F&Zfi(cI__4Ctk;Ex#4n2JP6WDW(BU||8vAJt(Eomiw20hjr%ECX@W zH8`n!v=!$`;s?HOb}OWX@paQcEP-`I8gqJ^!R&-U6Ss3zOTnwtgop)#EGUJ!*_Z@8 zX(bpkBpOj0gVL}ar^J%;79JuYUVnfo3XF5dz=E6>hAjlc^sE@6LOhUq>j45&R;N>T zvz!9n&mkNKbyF~CvRAPmNv=?DWOP()dUxnJC?2zLOh{0nC zvhR-@=sOJm2+Z-u>vst>x?BxeZJyQDs=~2V7S{C%N#Y0T#&YoV1Z!)7X0Sgf;yw9n zMT3aNgtC7Gh8s+4W21(1=}DM?pvBYV9;Z35n+M>OEPTJ5@F~Prey{hqw7>}GH}iv; z1?2{ZhlUyK{o3+v>B^xRZhk2WS#yjUi7{wfq zGoHrlqo%yjOE8HTGxi;RQ>=iFWzK>}o@_~m2rYqFotBz9Nb}`lR0BCgwprXgJwu9? zb(Bu5YL{?RPUUGttY|1SY9mI zjmn3=F)uR~`Z1uEh84Arn>NPKPL4UAvs89t^i#M&EX`zs;L#63Qp~2+QL)@cPx|`tw3-h*Q>_!kpea z=Uysn?|mO1mr>+5quc`T34E#lEoLWVgA4!$uv8heIzHO;FurFDMv(Q!e(XK3#Qws; zKWGw|DF}eGt1TB!1cEzIyPn4Bw*2)>EDC0UmRXo zdthuMX>0@1{O^s8!{4^u=>_8=5TaWIGmpjPeZHYTNZm0vk-+jI-`Zq=7Vz3i9BV45 zv7b(Wwq@y}ZnPl|5d#Hb3P&3uf7BBpuL#*z%O_?g_4WjJ<{DNo@Bu$U!_^TJGN$jL z-Hl$NtXh-CSPJnYrnjkpl4o=<#=qXC^0NhDEM>Q_#4O+UAkj z1nb9u2}7r5SQA-MjxD%ST|^*H=21jXf~Y)}c^D0&ATn*z1Zr!#@s9+~T*WeEHFHYr zo}NtfvkKXVB0pn9)6i~D8ye{oV>u`!){-1Yg)g1vg9@U+4bxP78e~K4gnFj)05$&^ zxDn7zaNkiA^G_Z`IBE7rnJC>yRwhid9={l&^X$zh08Cp+zxPRv7dD!tEbJv8a|gF| zGn!9~gv>}Od?waODQA1ygCJP$3|cc(B-H2=T(hbWL-ha?S^a;D9$-kLh!a-5p9YNzF=w zK^KvM;$F_F0fK($$Q1JrA7Xf{!-mdijT^k?XU`cLJRRI44d=5Qt-U|T39Q?E$YIG! zJs7R|E~lXI<@#DqDw+VnXIs$bRQ+H9E@W7| z;iLZ~jJF_nst=&2i6Eaj-Ypl%3xZ@tAd{rkR!+kCG36**cAqRygKZ$oXxv;mg>J$U z{k#c&VFDs!&ET{RfDo~yurYyCmLlzD8a3T!Bug03krnrgueh@a2YtZdFW@w!z&?z8 zfT9h9O=~g`$K(PuIA@`NBOC`@tdBVj@)FdW!dm1*gO)?WtrfWs*l@Q`OddpVD0LHX z7U-T;CpkxGDBYHUH^;EIFSc^OSFmE27Z)u(;k_%FZxU(Nu%F>@nxyLo5TRpZ8#)F( z#fc%#gflVwB{W{X1|n(Tz>h;7&=qs=u|Gf3>G3=j;Nj=%Epvr+k1UWjfrGnh8@0d; z$ev9lI9}5;2cV}k?IhN*Cj*lsBc($f{$GP7=&%_LfS;uju1I&jn`1it1|K#$S4D2m z!ph;{-}&x8H>DL*1I?#*ObX~Kta12pu`zb6NFwQq?0BBs;f?+9vs(0~u^X&wA|~Zok&k)`&{nZc&y?at^sJ=gQ zD~_+!mX@ZOn>zhIGQ@3~i8}0zX!?5E`KcfUXsB#oBKv)eZw`1O9V}5brjz=B8Kgi` z8{#ea_S6^wo3R}?;*wiTXDi4#{C!}pp)M{6=V0dN-d+#q7vmuVIQoD97#?Y9VV!(G z*+Q9s`%Nj)gij)y{)@xlHF0GjH$jpNjfsr}Hx=lTA?`j)I)qJQ`ernQ17ailvfCuX z^gH3qYNuubs-zjeteg>3*w|eFkIPt$>2)eDo5dY{lRO2PL4GEyc6=62Zdud>2wIk3 zBE<2$IA~xeEm(NF%%%aU2g50cSt)_TiMUUhquZ>UNDj#u^NUHidIhye8dk*rwoi?B1|}y-s7vU=VUn*;kKbKt zrZ!<_%Y-%;yT^@EpkLj(_%s^10?kfv1E;(*>uLJ=`pJ#cPWx0^ws@pUbV{k0g4@pN z5f@vB#T^cKP50!7;HPWsE84?+$M8E7{B6(yZnw83Ew)c{Km~i+w|Z9Z3@+}0XxyIm zX#HnF?T#@n#-s-7kPRjN-=*+U?csR91CD> zPHmQdZk~zFN%wAKf9n_I^9a)imPawSn8pw#1A#;;CbX@Tz7WLaM)T12W*8I_NbV;C z2h!m&Ni!^;w#;h)?~fN%!6d}hsfne{JmIh$o|wX@`_1#X7#KySf?sbQyJ{>OR5IeG zh9JjfuifA&QPQpV{?zk*tzyB}g{MkIXN<$hdOKW=awt`5n9luR**E#O=iT{I$prTS zXou=c?}drr=N~fE*f#fj-PtYcPhtHp)j6Le9xjp8kYu6z8n zy;-(-+AWkQwlXRv2!uSS+;E-{_;_lup`LaBsqxKS0qUh3(V2QWXBdU-qE(!akX+!l zKYK4bm;A6;H;5fYc-jmsZw#YP5t&q5cv=bPuot{5cr?Cg_k^mCX88Wwk93u)Nqt1)&7f5}0t(F)u)}cU84l3y-#^KYj?Ra*E^ICWQ%-C)6(h8K`^KLM`7P)X zAr;JzPJneUqmnU4Z7~#j{U1}>JU0DdOGmc)LK`d){go}XD(J=2F1}}Qp+HD7ew5fx z)tkKhMnhjdfJOFkt;Y_qM17wi^2lQLMUfpIQtjl%u?=7VrKvhVGNUIWA=0qEBQ`73 z8q1~iy=~u!hiW0Ict~p`#4#FD7WLLMZ^}3fiw+kIf>^FEyZs8KrmDt!W>H(Vuo*Au z2Q$7`?seO=%w_08PEH75SM$>d@p9ta54bYR4(8oe3}+k*ZRZ{_{mG*I*aJU5T(w<2 z&Ya=(fLZZa9kWErImtTHns^KHx%Te&1Uh}e8+4)xDv<7oFYwYz#gaM%8}&m^ZA|c{(x3ReO(I%>|5jl(1sX z5%5|W8;OoWE>|w<22R4qM(AqIN;(M*yQ8VkSm8SMut5rTwnWKEF>)81WZml3Sm zW2ehut)+_^2b;0viAqags4M~pp9opRV>Co+_ikWw3VB-ULnJ3w9Vj?V4nal>wiAQE zGHT+Lm`b8Jt#@H_ukOwE>xff^Kb2e!2Lb^Xsnv3d^bVg+MihVZW7H?JH{c;Jb&dgJ zB!$KoYSNGhN;teO)R5v!noq?Ub^*oPf+k`-9Uf;~=Re~4q9BBb8Y7Y=&<3RMASr@n zjY(4OtSmU*HnRk2Z-tw%xy@G>ZlbakH=f%OW4m%<{|IrKzM$)8p}@@2qP9%CBL*j* zf**eX28^RtqV*rNND@UF+!eG7f<0VNnd876as~I2)Ha8WKJ_7SZN>-ou_{pNYTDDj)q(c-mD~A{-b^4=_Cr54NBPOs~8UD2oo>QqGSBG9qnMam9t7t zH+2NRBgzQ_VsB_L2SJ2f*i1)w2tugX-xPjqaYa$BgFZJ6Ia^GxQXIMd+V7}MQNgo( zk%OCqDSTKf+|1p`93|-QJU>Z08LVXgh9L+PMDng|vT5qoGiO#3^@d;+f3fi-0*7PX z9^JEEtiw#r)xhJ;x+sf;m;aQVW?C|k<(Z9%5pc@~(LD*wm5`8v4F;|gykE3d{-&b+ z<>2uie&#}sO_6LY`_xBs#dUSe5GeClb#4<#!J$h_;%S_&2Cl;BWnf;f8D|CNb$0 zmwkUlX2sHf!IA0LK&uu1%%ltt_1Gi_^R`y1*=UYrsjk9U0jb7i0+2wOn3!r!50q@h zO+@y5S_b5h2okhji_|QvAn9d46Uoj*h^wscd^SJY-+A|P-ip+Bi+Vx?!u-hgo_-5T zTS03xEpn*_!eN&W9er0weH8vFb&oX>w4H&OSXwFg0d(^Sl};lR)@L) z((VbjY&4l8pkX`1kPY&4KxytzY_bOZqKKy!!Lg@GX(**Bq0DxW3*Fzv=65_-f%@?r z-_*)J6-bvr3veiWDOpeit|rqtY|^;GnxYniLWUJl%t8@y!Bf*Whtlig3<>nMKRpCKC1Wn+=w&EWbWL!clBwIw3vGELIT zXPr{T7>AV~RLDXa5!km7c{bcKrA%Zr^Z_rxS~$`w6)F#b zi+n%-RuINnXGEEUL?<$FcTcoGPLT8S<9+KL&P9ZU8fou#1)|oqD@#Y}+xO))YNIv? zjII~V7lY%&BAw#XD!DEeB;4&CA$>&PtIvub`UaZJh1!>O-xKWZCZFKDD{R&-g))aEpD@rXDt(aa7f&A*tA*jI|r8;CQIynE;ELt$-yZRoS0#wB+Sp=%Q)>? z(eESe;3a&10}380v%+p2@$XQ6D>GCR9@>m+46#eSE|e!g8X`J_;G8TUPo6g=2?ib{ z!}u0LE{-H`XoZ8AMjI!i-^er}r;Id~bLwM+d=!hHx583%nI6Zv@JLUGETvXLoO|Lt z6nS=86nEztAW79GGlE0;x-uG|SO?2)ru!x$glIcj2Vww`ZyA(`Dv$164s^s{t+_c7jAgqcHHL_laD=FgS2HMiBqLbPoRND$(sN;m zUAv@v6xoh>AZc{PXhd)|gKZfT#-_LSP{{bg_@~XSty+t@dx%ld<}hz;MhH3c5P~Qm zI%{m!Lw=V(JlB*twCtg?Gcd;*NN7**El8l?4v(<3Iaq4V*WgvbHAaCwm2$ci>C7*D zt3G=!GCe4}9I9BF;I96CP5KBGkke*!sI@88h6?*2(=!uvJSWEc&R7?LgC7w!7jh(^ zm*-o9?NNb5LtrbCuDtUm1iiH9T-3A~byge&<2I_-Rxmcs8(FDLl625yMR8p>Gf*XG zYbuodgn4qKz-iSEY8C{QlhoCk8Q*S)UxxAI492E1!XBgwSd#S=iIbE7t`1I1p7aT> z&^(SK&CHh{p2wV@wmd9JEoZ3bUAR<;UpVGNRn+IPNfacIH;0i1I4r!xI(aJCIrumr zvA*-eiz2TZ>+30J;7jsUqPiLDd7F$*+Z?|EKjaVnG~@n zEvxSkVJ6HLCWQja>NE=d)~2k3p}(M61NIkjQ4x9MQk86JuQ9MnbZ3tLrSwBjyC9MK z=9`$N5^s+0-9gTbmkcD%#DZfHB|lza6p?Uu2IBJt<1c7D`+*bb68#i&b4_Js4_&7Z zvMU4gQ41Bq;c~Uh`h9NG$ha5_Cf6<*Z~Pye*9Q1GcBM%t>mbG3bIGS)ioM3I-?gl{ z;`4k@V+q6*KmRt_^#R(q*b9vZ(1T)my$y}^xM=U@AJ5MqCccGdRDO?W@&H?D^6rft z=F)Cl22>Xdef&7lGqAb>Kb>JL+OYU@{l1)xOt?IwIRQ<(iNiwANt+D#9HNAg5X2P} zVa#t1!=#rZe1#L*&UykD_zR9fwbZLB{KV=`k=vvNd`aMX??nS&AKJq_@>S{UEs1(- zuD0r$5}@hvtt788AHh)My#*dd`f?dFeF7aOVx%=Mt7HfvCKRP(Lv85*;BQq*DK>jT z2_&?8hvjbOnf8%W`b|Fnq#uSSSdc$sD}T4EvjO>7bXhok&B${3-K~7M16f1;8}Ha? zt}yo7aPl4s_bCZ%7Kkr$W8CJq(kIj-qsHu(!f5+gaCh#xE!D+GB^#xN_1*Cao(@eo zk+AcUD1x**8xuSUYE_@euV7BFJ{3#8M(ci|ZqDhcsEiASOma%LYtiL2HF1bxwxh9b zC*|=TSPjb;SZ2`}(^aW6f2^p!L_(3Gg`+J#{T|};5w2&Mp>nIIEz;%u3td=fT#5d} zNtW+}%Pqm!=ibhMk1Ku~I6@>11YgC{=!QZe2jWF7pSX!Ge=lC5`VyxCi<|p+b8d92 z+f%iVf2qLTl^V^Fy*LQnX-cVHh!E{V`Y#2RwLDThSISwktgJd8syJ91zb3`nNjIyi zDr(A%rN~2Eb>1*Gj(@kr&XATG930faF!r1_S(b%faqYVbQ#Y1{F2l!zi1CXl^Lu(A zQpwU8vVd!R5k%88X6whxgCnql@*ee{XVTU4nW3K#8UhM>;(!rNV>)X2G&6HVq zjMFHmPyI9B1SI~TATAVR!}49MNRkj-X;#yVuByfPL z9q|}yEoYLOhu~A~k0L4p$2&+{AEA&CPyX*mnL&|VCt=}Di~xu@n7L1U{BI7@5*(O3 zVgM_c`Y)5_PGVHY_rcWBc+S@@^!gqJ+?;C^&u!HT8x)moR7F~_k`tu4SKjnze;^4c z3(W_v{{M$bT{t1Q!It=zPi}W+w@Q8P-XY=7i&Kz`*L8L9IDpO1x550N(!2Vk&oGdi z|4&@jlNw0N%>kL!G_jeX5;-BC{bh9eH{Zk;A*8)Lm%>9>cb-{xWJXuiRseXJo<);&DSTr() z>_XyuQ~s`XR0lp>=%L3-ERc>%==|ar!}`9<5|g#o-=yLoX|8(h;QMb_f2_bjOf(A3 z0W`p+k)*l)yI~+}C0}o<5=HH4as$7`HX`Dh0p679OW#)2ExExp;Row<0iC|TZ7M(j zw6VY=%;fgpglrHML^M)NP*Cx)F0K8;mzK8jn`oX-_FYHP(t<8GMsEd$-$#ZM3Q_p{ z@OVz`ZCJJbn@ALvBFTL~aDAIlF7;0tf!;v}N(+u%FwbYa{zA8NPQ89&BqKggp&5ys zd|*rZZrP0>TxOmqkXNqOrb62JPrfymj|`|7-$Kmtf5F^FKd^>p??jlYXsA?r2X6hC z);lrIBq~BksZqIY#V%&o>F{}qE}kF;>Uqn*iCCflVC6lL>XeTB-#>&&bfe`MO^F(Z z6el5nQ2#aFi&~jl# zfwGPxH-9O01cF+wM|V3N*VigjQ%8KIq5p<5jey*>W$q-9AkI{@F?aem>Du-Ng3)G} zYx#%KY)l1FES;~`1yYxjFa*zH^rZDzlSoVVq}V@=1_s274{cEq!L@SRi#eBzGL)gk zWNQAzNhgcoHX) z$f;F5RbK1jUk#-C_am%^p>Jq1X9ga z{ChHjgCB3-2I<&o;_5S|BBR=32c-Ww-&|cA;Ip{+R*%efpFFGzw`E$=_b@4RCRJzWc9aTVa0oN*?>~VA*4# zJNb!17k-Tqrn%o=*QtGI^So_ff+#)&HLK)#5<3UNgVS{V6yUr59fU~oMhf#hl&$+a_Jb)4b8mh1L5<6CL`!3YSO0hc0k{Vz?qe-q?+Z~!l; zlH79)bjQXN&iCZ4`QE-FbN)yqB>7@l9{$f2B;5M?ovSv=$AA0VePg)0d5Gxm_;^`D ztF#`FvJgKxHIP+l%MmjHP#i%#mE07yMdd}xFadQtz2GG^;4L?&DBi0`O!vH;csRCk`(^_Lahc)1;mAPwf6r96Tp%U zI7lmqS(NJkkNCbG+T_=OEgfg;mcKPDB<5?9bZzz4Y>I&l{i=(Dg?gFrwic@b4j!Y>UN<0}4r~ zMKa-+;B5KNpT7-?r4}D=H0!>B5*STOv;4)xNx9r&YHY(%+5znALJMI~gPpH-YV)qG z-qBJlFH*$xSW+XG=qhxx3{V)gkJTyDFI$O*pk{#H3I{Ky|j$1jN(VJ z^FEev)gFgRjwbDYj@JJNa6ym0Afgbcsj01~ayRis+dBAdbAP5Suz2e0$&v16Y;1H| zN>WTrOk1>Q8bv@6=x79ZDywU1t18_-XIGat3vDh+orm^>X3QHG7nhov80~VkSyk|a z3Bj7Ygt*`aM8;L6f{{^=;ZX|T0QVLh z_dRWJc6ij8TB!&q0-cNiy_xcgsw#JFY)nj2VjRDWP~zz1Y!6XyjjU>DpaWN1>nYq< znwFZ9l$fC7&!PZWVJM}T1hWo4DM9#33cOg#Vly!Xs!>vBU{k)h?lY-pfK@wF#) zbwzdQ=_v{Eox-VTuWD=SvNF=z&jU>foi1l)MoM8(8G}ya<70b5*Ho#XseKJ}Is(3q zMRJThw5Zl&>kPrjfmbODI4@V^en!#Mr>w-$OW3GU=4RD_I(($nbEG|l8TbD%HG-0OyoOksY4e4Pe!&U zCnnN?3=!MVdkDB-8L!dU(1<;uRBq^IcSA{~g59uEBGV*3uI$*gmoWqOhM1z_|MK#{ zd82A-Yx4&7<4aOxlz->WoyEn)y?gi0$;oL|i2!5Wx^)cJ%FWHq%*+g>fkTEBw8({K zj|9fdN-0C9$2wr#b=*=R#sM) zpPxT{`gCsB5VOE$Ams>AK2pW#$5dCl`Rd$Fc%a}+y`-n66nXGgR%cu5$aL$`Xpwqv z^n`LXZvlwu)M>JQO%A6)UK7Eqz|Z(lR7sK$c}2CmmZ6U+DT$%-K=g882fHbB`7;{? zM*0Q}5ZzZ?T2Wb*k)G;@TX2vMfAvsqA4Fa(T(x>#N=nk8f&GI>$|W|IuzIJ5m5T@g zrGgPrjorHt((uvqrqeTGSYB#Mvc1)S9Ijrwk>L*vE8kMEGk;KSShGQjMs3j8{o)tD zclp*Cr_@bs=DW%d%pYK@4^X#FX14{`PydJOjp~l0~ewg z)kMQ>Pm3xkkzIdtG4x9ob}`Z-DKnq!@f{eJlD_}``(OLo*LLmNC7KC8_Sj<-q`655 z5Xr9}Vp1ZzVODv(UX<7)+bkw_+JO~I{N9TpirGyg90om`)3{67-DZkFKNYDn* z2Q4YBpjqw-v1}*Q`lg@ zXS-QuOh@EnCRR0XQZR$7%~DuB9P#fVK%*qhktOMfqW94J6~I!W$w>(nmF|?}!}MpG z=haa$e2wOF_1rh#9h^T1Dv0IH?d+($!cNa2-N&%AUJZ^61ZY7R3eP}-OzYiep1 zEn2j8?OHxI;xwBwWeQG9yBP0IUIsYUmMmGab?esD)YRd_hfkX}4H>&JBDQ_|c059e zrKF@FcJj$5BV*UbDQ_1J-UtuG5`0r& ze)idC!}MU$etgev;Rhdluzvk|mI=e8Ns}fG95@g=(Iz%XkvaYF!w(T-v>amCKWNaP zfMOqg^bumvcmyU+oQRjqPYhKc@adBSwrk;e-cE-skPQr312V|(Yq-??B<(@kJV=YZYU}{btso_KNPMCZ& zf#5lkihfaYGD9B&Kb{AZV-8rkT#c3GmGkE|6c?t2S@t2-^t8n~R%jG~u0z1# zpj~E??!W;#+js2Qx_#H60sUArxgq)){`bF+7=%#7=)og4YSbvZ7zF8uKl~vB|3oG^IeFHsS^xa!KTYE< z+7>KWK&H!=FBh?-q@-_r;~RJ1eK#FbB#ER5bXC9j;tM#^d+)u+Z)s`i-~ayiAOHBr zhxK!c&QN|ahW_!!jT_-YDe_>Gfzu2ROwAAcM|BDoxW^wIzR_rDod zDy5Sma0xNIi_k?88#{LFzy9?vIFE=)B>DdIr#}Tr=%U1o9z7Z;X3Ur&V)DCY&6?}4 zzy6a?J`ov+&;=JyWib^ z|NUJ3$xnXrx4-=jPS~jw3-kaMUpMGc@8Cv3xTz=+Rt3Lx ziF(v8jUJVi+Va*_avuUI2bcr+3opFz?z`_ozc0W1a&B4!myt9v%>yWD9vLKto@-2u ztFG45V=E{KIC`ng2>9+*M^#m|P^rOr1BT@HXT2hp(A~0a*Vb)2Kl${FbIv%4FA<^K zc-+-cFE|gMk54*bqWM-rJXDI|h}cOZ$?1ko1&f!i!~;Kh;utm_puya|w`lpwbssJK z95MWLNQUx^AX8AV6B5fhlanWo?U&OF%C&yewvAi1eYjxBd1s%(-sGs_@!`L$tn~3? zNAlqDuovws*}iM^c_GhEfGR#G`Rlk^RI5)v~}WyG3TB&)4XLwcV$)8qfflR*H!b$ ziQ+?2`{(9dc-|QdU1iMY+}G!C-L}1~tRg!rW8{dTD50@xUUxJmV8V{6mPvYaGX#_h zcC!lUspimDQ}m3qR7MvR?ATqfefOY&eOWlUZFtZO(W3Foi$TKDkV!Kq&AkZFIGI56 zyd*91Bo{GxsWg`~S4kp$<>oRb#^fl%=;Sb0Lkyq^YZdAhD)#%#+p9={PQHa?6S+CAG2rA#zRPllC(pH41t~j1TF4RG(D3=^pVRxt5!A0=Kjb%W-u%WJZy-Rw_JIc; zpimr$F?sK*tFD6Ekcx_s*F%V*jGUViyyC@bZqu=xGZfHuS+mJ|xHGtM|;`0(Md=#?v1B1U0xs7gi@ z3UUioM(niHPQx$v>Z`A=UcDMiUVH5|I60V5_6-|00L8Lp%W$+(O>j%9iRz$4a7dh) zg8!yXo4)?_uj7)1fm4Gxd!K#wS?UonhF_AF0EUql2T9P3@n|5Dq8XAe0s(&oPcF%w zFA9xLNn&Iqo{|!fFam%>3?Co}^y$;*oO90M8$^73Jg4^L$cv=pWiF8xd>GJ69gz?- zM8GdOr)Ut6MU`ma7n_aQ5*8&06EJ#Aj>w2MBkU(byxbVX+ruWu=9vVj%y=AAda0KKC%Ufk=|IA65v9kz2NI z``f=BVmuqdzMgYaG%iM$tD>^<2j81D?L_WpzN#FwgyNtRFAe=sW0eLJf%YTddr}-O-Y$CZL(QdGN*SI6tS?Vm@VC)f``pZ=nA(z8?W=Mj2<~OU=r?R+TnE@wnEM( zO&C35SiXo2$j!lJobieEdw0e3>J#PkdHi^g54<}LQ5EHw`^G!--~0IL zD=)EAL6L!6mfnpMBcyJ%zu!^>3ej{sm(PXg#dqU8yN= zzWv@WZv1}Up(8MTHD$FY^^FaO=B{Vm0s*Ch-J%-Q2&*-(Z=mtxJ5x^YY=-{Rqz=vZ zebZ_^IyC+E-l1tvCBUnQcHBgo&YT2HZqkMF6XPnTnn)3rPPv4w{*$l4Ma-zO2kgla zrf0@gd!$`BNLad_xR2m+c!6*S(Jv(H^UpsY2OX>f^67_trWe6nQ(De%gT*BnNB3_8u- zi@YQ;H!xQTQwk$PT7gvR0s*dyLc)Ff_T{PwNTl4!CbPe zmo~yWLOQr<9jx(K@2Q76LQxoadElu;gZ1H0S_6-#VP8oZ-G0 zvynXmh?ImuvPA}MjA+He)YN35lzc;>Wr~fBnRxW5F{6eJ=+{TYL=RVKABf@c;?qmN ze!b&Y$Fg&bb>5!C*BCbJD=eG)Y~_1zHo2=EFlQ)VV?*4?v6+{BGhwv#vaVmZs_f;b zlBb>;KYAjiZ7D;hX!h!rm7IKVqN9-nlkDsaSDEwJN#l_! zHGv<7`Eu|X`rOhwOODby;I)7`FtiUJ2t8Nkcb#<7Np!pIZ3kg`V5}EPE_lH3b;vX@ zomAMPxCTtE8J zk1zuSL=58@)k#8LPN$!KI^+VH2T6mIU;rcs28cG;1w0TL%Jtjd{&w!%xlC_k37iQN zCUAB8_U({Q0<68O_0b`sfF$7(^#Yj`v2TC-+lamO)>~u> zdxu?1DG=kh?Y7&#{PIia0Xz^_Etg0YR8Ag(OfSVksdEPQNlg;))E@hr^s*HmB<;8ySlw79ecmc`xx+^(UU!86WEyu5iCWlAb{O)Za}46QUxa(^~L002M$ zNkl}z&d;50PQ7&g0NML?UY ztKBnCKH=gEte2NL*h7B7!X;cib>_6+-24NHU-J3N*x2Yn19CZ)iZ=2`B;c!Ot44wt zLmeFv+qze#V5Em45D)}&au%+_ZqT_0CbK> z`ndf19cUwlelQ($BZ+8kkT*KGh(Y`~HDXNJq7kGc3(X)NpBOSw zN4mTy0ZC{W@v~;lA|AdfTn+~pr*z!$U)jZQO%PA1nbAZ%xtP453hC+TT*YrjWWW*; zZ$Hq$DGT8+Ad`glgDSv65#F+8i(qAM=;;k}RiH3B#UA?lbl=g2i;qi`k%PQYC>BB# ziHFQkDTsaZo8PpHaXZO=34lO6tGq(O5u=aJsR0(VrCxxPc!(UG9>f5gyv*^R|NMtY z_zA2DG1vw7Au&{+ks-jCTO#qliI^!D98&rj=ojjULlQlFzu>p|fhr=ISSX00Qx0et zeokl`0+1r;2dqO3J|GeWq_8i+5Py`3HvzoOk|TEo17%G-0TBDgKmH;25_sA#nK0fh zl4AfBdg-N?ns@?Wa??#Waqn?e41oC%57~xr%I66q&!}1|MI`x!p+UdRRS7^B`D7s3 z@e2uLnO{E{XcgQ~WVk~J;2B1Salg<`n4ZYP7{LwNOu(*zXNqQry9FCKNFMN@X|)e^R{h$2$CpZ7G zs=E4DzrXX@7p%u0LTDEjOW0_B`_8>VieOKoVMU--2rOIiI7k3aKUH{85!`%X)`9hgA_RaS15%PNWOkh9XV zvh6!}TURaPtv<;ef4OJ&i&nn~c}cQ<*bUn!%uggA>J_xV!8Lqj z5McGNxU54K)T*zmXZ%u|t)U|erGZo7Z^8M-XMVUD9Y>r4^mO4WIGUJXhKmLE@{M6Bf(VN7MLDI^NN3M5vab3YJ65`K|n??*}mA*Xa)MNFWhHbsm`#Bd;kA@#*gMGUCg z#Uw|5;eoi?=#=C5LJWSzn}%te+=pC6q3EQVxQg?Q8&br0(K2r^pcp1`mm&9 z2ohmDz%UtBVc}Ac14#;t6GtReuJih7#ZQ2 zXG|}w3|Fd@_pg8bE5sGJc!WU-s5!+R_8P%jvL-LMBDIM^J~6P;61G9E$P_4GjOLNV zBxK;jutemcYMgTK;fvrS2zFv26KH8DxMLrB=pp9fQfjim5`KeJjr|gm;J{N1X9h;H z<0DDmI--u0~)Bb(?tX{Y2v!$yXlg9MR>CLn^_~*(s8ySnrPy)uO z^5_MB7I>%eD7$^%Zeo0&?U1K!udc4K-gefjm1*j+vf?hDLY$DSs`A!*3%CX|Vf-po zWAcR2L-Pk7Hm~5tZ)7T9QnJkbOFR9-v~#~2J8&>bsu#Uq_S*BcU#?*Mt1Bhl(bV8g zO4;E`*tUIFv}4z}dQVJny@9TZsg^ghwvMUAG(l8TN^&xLW1M*Wfxi0D(&cc*VE`TpfEf1#U5Z;xSw465YYJlbGHbnoCHjEAHX zMi-OLG;S8C6zwAu&*+1~!FVsxiG=denX|uI=p+8}m%l_mJvKV249_D>-xTA5RQo*g#K$a&aoG5*pcDj2OhIP{asuil>PC88VEwO0IG#$RRWU zh66azR9R98dBzpPY+?J{3}}W^$&6EYCJYVD&^$;TBROGEj6vmN2f~aQCLxaiV@J`+ zQvfn#@w1=(jC-CDl3ZdOCdo<5oJ?wsk09LVjKYP|aPUS(rgB0|k!~4;^6Q z2*uCIZG7Ez*YRzFc>IbmNvJ8p*vU5(GJ=#sZn1~lA+(S*{N#*Y<}pMY4=hhQ<-$RS zgPj|j`h~Xf;Xs_IJf}QyG!Zlo+&LuY7UJ^?%oE}d&pV%!$c}jKM)cr+7T85Yh`(5l66XHln}vac8>E-gai4=2j$f!RcPVWV zycZHkb_`o(LMh}~2ss&1%YNA*MkKDkpa1H1hP3_p)?Z>gNHS8+Pf|}6fi6IRM`OXF zWm~rG%uG+q$;nPkjA!YQ9Xt1~{Bk32|FNTnHNS^jvWhniw8ULgH|3bI!-fpv3Br@J z@U!Lkih0Ur zYSpE=pVia6xo6It`OVyU=XcNCx&Q3#d??oJS+nnZ@X==kI;YS4)1Pk{fBH%9efwMb z^vNdM_JFyCwrqj8{y@KvKJxI%lc#_0hd+DARhJ)j$QWHr1>3?rd!OEV`@NUE{sJ!o zG3xZ-7F4ijiI)Fh>{+c#t=Vl5Fl!W#t+(5yP6=0!AIUH0s|ovvgvjSO3Wf+YaYMPA z98&=b`2sJB_>sUs1fAF;l7Nt)M$#O2UNT{64yb)WBrNLVGMaR_xx8nQ0mZ4c5eQF? zHud_935W-YV{pVheil5?DN&{f$S}#*Fk7N{s3`lP+4LYF%HdO`Y?B}sr6hvCMYvA{ zNj_It!$Tk!<917jSTvtoOk2?mrc#LW1XtmGDuO4JX(Ohw#6nw9L$T7b^rmgfzNyQ> zr;i8ctkFdWs|#3(Y5-4@fGhvQ`xCgGKN!<@d=H**Q5o_OajpnmIk1~WN<}yXVj%e=DCnvyz)it-4O^? zs)&RM0YN3E-smYd)2vY{lBUExrdZ*bmkrC?|EUT3NQhNP5ET?4)h=n5gKO4QrjJ}B zY;aRRpw%pg2jZ(0d!I|hDnW@4eE73oTK&b(W;~GOhIwV?JJO7+h@qvE*`nXsJ^y_oViR0?)Yq6%mgZi~n z!zS5rp_F)IC*kxsS6|n2_^`n?BlUcv$eAY+BcJe^HS6qxGvTxo&N$`xGfq4Hq6^ME z^Yjx@(1#v*%HErssNT{qK}Q9?y7%Z@-F3~pS+hU(?v)SR={{MtO}~2dSYA{AZ0Fj? zo_cQSyq9dBTHQ5!`62gWOY6zx(b&?oV$0#|CG?)H%x2dNc{KZLU;fmGKKR~~jz8Ll ze|Cla)o*Wn*ZV#s2&ro-&P5$Dprd7Y6UM_8mtJ)Iu}6LP2iHCE)Uzy)$&h)!|Km-6 zx#h0QFTLpGlaBR^u$SKyOHsj&j?)JI(typ6<_f$AVzC&i@d!UY6 zPts&zb!f1xc=(E#Om3F>LJ-dqQ(+KJBoEbdgTf}fK(1s-U#2kB2K_676uzuL3a8WZ zGe21zJ}$NWHi&ULMVYvm#Uha|@s}mxzsgZN51tn&^5Cm{^K-Ej8^Ti;7?B^JZHQ4w zEMikpFtpJl!A=RyFN)C?EhSJ`0Bu3&>MQRLS4mXDoZlp4Xo&K$lt73U{WS#v_y{T zC<&c%1(mlT#R`|F<(PWR8T|kF@8T`u`jVB1~5J^5)c)O=XFV-76N5(Lx+rQ zbS1_cwYi2o?PiAs)q-Z8g<}9fg7>O|jDx~Rz@LD%yS`#*h^3HS7)IFDfyy%5p@@8$ zQI<~K>JvD6+)V_+*qY*?qm!_!HG-}zQOXgf3Wd3x)kvEIm=-oW>!L7nj3Syf&}war zj({63XPj{c0oFQzCZI=BjUt@U5K7bw6N|UeTH$o-W8=|BrzmmV!JKetpv?^9`enK2 zV2xx;nq!ZgT!74$9`wi1b+ZGi?unRZD1nTdR+dSNjsMdAwNk)meP4-v1O^0lMvWW2$ii%q5YD zo*jD7{&(E_(2SY$MvmBLlX-CCebRiUDw-z@=C7FJRrR&CHOrT;v|VSGUz$|w>uOIu z>1f}C$H`BgI%EEV#r^vB89i!WCb`Co8d^)c<6|!v%UKC1b-?Hm{-iP6`3ojiR6;DY zA&n!35AiVM+?N&(8=5`9zg0))s&x%3>QL(2wX2aH$0{-QfT*#PjfA6|~dH=f@50k&%a@SvOzVj_tyza!~ zD(o<~HP!8=WjDEacGET_vl9ZXA;7US1vm)!$oZ=gJfF)jTU-DW7y&T*r*O zU?jXl4FBXV^Dp_w%JN4ooAf{z0R@A_h4FG|h}akM<14Q*#DY3X$B~Og>?;@JnyZC2 z2P3H>$;*sTQdEj-!IrmO6jPAXA7atEja60npH8rd_*oMuL61+F)&UC3VCJ;3A0P^c zXB@y<#K>6}B#Ec~iWP-`PiyBa`3uB+rB)!tsz~Mu>!CD(Pi__fqj#cpWW?)u*kxjn z*Knmm3?{^sEfa$bjzP?5@fp29MvL+qB@9BXkO<^MOioL&(1vnoiv`cA-2f`j^v((m_L(y!9;1#BA(WX`?OELhodJ}!Jm5zHNQJnX~D(%W;BER@G z9*jiMEedtBgJB_~?_|;+mrPU?r=q$@i*ad=hUk6w0`lU<7&DX#{UPj=J&w>2pZQfx zPl$c+gCF$xqF|cKNAJ7tTsQ*zF`V=-F-=3Ua!UXu)5b6^YQ_Y84Q)PM`-Tu+R*mAT*5X=E zix65UtAE`fFf8+WBVmYs_!D1pM?ZT$u(VAg^QmRq*%MCPXP+!4969Z=Fv%nYwgv&C z&K2r#2dlbU=7PFJ4AdB_n1-DE@-J3Y@lrBv8Ool3&2EU=NK=6lvGl)fKjgSmDxF`1 ze&bb!57-Fw!HG|K%d4yE)x^G*5bN{xx5$8F!-Cpsi7=x^#YF!SdO&=XC3|u4_$=`(zu` zwtE)JLEfaq`t>7*4;wvl-!da(wv=g}JNwL2#~yz0-+$l}lc!93aniIh$(`~wMpS9M zt-th*udxTue}Cu4roqKamRgPc=F4A;2g)?$O22+vJXi0OMs24`6BXP}z$Uf31OoZI zR%T7x`9Yb;a6`)qa=U|12K(a86Ob8Obe0L{U#TMRH$x788k3AkDg+c4(7<^@bD3>s zTWj<9e}3uw(!^)@=&`O3hY2Q-S0@pRIk~OUVH+;mnplvx&?YiTGL}Jx!r(b_LR%1$ z5DO|;6w4W7St~-Ofe6$|-fwrox=$!3U!!pIEpaUQKG zQo-m5s=!EovsADs7LCPo%BWyyQ*Us*_*7|`e`D~(_@O5kZYhTF#DJs+-{ABU7FFi@ zGg!NLA(*pF1Q_kX5*i&3Iz?(cu+8}ou1ZI zMZSU}hgxR`7qm)X%_EMula<)W8&VfFy80Zs+DI66#vXx?$XohfE*=&iPMJEZuCDg5L-r>Sr{a2@wN;ib7YaXUggqNw(758#3#)!rb<1t{xE*%I zB^RD}{5T^JGh}C}rEUGX?ew;6Zvzq)+};Q${kszaF%NP*BOz|Ui_Iq#nV0}qRZc|G zc*)Y{M43wElBSq^O4WYTBl%!@f)d^tjV%O^QiP-#)~`tRLsI%tEJCL27r*#LJ6IGm zqaqHggDT5eDG61?ETs#^67C~bAYj73wC=7bCd?AyCIm^TV2qfUR^tpL;}C;#R!)Si zh{;zLp^buNV&U-@t*w~$6C$4vt${@W+I&UHG)IU`Ed(3^j)XuBzCkQ%FP4oo84q-c z2o+C-T8ttvK@^FFfKvjDK$8eQj4p<9G?iEmEXp@!%9JuOXNzKCHmXK>5KEj&T0;OE zblO*HgY?SLiE_d)Vh%(_4pq4r{E2C}UI3j0Z6On4(#RNIYQ(b8S7o!Gx-J`*p~^8( z!6*_KFXXVM(Rxvj@Ohh}1Ox0%VsMhgWW>0_aG?p+5UaFrQ(VG8 zbMc1pOj55yj!T27G3N6s2srYuk#GUxoE}TIrxfPtDL$8sCnT{lD{^UTS)@!u{*@$o z7p(oIG}cmHgF&EAoEGKtFl3A<@;Ivd0;&`6bTOJtEP6M$ka4y+ldN8~>V#vDI{&;0 zWjrviG$j%OI~IZL3fgbqVb4#RK55Df(Y%8W9BtQ4dsII2>?Hik1K%U|&A3F)Myz4J z^PY!RG&I>Dv~Qn!59SpOR?zziADsB)iWMsl*nhunU8`r!ed)2spEK=@9yz?XxBnHc zK@*54`tG=A;?(K0hYuTc((y;i$tx(feBMPmYQN!)P0bS@d-|mX zi#&o@Eaj;^Q|=3srkRdSVoSW5W#%0GQ&4}O4zCU^%|AWI9oP8eL#w7u@*`1ISHEuM za^x(kw!i24=GDPSkw9zNS1n)Oy?<4t$`>UUlO)ak*8A3f@b!~8rw^zh@;X3P|EJ>%4q z2y79mnPJ=XV5GorO%1!cms*@n-Iu-L)s{c(yU$QOkU-3ns&xZgxY+vPZEC{qq%2Xv z-Kn>dt=k)cnB#bgXd7qC-F^O?N!%o2(l>E@{q}@2Xa3}z z<<^D^J96YmKB8q^cp4{LK@%$(iK1{`Dva?p=HEO&Pw-_uEaz<&7M*%Xi9AzPEuJ7@ zkz2eyf|zaSIl*|B9Wscq)8tN!uZ%f_82?)XSt#TXX zQ{?yisncX)@g$_!CQOuNvsOnFLKP))q#fj=?Jb)YMeLKoAqFg>V&7K==Vsf=lKbu~ zJn5`>Fo}_6r_|1qX0rehV<j)RJKg=H~F55$~z90FRiJtHeB*my6+n{BNBVJI&$= zmJnOTC)Q-WZiuNGt3)*!5MsatviPUa4Yit2M#JGRp~m2!qehL=Wy%t>2sRhPtcX#i zI#gEk*(}E)K}-nF?w1;6PZ95m(9F2$pj{3!pd;)vXqYaO@JO5Z6??2WwNoVyi8Too zk{4FOl$531SM80uq}w7R78(X&Kl!}gyliO0pjhH;$SCfH1+wjx9$xhUOkyFK zccrG~N;aFIIW8-zXh49q%g&#urt4Y*p)H>oIs#oJrc=Tm7b*S5L-FmS6>DT6F{iW_ zm$o!?!h{K!wz~sHg_qex7gQ`Ovz%6zdRD$jqvEKb^`9OWWUvE7o_GX~y8x(ryxRk4 zo@_MvxU|EPOl@gwM5Xohb(g;3HPV_E7hUAH^RuOe?JStY5&{(w;ENlI`wky^!m(qY zc=CC#6MXcE=UfS90<2@;fIi2K8|wx1%I2fkAY8R-O;cl&$s(@M&>Z7@%;*tj65Dvv zGx2e2Ofvm;b2;MhgT@{!aMHv6WlOn{}Hb^eq7J?y;i=cgmP<2mo8o5;lO+E zfAsGA9y={s*w;-KG z|2p+N>cpA@$9jL@vRiLhJ!g8Cx?b7hQ?JD+;M2LPdezG1)!loHKKJ~wZ@y~ap<`RC zC3~y$OAA)4Xn5eEN6DBa8X6j{_I%xi=XLGc%?>xS=giym2%o8K?vmOr=HkaT{~<(@wV#!{@zc{IPIhpj~mDD*f6(u?tIDXE?`^y zl;}!_QS`?fZ{_d3^-ZsD&Fnoszua5e;~ShBm=I_KfpC?0=eQGgCqm-kqF_o+VG9B& zCJqiXMnsQ$@SyuMB5@Q zk+G8!KPhWb{*|QSEI}t!ZPA^oEqWwCKPKkMN!H?EJbQ`2HZR^r!f!JT{AN2(qV)K4 z?DDEQ9j#u7>1$uA}=0f)g1wL$~1V0Spyt>6rWlhEH^JzmQX~FHlom? zSXi;EgE1i%XIUQFd=>ih>7@=Ry>yH;H#HcKep|c>@VMymit^%l&Sn#;LzX;Whw_Cm z&g_ncwD$V6KQNZ=js{w2B*y|&eDKjCkg$Ni?W~jpsK4943fmVdBY;0P+dQJE4U2AJ~#&#;%A7P86@mS+O10^(nxL3+P-hRs^{^iE&J>5mf!NH^=npj>TWB9 zQeEic`gP4qm+W)GNrzqjwvpq{qF=+^fZVjA4Tinv?QeSRh3DlDOfp!b`u6SJr*|)} z!T7|7|4|$jp;9(}Z5al)-o5JI^Nu&4cFKt_Oqw!h&OA?D>Y4)%7=6^(L$c}$kyu}& z2sF$mMH{wbdVMG4)}A?lv@)iwqG(8-iOg$b{&jCm=&*Do1S%rX)YLq6`W$OBDxPhB zW+^zde#Od$2OfUHt92AKS+`a^PEDUN=g}vgZB3?)d{daS0t+H7?ooN^nXmjvhXfWO zQ)mcz^EK*niVH5^608vRY2_8(Vn*Pe@~0`}9U=fRNAUiY4F}rtEtzH(R1hD)xGWWm z<%M*j4S7Wz-{2m85BU1RXo!uNl1F`b;@({$ET$qyoGN}sO9dZ9oEU(V z@)vD6Xd`}K8P9hl?_QTr{%$zLJj;kS1ZQJO5Rzz%mPP{+OJvC-XZV;L4QG4_#SBJ~ ziV)LwzB@F~7&$l(_cyF%9yG8B#PXVJuEABHfw(H`4T@Nn8mX$-w8&(C)IvEz2qXoaWD=xd3xgIWjQJ&2{I86KFqy)>huTqg~XkGe*U8s!;0XkgJH z1|^zRO-y@ZNgjBO5PaAw!4T7SVu3#r9>YoSR>TPl%}eP)R)LqW@D#eDI992>;<4V5 z;47Ab6%&JM^g{k=v`8Cf5YTI3QZy6V%AzPnnj8!VnH00N98?;?&~Xt3H7erelQbwo z%&iJE7{iZog00c7EAti!qsc*c!f5s3bB--L|fR#6Hs zt3@o)S**KMI@`4cokp;M!$_Pz_VSG}FY>9ZI~SUulqQGTkAC!{^4+Siyi>l>LK`mU zSvzf+GG)rwzy9?Iz`FVfIMB|h5zYn!SoGS+kt1blvS6upMWyH>k7r$caC8z)V)FZz$Zp_bLb^6o4x~_5YB4-I&jZ+fb zmLFODYbvN+TghEfv8{b(&6&Ss*@|sw1RK0P5+SEkx~SQ^b3XO?|FBQY&9~is!(VRu z^G$bb{JVW)BFnk)rrU42<*rYE{u@8~>2Hd~rTx}Un>p7zDoe`CPVQ-6Gwz;_gg`|E zwg(klvUFK#UNT)&K6pP)Y$X*GC_{4ibK%_6>Ci?bB%C~(8TzUspH9V23|CxnMc@#g zCFd`ml~N?XvBy&v(~Gj<3wm@-G>J^{;L8S)5O53OsEhX|3YSZU%NSqT7RLCr(l0V? zuo(qqwmcZ%ESeFImgQ>stfH4rMXQNPhzONE#iw>&vQrMi4w7Eu7JBj~<3b!=q{(++V)^5g zrTgL=?pqr$){aIMo};6jNuH5YgxUZhSec%+ep0@-6z1JmG)C|o!;FH;>i1YrOD6AS zm;~WhW7r%I>k4(^x5`FmNyx{|lywx_i}PdGfef+M4$I8AGIn-pO+bF$z>pESjV{Pf z1uSi7=-4S+Tuh%i*K<%EE1pUv1U8F+p=)@@04}Cm7IZcQTAqrW(%B7UfJ%QAB|}MG zX+@cKX2#DZ;moqqQB>=v#@7}{Cf82=cBYv#cRmem$!|QnZdLQr+isZkzRQ1o?n!rl z<#Q`bThl4H7G2z2jb<}bX!%i^t7(w$(ZSZt;9C%IR=eChsz?sQe!ic%j&NJe*}`)w zZH)?Q^gVxj@Drc?+Q&chwU2$~tF8TIXFvVb&wuHglcvP4$;-X7S5d(#4~mq2#nMMY zpd100_llJ^66|ROfBCsB{BC)GX~NVtcu;@NAD64`V}%%EW7Cj91Gcc9Oq^!7(#8G6 zQgA9eI)oDW7FKZ+MMC6<|0mN~;RQh(3&w;mLW6v36fLDgX?S?NTY7lX9Qts$3B(!$ z@h)z$8nw&86BWeZM~)vq-Zm|w4ML8g)3n8F6szHAnlc%H=LmVJg)3|gXKp9X##<%8 z1Li}QEoKNYpYqL|A{AQ1PR>0)9sv`V@TxQxI&v`;38V-j5n~21POw)CgjgG@I*e@Pme>@!j)*Gv)Gz6>gy(jp{o{MrU#{UTi9n zq;RvD3Y`ILj8e7{FxZR-;|Fd=oq<6issp94+1QJnmV%?wV24se0-nq!6M!&}*#@Nc zLRDRsn0hgZ5$HN{0E8v61XqpGBHC0Z-{k`q?7j1UUqQ6@+oWx5n{8}6X>2sM8XMo( zZqV4aZQHid*f#I(bIx<_z2`aiKe)eT=QaB~vuDq&nZ0JM_s1U$d>_nz+0d&Qj`7Rb ze#Ka>7X7FKX`f@1!a-^1lI8C*I|GlEoFF#vzMh`Y%mk@s^$zoS99*iOKNZL-!`RGZ z45?NHIp50B%Zkd=lhe3bwHz;|*O^sUg}k}P#SPrqz=KrqQkof;kCmgORQmE<8j|Qx zCoG~Vxa;$EZ!|)=9O~y1_YQPd)ua29;UktKl4{mX*(fVgW_CVhz@75hOHZN>BJ$s? zF&^liJ}|vl4S9VpU8WBt<)|u)kPq(4A&VPyh7(BF$nIQP*pIPmZ488p=lj(X5e#VRfa7jmGbD@TGcA1 zuM5~0zFrEkebMC7x?4bVH0x{gVQ%W4Gy%yc9W*LUwC=%Pb87;o!$AF4{6R`^VQ3)~ zcc5f8w${=@UHs85;f*Qj28xblwO+;Mj97(}g@G&bv@FxBET`-8t7V3Ovt5s=J+49X zE5>*Gz>oxGZ!Am4AEOH(Y1*b+8UvMT7aeZZb^I^i%q{xtQyH;YC|g)7Nfx!xglMOr z3pGObuC2)HWH0LCrb)CWmco!TP9{-J(L0d+^Ng<8?Cs(0=BJQ$+tj`#>VKB1qBkLB z=Fhj#B9bpL^;XcdIMPA~6P`c{&7X7CR;YsFz;@OrDN(1{&_FjT@d^5t*uSw#fm^6j zW?9o!+%XZ6&84_$yocTJySHW;T$64K;RBG)UOypR^6rgI)2twk7l#5)J1~3!S>4BG zAo-x7;xrx$9X+^kPs!yj)zAkWWEm-~?-Ll7kh1y@?%vyEr4~4ByRi0D&uj5)-6ChH zq~Mp`{-sbHJhus|5K{e24fYjp#me-RE}u%FTf2nYiPo?I9X(v-H{(P`t6q1H2!{2g zvqq~xt>W^6xAxhf(e@!02}?-nlGjRjwgns4Uv4%g5`3kU9l@SA483rs-h_&_k zV-0IT7F@vk36gFiv(EiiYRE~PVmA1U}@_roJP~| z=}Pe>Ax$ZNqXtmf#jaPg5mH0Vp|-B3wi~IF=Jo26mcuO7d!c~QX2)GUYx&0f%7n_d zJ80hkfqhsYCBJN1%FbkJX;TTs*?_hbYY+IkW8EBZ8;w?f{ur?PaM3=isc|T*NS3O5a^F3h zX#feX4a!g5YZ)#$T|(ifmNNwWt|ub|FHbKo;t#E}%JVxzn?1gt+mNjeg=pkw4z{Q* zx9FU>7n+vsTcKQ#?K7COGIG}}rIxlT_ilt<(f77c-rbT?id!aQ+x6G49f$(PRYjEn z35#RaD2vlmXgpvTs4Sp^I<40kX7SmI6^Ba4L-y0GxVvJFeC8?$7)`hIzx5>Q%KTdBj((;UyS|qmY(#pjg<- zug$;A`nl={%OPm46`e8;pQ0n^m`^YFl7jA7jT31XW;($4hk?Vl)%6m&i$;0`;@4901?hc*zmojFm1K!+Dgt$)?5r|MHJG%dFsUfW$n}%as@|4N_t&h zx3=MW`}k~{p(3@v)7n?nd0dCIiMK?Y3l$vN7lBlMHaC_3;WtmWh1cbwNU-vmkKcaO z>6y^H^1Ic=Q6GMpPP@535}RXVq&5aq{BpIiTSnJ)NWsAV%t%N}djMn)Fn#~PCn-_= zXXPm1t5{%&@TvgvpuyYZa^>KHqfwo*P5LVhp(WI1Y z6OTfFiT_tW8FZEjPt1t_>oN@@`4lX!rcX72HM~@x!)g`p2`B z=h^tJv?=u+iiIU56H=l!!vEE6dr(mD{eK!qBfuzA!%mD#{eQROKN@wf_XV)g{@m=WwEw-C z{|YFH5-D_#H07=R_e%el(Sjt6D%v7gYc4kXKbj4c&td}lPi5fy-amu-e-z-w@)iF2 zur(ngBlVx-{r^=(JhHe_6%!pL`~TFc)9!y02h!QvT;qS58b$y5Bi>G;`TwXCs6r3% z-y`uhIVjHl53LFiFar9cEL>_t=6`CHI|KNa>0v87L^&S1wg4*QqtO3jS>mN~D|DE^To> zS*bOVX{>NZ=3fa*X&LU4(EYEcstO~na$0X;wOA;dcufXzkJU}2bE&F2A8BZ7uh*~0 zT;Z$Olu!WuYswD}=-O+yyOlrbkGJ5nbJlIPY0cop$dCb6{LJU**Z4}kRAPmxrTO_r zyQYT9yAZ7!ZyGFY1}gE2#b5Y6p`KRViV zG^$*lLW9oN-JDlod1mn+3}c;tN}kq3#Wii{*eb_!aS&*qV|A{l)VQA=f@|QZo8~&6 z*2|w(1=$36@U}hlh@CaUPn>YmYB$*Tk3clmJu#~&TjADg=7>C~YwiI#Bqk2CCHd4U zbXs*~Hk;2Y?Z%#S30d*KP{FyOz)(+f`bSe#%AF_|7Vp-y_m_@KCgzpbYuzR8>M=Cw zfbCtkU;gLkyYmKo5=&|v92qTMck7y%T6lWYJg&*`MJG%R3{ce>(#lcC{(m;0}s7=e#-Q1XWpDM{4pJ26eTwRE>5;`OB7e23Q{Fqx0q1a0{J!w}G-|4A`;@C4%nL{y-rv0 zh-Vkkd!va!tl%y1!6FtZbFONo;*V`Log+e)MX{Oz{U@AsQEOWm-DBj>?|LWl8fi@t zMy7?vzFlfs1qGVMHpAajYfDRy07z;wi80q^vttm?VXVvdGifJlI^3vSeGrzf6~zF- zpfR;op;`rbqD?M(^Cg>fk@ZHKD=N9HR?V+rua`4tfI!yhT?C21VkzYAB&O^XtX^%O zZ=~CP> zx`l5*6waAqquw;i=+?_^1!}B?$zq+9rRV)~@V)r+G-JF>I@|1XShqifWkbkgfbK~T z{k^f&>A=5La3H<0HHqK`#k$sVIh64wvZ%DYAL8a^lR&}sz2A&Z78bf-Jc-r5j}I?; zJ>QJ>Y|L}b`)$$t4anhfeLgDuI9BZpo*rV2UU0vOPXl$?< z3%+J@hWDi;)jmV3wFt~d7O|pLW^Bz#`q}SCtl01oARUC>PvtlDKE`lSJxI`cJwl!7 z{r&_XU#MR00eX?<=5s;`H3|K!LXe!Ari#sIgR>fqQao!Kwh`wU8uf~VP;$)n@>w(j z>BoWh`+*SQbJQvZ0Ss6{2o; zAfWW=)L1r~9ESdBJ;_(qw#wvd4~>oCroX|YA-*VZ_;~k(gg-sNZpaZ3v7k0~^Remf z>4(LnbqaP6=;RZEc%Xm(srh-2X8Z|sjJ&0T;kHosfUNZ}eim3P-oG-+JP!AiLp5i* zz9O66TE`q?q;kc^=e$cDbXrrT7S`*5-g%01c+llKURN5ycQzCFA+JNE>7k>!V`rMl zeNB0ogTFFVqx*3_5KSu9cHZN_y>KKF=>C%nSQ(4io&6h<_%!(7cCNNPgYGLz&Y3{Z zr)IMz7mXKFF{qmpu9lcqcia78m^5k(?)Rs(^^XiNdhI-3U; zr~NJf{h{DfOgpl#2Dl}@NuO}484U*jmK$3Cp)U^|m)T5SGPB9T;vym;Uz%Fed-HeV z2Fv9dOkIy?Ab@1On`KS$e~KFQ-D%ziKEj%&KPDK*w{|>n+f^M`X`okHPu^hO2dEvsi@f>SPH!qBZOiPc`q% zj^h7j0kFHDOFi&jK1#u2B*VLW)q$2@&v(1rdM`|Olm?(4VW@_@OMbp@sm#LODk{{eWK0@kAT3qQ=Ya&Kzeb>U28y1>ok{ z-YNuJ-q+NJj>Pz`WbjJA8$rZ*8zCX=cN`omQ(v?ij^{jq3e&XLz4gfUudO9UM@CJ8 z>aTq4?ItLxapZUN{d-4gluP?l@fuLHm!~WdlQ-_TW1U${npB7yQXIEq;RsiJHT_%BI;@e*NeeqatAH*xcwgHZ zMhEEe$S$iRJx>ey-;Si{+l2{QxdEEh$4TeA#ZVNnuR{ni7BQRixt2i*6yt3f#r+sF zkh#SF2IR7@$K6R4%05p1qt^hY!W#jbGaLTqbX-rQWy zDav_1&t_uad`E)(BfQH0+RHC6P|eu7^HHiT65hO*ph?koTvFNr;3W51NI!m4sdNeQ zSLnV1;LE3Rn~4ZyT`x5!D=2k4@j+lgK%##tAF@IXag`WfC6xEr>Y=*(e)AO{c5-d zC2&U9dRjy8@fViY{eFx(Lc=2{TK9P-QR!f#`{Tv?<6`suZ1Z&j-A4AHXO#;1^8HB8 zyXP}@RT#oJaLT$ zq&snkrwak-$PFOO&YpPx50LYqi{`j+VEiVcnz|%{i(b#u&poKHu68t6Obe@tAOC{enH` zbtV44uPr-ruF={bSGW@12A-zSKnq|A&bK}Pj; z{L}0xqAS@!n#nW|ZyB(Hk<=l;e*p5kKJYcVKMXQo)(+}YlEibXh~37J5g9F`k`!0`l+9b~ zwvwhOk~MAGS^KJf^o3LLOlj!im_nq>@18Yo{t+N~T_WQ`i-*a~QD)z8A+-wO;whf7 zFx^<}xMw}(O}Qu|we7f>;ErbO6__oXk0X?i6Q=S;`zol9Ezg7xBi2zToJXptO;L)s z{+!U*CW8C@dPFjxU5nMoXqe(7R479m&{m0q|2@6}6qAUZkh z5BIaUmx-OL8jA{bQ&sc?dV#Q@o&^%aegt*j@IcHRsNN)kzu1|6GSX;p&Gp8nc?S%^ z=jjRh^_PE{D0p5dnTtl?y)SBBq_a7k#}(Hi;4uHRTg2==J~o%* zFt#*xD3|frJrc|Gu%4O24koDUq^>rgou(KNcad-xNv2`m<#JYM_CAl<{IKljWv^7P z%aYQrEU8#e$?$9|ao%h()T!Rj=bZNsvYtY^{t99ouDa!3)N#HkcW&PcT}&(@`kAhb zvn`cK_c7M#*-ILB61`lXf0t=$VmNCzh-Y^?Ew*)|`~KW$({XqHT`G}L4HFKZ>2U0d zymMz{G#DOV>$D?=Esgag#1ERXxs<4CSnjmKK&i&iMuaE~{{Dp7@-S;}RP~v62?uNJ z{m;y8T=QMBl=Ah3XG%6nXIvw(_fFlTeY%m+&Gxk}235ZOH4@XS#_t3Jg%YEAE2uULP+f<36=od6y=0IBoi z?LzY;#O(DlnEx9_?Z5r7)vgtQa8n~$yrK7gg4I=|{3WzYYffGU!y}Idb_SZ1r9K&?p%o2UeYf- z4ovu8mH{-xiJZqa>clZ7?{{R{@o4oX+iZk-7goV35kZlHZ3Vl>J#%#ExVb&OPEr&Y zE?70t@KqR{l|)HpKf17yHinVITeeJ=1oF-xJ|wtEttJLJQSDN8vqW=oay;%Cjie}- zCB=xaCU|BIj!>kc2)3@n8!_REN6;@XHWYIu&>TU1-Tf0&;hGZ3ZRw~?f-}(J=aj-o z8&l7~Ty_PNkf58x%KjaG!xO2tq{5;F@TQutL@k$gyx%Iy@jR5xPVn8-)sm-OkJJuk zx-T5b@xHCud|VzRGapdQq|}$ItEosmrywnne32%q&{aZ0jGHjfvsk>pSZvhI)QNdt zDWt{CP`{cl6)BQObL_X+-STF(k-_zKm9rD;OSs^*F`MEcbl(I_#}C^rGKKrO5gDw zTxu*c$`VIb8d?H;_3-{0>e^_h`tl(&=??*4rSkwJle*vRRkrT}d!}6wrk2-m;6a*e z3U?BJC$M6){WT^^bGp8|a2-sY$goYx+RXfL3sy7e5B+t%^(3r|)|#86oQhtgW{x;& z`IgX2urM{3ri~7UVB+a{R>pZli@+%Jt3!F%y&9}gsxHT2R#Y)31or^rF4-HA?@_PE zPl3lhmMPMFu1LYyBe*HKT@>jPBV*%|*dIJ>X2Chr0B z0x9&P=)Eu*Ue9I$!b2h7_*9I^-w^tFg(%+Hj7HFYS>7H_L*m&AEa!R}eWmu=leR!^ z*gBOUPygoi-Ta&9yt3-L-5`Fq#0Ic++(vI+0xp6z)Xc@DRR~RrB zVkN2(VjK0B3E%KE2Jwo1gGU2|xhB)mY3n3u`#N6t=>Mrdo5r}&y3I*{6N@{&ct5}B zk`kTB4kj2$_XMPuSw5!42$fZgao=#&C`%*p{KN(2e0UGD{IBVz`me9(3=nt&Fl_KIkXa-Y=C>iOM83BPeI7urMYxD5y3R0IA7f5LU|4MtpiQAn3l7MPz=H=Z?< z$G-8)M~?lbSA_c++1D|RDW4(l>o*ACN<4t3s!$mnPG9NcrC{7}+uJHqAUd5YxDMiSfdVwjKT{6}q zM*)>qylk_RVC_L~iYn`BM@%R2;JnNL;J_<6rec>)O-U&qu$=sAkzNS-1c6v{6y9XS zrcbqp#cGgQB*lf2AIheN!YN}Ms$3cjM}Sn_r)4RHdXk!olbOxT80`hh_F4CBz1Qd;=Gp3k>H|<||Q@HeBhY3FxA6&26gEk#c)(L3&^lrsVa&oDU z*6dvMKlf??^hZVRi4pB=GJ!l&VtHh+*qyl}hm;gmWuDddu^HaCyAAZKu3ovO+=491 zCbhL|f9zTsnDiZ+mGIM><%2?=_qiJ*gMO0od>J`e*EEPxCX=fThJ@GR&%a$VBA9V3 z$t${{Y$>r=2guC>NTIcM7b(fZBmewLkbq%nKUS_B>HkL8yc@$j$)ng~KBl##L_J<8 zHXPdXkJ!x_-He6FNwo(xa!b8edpblL&au?_@zRN!#h((X08lg*depCd84UWMID70w z7{MqyL%jID?~=Ng^}zI99kL&Z&99)qUr~#~Jzd2I3ebhRUdwbqc1`&+L{M8 zC_eJqPSF&V?~aun&O6~ZNce*E@>vkUUIz^o6!%*rKurfvf})u(-TNv`SItop;zpIB zMIz@E>Wi>NF6_{t{vpn%=MHMzg~}Ozq38UOCWP9H+6)^ zYIO!2R=-eYK{2CMx+H1TLN`#hmW2j)*e?1I2!IWy<%Z@6Fj|EZH@cpk<8a{eDx<|N zXj$2XR=y5TdY@Tcy!za^5Wd{1dfks&j@^6>y9WD%FFZaLO_w2a6wtLmYWv`c+Ua?A z$G+kCaEo2b>&Xs!F5`dOo3Qb^Xn`acIbUrM^A4Am?Vt4a+#$4XvYJ?X`j;hD*|_1b zs*T8V-{;^&Q~2%IBi4t*`bEXtunpk)lP7ya=J(idme_2$f8Kn*N`SB+Vuxu|0Tuns z`{m>mS&)-XJP`5p9_js7y17~BxC~=s^n<8yJn7BbyW@?&^JW&DB!)=Zp5?eY5us-u4pzQsatLZ-LHM6opj;|ej}r}oFUSl zL)}ziV*vyiXeLNnQFI?4EWEBx_(f#9N+bIHTvOW6I|=k@p%0ebjY&kl-fT|A0f*GL zWk-K2)!xd~Z!l=A=^U`bz*A)$I$X^FA73>C+T_@q`+KGf+Q;gz!3XrF%Ob3Sp|1x% zp}MZ$q@()Oy@*kIa`&=J#EX7(`I5@Ub6-9B1dj+8Nrx?w@tWHud>aVdm$;ee734>c zX7Jw9HjwGD%lZClSWN^{Fj6O~obELgLFACEf0ID$kaTz~K>0guz&-6zo&{zI{S5O( z*P&Wbmh6#inQFH7I7F}Xbl`<0eK)3MXUn?t#6J3<{J{`dil(hXn2N}w9C|#;5?aJ zMj40ntNC`6FF&sW2}>kh__XdGdt^2taZ3Lm8sJ~;g}Uo0X}?hA9EwEi2@@^* z_h2hc8&z;8ma{g};r-w-6gW*;uCT6TSJD=5N#4GPv}zIB)nY*$e+PVM2NphMgyp9o zk=U!L^k~wq!=S))7dW@-m5n9r0{kWf_f{XaSw1_x5)4kSWe3*B^ zaBWk&NncY)wB;VfmpUT=LD$5roEysB~e|VmUgpOF-RymoZ zsDCVyxU+9~G}b4w^18FQWjJ{jVirpCpP>(8?J7a@AYijx_7BgJ@jc%{5B@n@WKXp< z1}JD(g*situUby@?TqgT#obCp)^C&xXgSgk3Ho8EPfpzzmm1iZiMtw zq_dI_+02Ah5>(Aa*XK78|9cu^lpftzP6zX|&yg>?iHLR&GU6wPWCK5qbU*Q^Y04bl zZ6y^5jbKX2feXmWKZdxnA;xOlgBF7*)mOHg*y>7Hx?nlP*SQKPRM*e~jv?XtTcs*r z9GjD*#gdc8HNd$He1$?7vx;P5_i(XVh`wwy+d6S~U6kSxbjp=zgo1U`S4Rs8_^le? zZGTFN_o=GPCZ1Dy4dDO2m_QKJm(J62bnkng;|irr!`;Van;lXM9R}TqL;Tn&Qf~U& z9m6`Y>8mXuchkz!*8W+sMhz3U`$UVV4myz7tAbT^1duI)q`T6@{EAH-K?}{+qd;k= zy_f7=$c zbuIYQ7Obl9gA2^!jT!r{29o-7R$zKaX4oGBrUj{&l1`o$vz!nd{+r~I zj4lPUDA9bV-q_+X2Ap}nVw_kOES`I31#V-ks^Fk|52oKqfZIPQk2a)lQicp6%=bzN zSuk!sZ}9e>B`FaTjPjRV;m#T;1( z>J1VxTveNV89CKvvDM>$@ax~(Z?BvBW0zsUt9RSMsI_W1oKjB~P!AsgzXQ00jc>iQ zOIm2OTVAupGE$gr{`N-ldhVN%Ck#m)kRssj;0Kv&wK*okOKBtb1SJ68mfG%KH%@!XvPR4FgM*1>JeT19e}i%TIy3g|w>qRgH1d#R&3x*MxHyZf}K$xeo| zEq~EHNv^0vE$#9i&^SE6bv2%H=hvEV{7C5!OL)B>q&{y4jr%=mo133WHil+?0xf>LPqJ?+qYIFaghgcW`FYe(T4}hI5B&Uq`PR z`be$y#oE)kJK-xphvK=gd^wxPg+3#+U+wGBmK-Ll7^X8)XELr7%yL1-n8k-6u5h)c z*w;%a3?{v6S!_qp+~RU5H;_qxMfBME%p*cqsTNf^0RwO6kZjijO@7(cL%^2+k6R{D z+Y=i$jJ~f0qaGqVML*DE2WO#xKZ?`;hi4d-FL@IS`aNIc3qf-~@+b*CPH*tea%uEO zqMULz72oKm;ZB$yq%F^Dqwm)qW_=P4w&Z5<&%cyIBn}`qQ)+e4gf2(=9+z2iMZ1;D zlojzJpbME*Mc@zs;#xIv%A}@pG@E2tP&U$+P>2IJk{{5Sd_EDXlu?a1DpA2N7JsR2 zG;aZfCaq{5n$?%<`ADOY9csdr9~|N$I+WipuUIw2AR=rijV;moFT4JM*Fx}mAfa3% zP|i67a9QE|`~qPrhv`ujDPskF;8W-~L?C>T`vavF9FU)K2+WApcjQ;?f|@A$Zt;l8 zf_XR4DkJAmlDTm9O;fx7c@KOXVBwm(e3Qh@#9PcyL;MHyBvr;l$&4#F8Yo*#|5i`o z1!mbM5U9#qbKl4WC2@+)yKQem-y28|}H{+P+Qd2StUI#AwC3kOD{rFnnpH8$U zx5A5wu27UH1C9WHI}c-$B4N;JeJC7Z?ZmKTbYA8TPq&K)69WUWM~Vm8%kqQI55Lsh zcPDhirNjTjp;L<#;a1bxFfXWGQ?{t`TpHFrl;F+-L>FtR5H`(k@2lr+ zgq|xVheX!yt3u2{)*oJ;bbKGu7k^`+YJt#$i;1v#>^V>IKJkVgx*9BhrlR?{DmAmaX%4*|&%1tP1Ho9#}ne)U^q@+1X}K*YWBC zHI7j~(xIaFWk%|taR1*>zMUg z#&f6lgJ#9kx!h~}%b9BUnoZMRRi?7p^oO;NFx|JK`AV*ncoUmzq&v1!@5Y}>@2SIu2Gz*Fhjv{fY0j|yj~~0`C5r+Ur9Q>p8loCl z-^AXkMSJ`uf{V&C&G>o!712`}8GbiE456LGzN{~I9GtBh6M=dmKs9zp zN_^Nb^anR{j_s7FD24J+h*xUmE;(scL$%TJ$25x+wtwx_yIoL-+t(@&J9Od%5}sT6 ziWzwdNDILRN`6B$lthMuk2VOZhFtgo$3XOh=#>37s>cCuS190MgC%0s7>(VXqY+ zk&?(8z!X`nB2yF<*VU@H@7K{g?Yg8G*a?#{bAb6C`k-;NUIT+=KxuMh! zeg|041Do1#x)KJi+&}tBZHyTC9wXiSKg85xJwa7J3IggOTvFKf>yb{DvT3=r)lZ;= z+ZcZduc-E6l?TGbttrG9HnlJZeuhP6hGqhjt=3s_;in0A+3NKEYxtcYgwfc#fkDLU z2~8^BQBaSY*oPE5mf)(V;17;_)tl;R_V(q`SARo9+}WkIaHit+q0fd`FP3KU(OSB? zxBw6XFTo!;Wm{6L(>MB^T0EZ;2KF|=eG(_7`%%KK*n$nc%bFcoh*jurFR7Hohp58KAxgtp3a(Yc)w{g=hb_P z=YN|}d^V)&F3HaBmVx!N^kCw6t=v7aV&1}SVWOhK`H&+uZ?M9m3uN_eHU<9*%6dp5H3;qIYhhc=jiNfh+3 z@o_0v`H^0G)>qpbR9~2dU@*|S;nfcWXmvcD5xS_HQZ3FC(k7fFg$79v>WEZ`z zreW>ld!p1|DB!Kg;D7f5+<%D*Q1!UH7gb(+av7B8x{mk#@R7nX$bgE0s+K1??5o`S z;pQ}23^jK-_tnH>k8yC06u;yf9v(OA_1|ABi$82*YNfI(d8R`QFOR?Ia5SYgP{ey* z8wD~pH^?gV``&LMeXu6b{}gehqxnJ8_BRAwxXMrMGC_+8GmJupZh%|`b``IyJ6AlE zzwbuRHGeDge%}BlXPH=~EHLR)m`UBQ6*OT}@o1zj^DhtJnxNr>KuyL`=mqCfOe|sv z|61Y#MXX+`CeH7(#Zw8RQ2e3R-3fcLPkCZX39bq%R}?6jpWsl2Ruc?)IBr#?7#UEY z7)vNvAVvVyknHgqiJocLUw*#A5qzvjtI_{BY=i&y!3Jsx5N_e{{M}!^KzPDWV6#9- zrkoCy4xutGZAW10a2e#PW)d*nIwl7uPDj>pq9IR-GB5}@G7f}msG*xNiHr%=N;}KN z^?$Oegh^(Q_){H#bPS%Td^pJ;AK{-wpc<&S;HQC(O@xI=N5{woO%tZ(B@ZQV%`y}x zUmW0;TZWggCGHZ(nmO?Kx@;I#645Cp+_b!)zh5en1fj6&#GaYzT9Q;3Qy5Jtcc@mf zBZ^N}cBVw3L`k36txrS9r{|tfp>MXT!fn*F+RljA?=Yq@>c`sZ8F{seaY%hj>#wVF z^iwEiFw`(P+L4oZ`LycHw;uMs&sSYwZ?%C_da$u=c$RZsnp=>Kox~DM|6G)b;bDuQ zW0YfcjUsH!MCrvi)ppn{k*Q*}w6OuRj-+(njDt{RHrc_#F}mgC)5UBlWhW663-;QN z+T4~BDbQs2cRq%zVIVv-_YY4~0h#-^Ci|}n1()i&s)jv%sz6xi!on|DnR%UOZEL5| zr#*8v3_Iu`d%rPCnTGU*vhlIilFwS}Ze=OguiLusa*_UnPDc=pcN)CK`s=r5u`Lyv ziu2{EX=S00Wl)<17*))FDB^@l0)z3IfU^Js9W?S{+V8`6`#vgsL&4L3iJ4HGA5Y_)5A+b+lN2Q*2>z*9p7O`=(HR_KK7pR0DYz@#uc zuBX<8_$SV2b>q~^g&R-mZQE>hqGOo`lES`15BK*e z_mm=7A*YXW>XZ%4c~~VTt4d01$SNJP%UwN!8jB%p=G7G_EpUwXonW{``bh0Pa^#Bi zl&B~U*va_Op_RYLgLy44KZf$y#)TE~P=>{+XVZm5AP-lkXCuRurs+T{c@T7?(Ah)Q4x;rqE)t)>zN3NFWaUM3Bpfvqw(jfq&0q()Lv6)e6Rl68J>%RVG)P|giutih zf8RKGV-mMCPS%kUyAB;zv>l9QBg}ihKOV$3($I#bSxbJ6UY9mc>YV}e$0Z%iNB1|C zVNj6D6i8LK&ns8SqeRMhV&0(^luY^|sYYw96OP9@Gvc6!Glqb84eeP<@kP)?rzGNw zRe-inf@e)yo|$jG6J5kAjQy>R>7_^3XXe{)1|UJ0ki7Em+A%0B=JZnPEtKD?uG6|NfOn&(Vpzhj=y&Rwt!X z9IK?F!egZ|&tYJ!nJi|Sv^H^2G`dL#gGTGMF_-|C*}T&qan06@gV$sO`~>Ud#LbO{ z0d+&(pf`ZN;Gq$c@aAx9$TJw<>k@4FDS}JLp?@;!Do?AjMdNQ@@vJeS)8#wU8$r27 zJy&g7Ohji-GlZ+c)ElOOR0552xofXJ-IQ+tx(tS~wGb^moB|BH2_J}DQd_!`57 zeIP+LU8P)*d&II%_{>M6(M>r+v0C&ocI@I?!e9Nja?15n%v<_^PWLl8bXUN=5k@JJ z8)e@52!|ELJJ%kDH|^p6)OK=~_^~C{>GBv&&hj@$HItg4NQ5iDEWb5mnR_W5h!JMk zDsIun0b?e6A(14d?en!xmh3U*q(BNHg3OH?9!p^(t6^7 zb;O7CR?67Hi&@kl9C7|+4$Bz|-8dR7L%M>W$+GDEXmGc?gOfpl-W~=`%MH|aiMMl6VUUB$v3aNu1H6%u=O)v-v*Wbh=bd0`-2 zZ5h>7NQZxCc?>WY_7@5-nfmu>vSQC@9IQrQsHBjEFaIK`&yxz%yCn`}RgU7P>EPUV7HdV-r*Q{#f^V_zb9$M(r{7=nK`Pk6rh| zZWqd?*6VenG?EXV^4DlCQdLqmSi#*J^cS<6w-j8Ds(?qd9QcZ5dg>qhc(u!Mnx&!p zyNWbVu%XP)D<2;EN7A_7UYV3~ITLW&Hw_{Xc2xI+S6R~Fb=Rhj{Xl)C-Q9IJUdnvxL0_(= zwmuh7j)3i~1+x0>ZGCHUy&^Mb?J1MObhf(rq3cjkX?*JY8B)u!o6{JjTa#J80mfhl zOwQ=zHgUD$X+y5XvEq644KwuOYJ_^I$fJg)^bg&?6gK`z?-W6whWx$!J^A=yW-T~EzXL2Fzkhv^BMulhLkTu*> zHWVLzDMF*Q`-6(abZ5hZC&Xyx@9UNGlS6a)$WQU|a@MJ!R|s!9`GhQCSuE0S2Tf}hhWwBzW2{Fr(Sm;h5 z8$3<^PDIhujA$SPhtng!LJ}t&U)v|&fg}_g2aiO<`Y&=*UNOdns59*VKJ+qD8S_-N zyN0w)XHM#*jow!Ka;$Un`qM)ANE7hCZ0t;N|0lor5 z0X$>Gl_9cXxKOaq%XWS5X{UqKRgNI!Doqq)tWV}~evTtHFM8RbNUCc(LK>(ARCU!_ z!Bi;0n9Iz#H@*UW#+v4O5~E`a_E3YYMHp`gB&)qkNprBv{#Q8sTY*6*Ju~XGUorJ5 zT#JNsCCD!zwE)VpTBa$vf%i3MNzWow)j@v7&g!VlP z@PUd^X#6NbhQ*}!++QBvU<+^fJ>KML9u6cnaO@8OgtUnv9ljHn1<=t9I#wdAm9klH zOcia=x$nyb`KL!Z7;ME9NJF`Ed<`5%#OlR8BeB1dW&jH*N@Gg#ChKgo@Bzt3p+1cc9^--Pb||SHMT0pOutcll(A+ z!*QQOR_$=;RGc&|Cvy+i_{4k_O@86Mg)E*e2M-?+Oz1V0yd$?yzC^p9{{%}DLW?%( zh46>(x(arZ|Db{U!|B2k@nzP(O7jW?wrF-ile&t#Ytj|7tFyWQ%zi@T-T;U!X~n;L*-yWC-31 zcXlvF+&VugybBzgE|K<5eS8s9b1Q0C>}&5lkV}kv36*5(5u{Nvh4m+0=BD|yUlh|j zA}?2ofPa~x851^cMi9eMS?Ou>L!lmfm_gmS16ERgJBy z&{(W{^;^&vS0#BqOg77VH5`@Pvkafm=QATY0*=1ZRGf{5N(2bV`CpwZy{9$3j_utFzwDIw=A8|6HOwA>hZAI&Q%1~HL=Uf#GNNQt;TG&o5)q>rM}7lskO zaSY{6K5P-IkLX9_T}mwl_I4lJq<^Vh!ol$gMz6_lsKj6RekP)tg(&ot51*JPLj}VrD2$Xy zDR7X=JXTA`z<F?t7^U%?GvEiXI@mw>>y!S+I-YBg2Rp1A%9Ld8bn^$MBJX@5?`u-3DY z`RpC4dDm*p)Z-M>yQ@Bvj5>YCHYuIhxt(j zoz&oBF4rd#qxH{Q^`)Ns#-E&K4cqmyO=fS*Q~hetDOpT>y5 z=*3exu8oYD@J2Z{HLq)KS21HB92|@rgeMXfKhaq(CaE2WPj~uE>Hts53q{yrg5g+T zJ1Q*NAg^NXw>BGEGw=MuiR-6{=kMj?87fy+hHPY&71LQ)hj`PijPqcL=?Me2iuRwi zL7szpscEwPS7JQ!Z9eurV3W5NZkHS^7#4*$LNP~SE>14UINT~EzQ-Unb2?=M>xLmGO2lJt|xl&$U>t5GpRm_2wiP>G55V! zChdpI@dP^-T-*ZANNOj|>=xIt&&#PQvB23o{Jl+~&xR@&@U>;n4;_j0vt)o}z#Em> zk&(6)rtI^wJ=g;;A|i{Zrx*{2i?w2b zfW?IX6{$kAh}p);rnJ{y-BLr=&*t1jiNgjnY7M!r2S%d4}n*0dTy}Ka!BJwU|(xF}XFP-DsCLI1$#XU@OQDnwArS$rvoM@(VzHc+y>6>k_aGEH_ z2cB74n2HCb2)FhZ!5gJza$=qt&xZGH-zfEFa}h0P49``;DuXR>ho-W|iL4yyXS=3h zpHOOn_ux#QOrqG`!ERHTA3~JP?;UD-lFoe!{GmofkFh0WxRiMwH2Q*U>%ev5)K)#3 z&3Z@8g~PWfe7&L2&*D7gqv4YmJX%sTUy@1gbJ%?j4J35bjDq)TlgLReg+N?is^_G` z^a?qQo15oG*|}VD`1ttbx{__a)^{XyVrgQ(=-fl}U_AN>-= zVNO`uey=s*!o=f?6VuMUqV2k33^J-^Xls2#N0ytTY35Sp@($eX$m?Re_Idw(+lJ8d zvrE9sk72dW6Ugsjs^`;E%)1MF1<7->*B;O11?IEL&UfMz;>kb0d!g@>fAP1SHVGE< zZth3hqqb*TW>8`CNeARoJx=HaJBK><$`00-MMJ0>s?rR(@}~8NyQ%`R)KTs5I~rxYSa? zQb3xt%_M9fLmHw?OSJ%E7(9Y=JJH|ux0Nr?HwFAPx1Y;WJw>|ZYiw;dx_qw+e|6kX z=jQLRFN_!~L)bm>F}hWlTs6^@B9M7fEO8MIa40!_K~N$PNrg&T+glcZZF@H#7Q3KE zJv#axEl1Su+v*6L9TpyyKj!>J) zSXXH*v8cx9qoO^xrw)z#Jl7hC4*>Wa zPA7fRdhwHdB5(QL#|-GwpAs!fbr%a~gG6V&igiHoFfjw=d<|ZX@XBd&ml(;J2uVmU zNpWO=ow71p=*Xuf>d%JR&A$7y6qzv8_Y+Co^L1=&%NKdc0M2gx&91|YBa0%3R!PEd zmkz<{Q+U5Z|4O9$_*LqLgz`xQmC@5U!`bB?jrlwLcBbFFD-B)s`}?;o$bV8zvg0O$ z>h;McC`j)UY#M&e$*i_v|E@~P3gKg=&P-8pNO7E$nMU6OvSTd6VV|i*_XXOe3{fC% z#d-E;CawpTqvC>q8v|OB^wSdfibhyyWbntYCVd(i}2*G2)@Ir=2`@6 zA(^Xy2EWT0VveHofH{RcFa(o2C8kpla};Ll^DMQ{^bvQdd`B1T0)KyPb44G(&XRz` zvzS`9WXG`<=!?D(Rz7ct!`Tluk(xG_a22ceq*Y6x#Q7cmh~qB@xrAkjUaHI*K@Y~( znGe4IS`MjV5LNBR;i#v^%H2l)bSEsWWygIQGBm`I;)-3Atl_4wXjHQ3+Hz6fJ36aw zLQSoM-6jCSma!}YR=|GSs1Ul^J5c#tP{x(*_V1T(vUHf!2ilyCF^32P;8QCRuwA!g zxO_{Lm<={pm>kzD@P>IV0DuJ1ZC{!s%aVAjfyg&AR~kT4#L_0trCqeyf>S zmSdj2$}PJRn?x7^-S2$w@=2L>YH2|FbVnx~Wlj!0-aVN`XEhq<+?%hh`8*JZs{e+` zd>b^dPT;CkJ-P-A7lix4!CB4xr7C4RO0UB~8}R`%q{ol`Vaif6hm15gi|O;5P2#k0 zPk@Qo*Z04NQV7Pq#hb1%O)di6fkw?gu#ivU?$RtR+(_7{0dXIHQa9h1hL*$%BIn)> zx(lM*$@VfyQ)ubT7|yb#=J@F%k^|4@c+tbb2_x!nFE7e@qFjSjrX5*9@wF8n&74R? zzXgMszrNjn>DC_QAiNgj>#*_5sQfg3Eelzb0ngo$m}~fOcQpy5KqiRmw+YSdeseH_ zz*)6X1i_w~5H1pFzwstM=kW{<4Ac0+KBFO=0cwH>j13B3VJLCgwo!1pZyd>0ySxE0 zp$B&BuqY=^Sfx@jg=L8(yKM#l06gi}k#@P{vERZaKj#F^K}9*M1KhD#k>6WtNtwpT zg_-1vPblkkS=?m;=wz>YrD5>$DqM2Q?dK*HOV%W`)V5)jtYlbB4)2hu zs;`3`vCu7fKkZO9|J)3jA$$000iiG4@GVkmxPfrW?Iy0(X4JW>0qO_7C0-;~>vP;s z-z)5P5m3&578f#!)MxEDn}osAY=lhG^K=W$cq_Wc^pPW;=lE^B(Zev%`LO_!vWy$t z-7RyiekUFqOgjUM?)&m}n44?(K~Xj-oM++~cN^zM5PFPz^gZZay1M4`&csSOs>ND) zswH7Ynr&J6p0#aw_sF8BzP-G&7o$tnS%UMc^iLOI7J?JODikyDR!OLqdj{qSJNwtq ztrZaft3CPM$t~0(KWii zEyN}T^wMA3-lkx?c)GJ(dbKb&n<;a`PmPpnqshZ5bq1z#%Yi^f80W2%cIfOwCXKxP z>+tLtC(tB5pF%gheAa4`Q@}M>`a%ILgdjZ_QyzUtFJX zxy+HqZJ8CD)s2t4gWbB)%i+R%QVpp;ROo1c+9sqBk-{Q%N}#M^wtJPyH0G8`ZBeJA z93};wUhtmo_Kn#2>x@cESv*)!H z6*dwklUY{1ZT1$)dK8YbCX>R#B-uw(-9Z3+ITsc{tdtK<5pQ)>mD8!+^py3G(BO&G zjv&$fWXs>?K!(TP$ODR|ZB}LxPhIT8!vg)5G6wSe`KR?=?19+*B{R(SA3U#Qv`w9L zkIy!MAiylX1}m{z3(0; zSHDC;*5(*}l~Q`av?L{282%tGS7{l1^cwoZHS90cRg_Uu(Hh};s$;m#IWqVjX(Mh& zbzWt1AuE)hLrITDrUF@R9>)Pn4I#nmBpz*o0>qy}NyDs}dBDZf!FGosee^Y7tt9=u zwJa?ilyMKBa{E*@`DYKO=Uw~r`_A@GUTx;2k))GG#6I__qmr*72=#Uz<|#keAn`0AeTY z2ot%C+7Ak05!IfHrMczEN6c4g@05r1Q@cM^wR;FEUpu&$bR5-tsw8nsdcqBRqh+W! zbEI?2`}Aa%!uhL!%z)TlYK7pRAzE76u7LBGGFz@5YV@N4ePhK+A2Rb`AM$<~p2D*K zmjzH|@a~_XaN+c_MZ^^(a&Fr?{=LR>3X}jZk3;&?^N3K(B$bqRJ6WE=y4mzOQQX)(&I*Bdly&suSCJH zdxkBfnyGtLHb1n-QYA)IFQU+}-`cAu+${+Sv|0p6h`w#o{m#USY)kqV4Rgv*`wId5 zf7E9QQHTbSMmSOk<#b6DgZH}<*qPR(i_Ao$&WITspI;L|F1kBp;fQ_`!V6b_@f+M4 zx}r!n(7#*W&c;ULD9l(Ba{WgR2ZM zGFVCz9g?{I4iUETUrYY41vMHD%=jbHKg(t_HQyt z{Ey;&Qql8q+!|HrN*NGUt;!|`aj_SMe|IO(C*}fbA@BoTYtU_w}?^S>w z68P`%@BLr1VV?M(p7K8z*r>23gM=Guko~utmBKmHgVoCo+iFhs|M0K>dDK5|OAdxv z*Z)s0iTl7WIH~%-tRj?V<`;=7ZOyHf)xhVs>Hm7LLYkTBeA%{f95P;wHKj_F9Ln4L zEDBNjPMPx1HRs)&|LM?&+K7?6!DZ?=@AhJ)(x{8_!jUS3(b|6W>3HI6&p&Lk1? zy8Q^!o>YdF^jnCOf&$7jU`_Vb3v;brL(Z*cHq|HL#az2rOYi5>rbFnUfgC48^>9J= zO~O=bYwPm2A+VmuTQ9J?yH*>zq!1o60eRlv3SGAUVH9#W01RnT(zUNnCaU$_!(Z~g zVJUxr`2u-qTYI|@STg&lTAWhlqoj*kF|6^WvF+@g96#7OI97VsSD7m)NP1yPt6sU~ zu7~v?vW_5XD=WJj8Y*vmC4vQY)~;`d7-SzE*|S(54-}oref0Ed6=>7_$*=Qs;o#u? z07qx4U;P+nj11=DSPZxmLFnw`IZix5?R!{f7_tfE`<6pI!`-i8jRLON^YI?5*t6J3Cr>dS=&lr>3S<-zu-FDRRN;uxJBk^H)usxAaF#OOe#nlsRo5 zPCQ&(KNXnfh`jyC&~VvM{5ht@xoI>%HT9PQ8|Fw=@oai}GCV<=i~)N`C6_i2T6|o* z+FPZu#A(*L8udlgvt@K!$AgsFgC&dko%o`%NH0$hS| z76(byh#Xb4*fGF`1wKB3j(M$=rpEclXtjv%OBWZa_Jh!}ON7~(8J&&8&owoZ)04`F z2w1TAx%b6KN4gsfQT##sukV#rFITa6#v^&T#W~VRzU^la!bX}uW*-w_IRU?}jIbGa zsC-#jn}Ecx@%=iFNv2N~ndS4(Y*hSi#pS}hW!F!Wc6nopoF|+Xe@!_$!!HGZ@47{I zmzf5!GQgCbzsX4FwNl;P8?$qLpZU%x!EOylf)t9+N3in6tC( zl}1t1(?9-bscD=iV|D*n0`>l>#_KjQmw=gg<-AciulGrw( z(R(^y{hiJomfzu;e<5l*h<;b>4Sk8-q);cU+SC6Y7@(=C>E(3;gS+JBIH)P%9&vDi zyar_O>53(lI$N;^pQ>p)OpM?_EYZnW@^kb@0l`DRR#&_H{vIzcFT-e2s@Xi{w<1wF z$bK>3BEg5cZ@p}GJp-0fbMp-}4Z@rdA399lv z>wGSLpBw&lB@WgJW%C_Fj>si#tmXb%u5ol>Y#*gKw{Jyen))5VARyhZFOR2tZ$8)8 z)}FQRBshNf0MmO1`QH4pl||nQhu4a6S718Kw~DshK5lTl(v1KWNBfQ^VLRoWJ|N(8 zq#31*FMc(uYu}EYmV70@&Z2&T;T{4WPus$B)prIlqTZAYCzpC(QpUxwDpagL$l`k} z9clb-C%aLB-%8gry&vQX)HF6$HJY55=z?js?P}K<&_12bmL>gFm`Dbp6ONT(Eseng zMsDrPo8I#C(h*e3A-^aG*mLJB8;7Ttmwc{hZSCpki1;xgG<4@7Gb@KQGg`` z)03es(9Y*n!0Gk^`Fkb7yA}+2?5XkbBLlA#q}(>0cRzIA{k;4X`AVLtr?Km*s}{PeT!(_I4-RZtnea9X&m zG~*1m2XmU)_);P8{G?&p#o_qpmv5uu)%k)DBZYWku#bW!OZU&ESvr~H$A|grvx63k z0pu(ttgg@}-Dj8{7wJn%3T|Wi*$=aq3Q^dwnv;!fK!w-u`p{w^QtA5#b3RLce*Wl= zHD~DM)fI&$4|G@?X7rkdhB*ZV39NY^Fkf~!B0uNlk#`a+TTdbj%SpjIo5U{Z>yzj4McgwfOgLk<>T^+!5|aPpV2sU%#4>_kD3gb?OoVm zr__-bGb%xr>FMcj-n_xb162r|SRx1MY_(bI;yQJ7bYMQ6T57Ogk_x+*wPi^o`!=LD zCYgZ2$UPb!ZHWM_+Pa>H<0#3etqcyM5u^!_ISk2nSV~^{jzjU=8pp{JNt&bPte6Jh zN6awfPM#UNQ_y~fcW_4kr~Crsd4By>25WI?2`aFNgO^TJ4Kn|pt)7zsb(#jsy+u>HouPz0 zNf*M2)M3bYt)EW31%`_Dy2#*600G`IZ)mdCU1%4w!4Q>|7E1{RE1C!x~|iGKZ= z*}jbYRrflBAEqYB0tXsiFbpt@HShzsE0qk@4xP4onv}&DE&Mlr0B$Q&c*fP5Br5|i z-6ZA4^j6ODwJnJ9d@X3~L$+M$sobl)uh zhSBo+ZhLOK)6&)|5O$rhrY!Hjef(YU=}Z_R36q_}P%|6wz6{hfbv%uiDnc1aDS{21lCC*~Hq&7i{|!$JJwA zyVZ+;H8IxM+FB~|#Q(9g;ksP)VZ|ym@>eMU_t1I&^i2A`Kx+o@xmSt?PS)PhG2qX0 z`d~Q$HbMsAeOcv?MZwKFqwj8H-6P3SREJHsHv(Bh)BeQTtk{*}!w|kiT%Dv-*L&h% zdYV96i_w%eiQvk33SLW}K+eQ>N(OuTPNJkHwW*1sp$Fq-f2Rg1!}R6f|4s3J`R&e6 zfHBTSJhf|ckkCIDZ`3L~njmtC{mpaN*WC@HzjP>(f+6z`>cf_$rQO`kq<(%jecI97 z+B_#Ig>;%M(NAehE%}i{e&x66D^U=ITKo|TdCJGtBNpw7LEAn3b(!gbKXgud3&;Fs z@811{kQ;FOrp&Vj>gonG5Y{l`n*cRRAshJ<6Byy;x28?bvPwz@^6!6C!SVtnl7j5j zH#|Z;8EeE2e)NQthw6C3xvwCNCOzs^Zzg@Y9AU5Lw_PLQ_1ooMGOLb%6FCk`>;6;d z9c}jxMYfm9gteSEOO$M`G4dh{YCJ^p4UqdR>SHe`>DQ+}-NjOc{+FX`-W&0^N3b^r zyqqZF9r-1|?iSUHB^J$xcu@KE-+KHz7>Lk6nAmEvzrP=5IT)c)>KN;^ zH|Me=?Nkjgcq`pM#2+(NUMzohbO6k#2Ymts)?ua$bB9W*fNLC{yEw+Tc|fGKb~m~j z00RL2sF1i>>nzCTb4<7+qL^?lR{^eD5%th08tiQ z{;C(+?Sc~~gSm_h;B(1auY>g&ur7*=Qt*uO(@6u2{PYB)-tgJa(Q}q(-%k)RK#4Nb zB>qlkI{ZGi=k;v8q!O+7_Vgn6zQ)nM&F-hq=U-}~DC<8sbao_q(<(};gXot4=ajD7 z-BjQF`5ULW_}?ldltkwGj9AwXE2h^|mx*m3Rt$^;)5~*d1jPJg7GGwK6Ga_*4&Cxb zF_V8d3N}#e<~>(Z^4V4ZrKmEyo;N5&RTgVjx;7x)gWL*+g_&kGzj}`+#hD-`+^@a6 z){g668$a*Iw@K)?S-`rvgu<=^AIo}_h=EIw1QX9HW}4CA!dla&!1)I?BPKO+6o!jx zV=>>f=$AgT19OIJ4Ewv{V^{tK4&R2GmII?UmzeRf1Qn&PL4V?Jh8{wxUP_wEg)Nq% zP5MoFfA_K>ulhfBtE?M(ZQa|heq8wr$B-RMlKz8#qjQ_6{{1Dw`*kVnN^S{E&~0#b zknbj^M99Moox0d#o!%H5Yw@;gjW|<+}U6V9!`O`fz_uM&*T*g zCVOWGd2}M3XKHw0PtApP={UGV8?oo`7BIcn^~hE!7}NGhQ8R}B+v%PKrJqXoSF04C zm^2?cdv`vN7w9a*C6w(4$o6DIYC#&gV>_o+BgEEH<*3ks0Fb=e4mMB6Z| zsbN8VD2xr(_@FA%buuVt*SQRi75fyBl7t>-$2;mO3`+utrK+M5Mk)@2Yhs9#)TQFM zaKv(<@8E>d^&yPzO$WX3uG2t?(@VC#{jMFzjDREQ)7%QtC}k?}+5Vwt79r7nF+>J4 za6UTCD$EYWiuJw5Vd5&B4B4~gotThN_n~(TVo9Cg-deS>cKi8?-*w*$xa9}jW0fpI zLUD26*yd7*IJuzAk0tVPau{PM^ERiPgMyZ6M6R153gm83*!~FuPbPO(ETZ9fplZfg zEG%dg_*6qdr*DwR4C17NhVgH;e{L{>Xy{7Zl6TJ)1=MxjvcN7CHzv6q`{GwW0*j~s z3aOwcJk7|7Ow;cexcJ&f!s?{@WEv?5{VqEM-%FYJ2HlhKE6!DRMrTQ^d`H$8t!cJS zg)taGL|9HMjW{1XgeW&h^WhBloY})6)rW3hJ4VS4sw%Y`?*z{EPL8)pT@(#=>wJ^> z0hbw(q@Z)-XFgGxdEwg%L4b*B;aqbNw>YhJ4T^#PLI1>4Mv`u_|nB-}fUN z(SKEk<>`Y^M&xp2wj@7Aq!c&L-L!fII{W-cn)Kn&Iur(;+vd1=csRNly!WzsSpTFZ z*YvTiGz^PUb;Fx$QEi8kp647dJlM|aPilV&4`FCOYS6v}%4EJYylO=! zkHeoL&!>jtkC$cQKIm<=t(0UPWY_gWaxa(#w@XWrdn&Om_ey)EL3UhR@#7O`o}HH3 zbCIu`2c@+g9QG%O?2!zuKYvgkslVBSO-7JPh;1CO8jKhPO0bzU+Nf^aW|d-@`9NvV zu_W)RBBG|U#Z%Kyr@POOx4(-c@#eMuN#j))wRnB5ARx?KeVwt|BX}2VKipSe_wq;m z#O4l>rYd+^N0adAlLXHgTJ>OzHR2&p7AZUHSN5+}6;0)ztj2Oij&mMBbm*ZsQxb9D z4R+agt7bs%YwI^O$sgE9x${l4lJRW2n0;Ja5x1DFwLM7*# zqOm$FF>c0@LZ-qT2yid{6RzRA!ibtpY!x152p%}xJ()SmpAj9(S^IMm9#b6uKF@pY zBat6=SZ^w6IxP;oaXm3By1s?ljz{E!f2bC z&`-T%%q;R@pGSW?;S*+3vx#*KZogY6lPgHi(_sGqhKSJa!&uyee}ZI+-EJEu(%+ZJ z0jNz$Q>ejgwEJB!Qp-K+uq*-_(}jp7?hklNp}r~#YauBNX8jX|Hvpur94%}xL@g}v z?-#RQgXfd(=&da2oL}*keCX25VF{qQ*?5rDmG=IyCvAU#)}b>DeaJYulq(vePv%a< zvS|`!j@)vp5Lc;)hNmO7dmE1AVQF`|WX_5o-iO9;$*xEzhv1YRzWX*fw$!G)Vr0;i zZ<_^EB`pMhxU^JYHrQsVPOND&GGDH7KjX#*O-Aa&KF+=plXh^)I!hf(94JfAi9zAm zaT^-bSi90JrK42QOr<>5o=R>{DW-8vSrSisC1)ipfKW4F)A_DA90XB9Dd;Iz$&4K* zqPPFZUR}PW>EHFhEt^A|h&;Uy<4R}o7oEwh*F#QMi1OY~KU^>sQVF{Yg}Jyk z4m;}Oz@|TUeMcu0E5C%u&n)=fFepgu44rY+(BbPDMcd50)~~_kV^p7*kc|2g%AvZP z@OR{bF{f>f6lukInN)+lDk>`nnV=V*9y8@a1r&wI_3ufu?@8acp zOU{oh9#6&b#rLIQpM%W7)Q|N~+7%C`BqL+rZ5#reViZR4-)zyHhKGkC?YBY?*V>7t zzvFIeEPLSPVfqFg{Gh&batij z{djbx`Y!u@%1wgk8Fnl6=tan?|J5Yq27&>P3C^x(5xouL2gRv>9>9d@tmwS>4LwYoTRCSx8QVJRVk7GWO0SDI!PI%zeeg(VAGV~DUSnJ@6#PK7JF61DJz zO2X*T?T&3BDkodL7|d+_`YL6%a5FMu3u$g}7ed@I!#aeUZoXqIl}m$mdbBq%@)2(0 za~EMW@mtLfdSoYVL34V_@R%SNj<;M!I!nbo%?8;7Y6HZOi}3l7^ge@ItCLs_6Lgmq zo$cPRshfqR;!l%pjq;AVc!3mtIK%=fMPX1JTThN3RS`E&`~|x=P#+|r6s?dM{exbA z9L*pFIanptAIT$9!je~2zj44x-aR`V`wh0qFJx4BM(TD3zL;XfM2s1=K2HW2Ed4?mG!vfqVt>5OQ2gLRZIo0vC9uBf zehK?%q@MlU`{4q4d?W=7yl6tt;!f{>a7AOd(cq$G;eQXMhhto#dmg2tdRQSj2=IQ^ z%dLI=oB!%F6q@iPD{;Bzk($1?e>Ur+fFu_hUsD7_Bmc2 zuYv-$kxZ}XQY5k@0c3(ML7QCalCRsii0v4w3wvkZ>I5X~?~XFff{1(r4}?T9ajfEU zqFx_PJh(VRsHG`ry7ABd3TAFxRSru$`+ud2dDhb5VP5WON~6+xfMcLXpRj%EEvc6+i5b@m~+=VtYos zm|~IPA)+E|#g*iH1pM+01Yp@e2utcH?*mFE)Ylr2iJ$g3zjq@3qj*;qJSRecM?@OKK|O-bNowmkUBxLu80KA&b$|UQ=2!TZt|QKK5l^o{&s@#8}E=UHnNy- zyLOQgHZpmbD-+>DF`?cP4F8NIZ%wTtHejav_7V*b^aP}VL){wWT|~5mlT`vo)ng8A z*MwNh7qSE7QnuK|&YS1ohS5`xMi(YOCs2qkrO2^Y)+NK`Blf#|e-?20b>U2gur2Jd zUj&af_9Z}wY0!gh&(r@l08?@We5&=&kLl8v!fT> zZ@%qj?>^Kg59%;3?S~^C(+gt@}i*$B4 z^3^hYsrZ4U@VP19?ai$kL@-1ia?a)un~mftLv3Ha0PwSO2V!FrMuEa(;6Rv23VhE>3su>hk^6dnn2?b zUK#+>;NLflI&ThR`cxmNBVCe6X$br&Wulb;LJ30Gn;7B3+JyAd!BG*vP!?^PnzK5& zr9{vcF?LQON_ofz0dHf;i}F*)0u@ac(0LiTjg%2v56jAD1;RhBTx>MkR{4lt@Ou9 zYKs21j*kkn2VVrqeR@56?Bq$lI3CxK9NZ+(Q#S2V=Qfk5E+ODWuyeLp9>!Z)jjntA z^h3&gx7Jy}^jqz_Pc+m~C6g{OqF(fHK5)K5bNJYSXiISwUsI|sUDDD#(&RgFbsGPsfu-8b>Fb64QOoGj$``U6y+DD*AK266<-_}& zAB$fdNid36_Im$%ZfaM)<>aXIxjNko4ti=p$I5Av6!Q8qj4u9orW{LB_|@?FtdW2y z;7T>4Lkk5YV8;GVYWnHdfupI>-dy9D6Yc=q~fKlnQ61x|KAxpwlLWqJipqquD3Mtq1&%z8M+7Uxe_O$ge zBvvRvxy4C=ERe)YpDzg^nnHP0kkeN8sHw%UmB0tq(7Tl8E^j6)T^2wJw6yRF7Dt^m zZi}sM>Lcdb(FH?RwqJ6#Zvuk${-Od)Wl4<2Mf6iMXy_0;M{*$@Bt-~FlC?Z(&2fE- zAY6Fb_Y|b#3i4#gc%ue+tVtw51mltD7=c1dj5LrwxOX>>61}T6*A;HRM9VRA5J{8jhyrniqN`Z^BIbvyvWJ7k0&yDt{7?W1zwzy`` zpd085Hhq#=z|0(>R_Zl^fJ839szrDpP;O_vL}%8><|D2;DL@f``B5%wO9Fui&^u1T z+cn5d02{~`O>T}2D7APVX??~@K_*FRW9az(hnk={UWYm4c3V7b4;Pg`bcx#v;Ab34 z=|NJ@vP^G#Ei2&TP$X3&AD-P*gb$ATgO7`TvHrf` z`A|X3uj?EP7NcHZ9HVjPHl-cSB%~enwcZR_iYX-LLH=CN;}1EhL?emR&Hk9X%dqn8 z^15#{8A@#3n{=yJ97+w&t?Vw&gV+ho$e&pXpQQOx@o6A+eT(VajV4N{raF!egv;bN zfpg0s-NsGngxvL2DY+1Z2N>OjEztgnl>l$))fU6cw8LsjG{ z4r{5h3^Gr+zdF#ay_%LRq(mn0$+ExiVqm7n#y*@ZTAe%JExwwHxr-u{yjUi4X_7=F z6Vuo++0k_|fPM0KAPlJ!aQ<>}VwIMfpPrI4ksJ79QEw#IXCCqlkO^}2X_&d}=vaL` z6Tg`8ZK5{k=l^(e(&%esp{#s~6E$JMZ(}L*)Escj%F1E;3x{%K^@P1(rXN@0`$9h! z#UGMOJD@L6!)8mPhM(U=JYZyKQgqQ}>4Q@Yu48gdH)BFBk1s%(J%N~PKs1K4ZRLUG zx_vOZEUUac-QSm13M}mc(k9Er-uSis_LrkW->Ye%kRSP3*Ro#WSF;CPHklo*`R%Qc znq4XZ2L&}*17F{O*{RTg%0c93&WJil@z@i6_xarKdPXm?$oc()edC8y4~2O4_Pbo$ zI-|CRs`l%>pX+|!_p#aYx57u^rEkR+4ZW?XFUuPxx234aVDr;E?)=Y3Ti=6T8(l6W zmOLX{u|_bLeko6L2KUoMj-NyA}p{cxVJW21gHM?@upszbJvmbgEdK_%yIZi zFN3w+AaTNdph0K^#*StlE2Ii4JS3D0mxjelOc-~I3erdUVPr-@UVrXC9DVMQV2=rb{8RzmMO_M208E@um^3m);h3Rj#*A1Bry0*AO)AWjvqs z%0Z!hNDO$F=GN`w)af)V&e&<#n;5`It7sdX0VJ64L2pU#8|`Rt0q+8^4jFJSuNrrjJqpGyYowr^?I5sN* zpwAEskpatNU{yz)d*oxihDux^s@@13ig!e$FXC6c5e-*>mPn8#WVW0mhB)GH^<*^a zWj|R64xefQq8(YUq7qUN9i!8bqPtQBvi>sD;IK4Ge?7Y?H9htck+qKWP6NCPSzVqI zeO{hyKb#^R(Sw>4Z80JHNO(5_Pme|w^-Q0h0ed`L(UudQl31;n<6&jsmb={CN%o)M zn#*r2d3cg!Q(6V2)pQQ2WBe^xKO^cb5V*QPnlkRoSURb@WrO0;3MRTX%D9dDpsuB+ zN6dZ=1)Fzv_FDXh(ll87o8VS5PVjjQ=b zWXvMQnO@^pgJEzQmbl2oVQQvX(!?raV(0lAYhD9T)Saa352~`)*!e)gJG&xD9PEwp zRtot)J+B~1d-s)Zuibq41Y;dv($ z8rVrF3=AyEa0QA={jfYsV-A;2PWA)CQyIm$E~CvY2m96(LDv@jki+CwXD;K0OO+c% z$-m(d9UYyztx+j0+CzRNR=z4pZ)EV$z%68l^RKkXQbOx{6i^q?1DhMO)MP z78Zxv+8E09O*P^^d%Y%Y5cLpW9DV+Qdf_(t)u|MWpZ9S&Mx&2p{w8qq_WM6_z`9?i zx?{U}*vw#}gz4XTw(rBp*xg6Z-&`tqk(|P|6No`4$821kk64LMN5#WEkNH==-KqNu zR#FdFTt6i)59hBia{{JFnC5Jeojy!ql>A=a=ZvonwYQHH0`NRo2$K)keKU_t7kjNa zqob_DsYL}q%A>7?B>HydlZb8(rHQ@Wse8HOa4MsV5#L>YgdaM?P?aV&968UCCblv= zeB;q zo`XDQH!Z)O9FU2xHierw9_J=zBZaXU0>?nDf!2c%g7HENo1&dD$H&Ev@Pf*HB@Tl_ zL00Kw;F^MCj#fHij=1|38jH$^2~v!(!`m$>1ag*^Ca^d;X0dv zD;U{{T#S_7`Yqyz;%G5XIpuTz%=c^%_Ta0m3Lmb6$u8E|>CnELyePvQ5{;uMwG?jr*ki_>s+ z4IprM$Ra88Rm`Mk8bwKr@LBpJyV2JsBZ@MgOzldq22xO7pKgCe-E*S*y&kE+>19uN2!_w#*r};G^U7xdX55nJ9c;vY843 zmfbS_e|zdCH3p}ue^xEvdn?zK{%Lqd0Tk7b#nU!ow>S~d<2yA?;V#qh5yXS)s#rK(ZT}U3TX!5RwibzaYhy=S{Kn;YG@`NaM{aiM<<0f zsF(n^#%BTc*}k(3SI$z5Gy?#e<7?%hLU=YhvD`a`gef!w8YznV`y@EHNswqJ_5rn7 zS;HYCjGCAOS=Snk_lgHc!R$epVeS*=z&JCcOaM>HSe^mukTcZ`0Vwf;s(?HrHpVe7 zT#KA8F$+T}B4JEe4#bQ-Iw+tS&1y*;87r$w?g?4w)8+0nKT&|tre;Rt44ClMBPXh3 zZhr|8YZ}46MSpMcmHLI@Y#X>8A<{^vimTI5L{vy%WfqIN(3Pf0-%Cpe=n5YaB8D40 zfPF6|e}hP`O?Y=9!n|ZDe37?>ip_+pY{+YUz=`q>O9HXX6F;Luh9im=S@w#Fk%0b= z4#Bcu)hpi)Hw7jJ0TKW=rG`2tQxpklL|40-v12DJrsewqI2chaEgIEA&a*hNjM2+i{=QJmOA8r2b;k9Hh55((Ht9fF;XE9;M6XgPO49aTmP+3kA7CX zOZH{s7`|YmPPkatF3<{!JR9a4?)w|7X?7Zd;LRX9)qc&SniUL39Y;TEk>dZDTwY(( z0n1sr+plE=g|w`pX%LdX1ovL`=K<|6Y_+tLc4P3EF3B%W%uzY=6k-6|Rb}G!J)02T zsUo@(YDUp<m7}JsQLwaO`Glp_x zDP2X*^77aX>C_Sxr?2Hc=z>KII*py9;DLxv#Ys|rM7VP&NAJB(=U7g^jUNqIwHxFs%Q8y$^35m7 z3CzU)OQZfBw##-zF1wZK3u*C72sUbxep~*XzMfMtF^eOyq~wy$6F(#$3X!6wANT6c zHjS>R0;XexAStHkJj}-^|er^`_x4(MyX7^Q74VITO(5 zV&AHCe)XV%bU`$-XAhG6Pi;kXF-!YaX*pzrer5K{@vujtjmd}_qsSng4ko|LNkU2C zX5GW^wv=<`!#r0NZvgvJSUMWIDwfD){Z%rN+#l4I5`tZDmk~ZJvvekA zDe6cgu>jCQWY7jpSm;KuN6UkQF;H4QK;^K6;^Neyh-iEwml@9fQZ`m!Se{~BBG-H`reXv@Gd;axq28k< z8a9c6r`ndzvIj&38!sQ|(hpfAcBH|nQ@V{CQ3lS2Db`QzwOHJ=F-)h>(RlPmcful3B! z)-1VGS+=j_!q~kSq&V-Q+~iY;Z80wqD5)k&_wmZ=0~EH^JSLgx zP`ZJ@{S|Z-Sj3@($Fg;zRMFHMc-X_5y)cvcv!jDIe#SY!?*pPD(ZQJVufwTo6f2>c zYt>I5J!K?zwyk)1{rw%sQczFLD@fioj4GYJOMid6mcrxfpi1uHKv6$;8BZ1*EUekd zh;eA=i1hwbnK%Z!{{VtJ(m!v! zX|S#fWKL^FyVkI7hRoR&#ZpozxQuCJa3kHWr!X8CM1^_DGc9U`4uw*Qn(8q@ksUm9 z>6d5(78DegcHngK%S!^~a5PfTTx?(`@&CitUB<=HbPEE<-F0veE`t-?-QC?Ck^sRa zxH}QJT=}p&R=S;3Q=3?G zq$RovIL&B}8S^1Uozx1|epDLnq;5jZ|tlRIf68;eEjVP{_9=yHE9 z{bp~94!m+%6!=L-ds5GYrngYWq>CJ3^f}CYpjNSvk)cYcp(`MV(YaiW`-oFoCTH^&UcNqj+829m1JjsAsd}jFG?x?y;DWfxU3Qh+MffX!7 z`SedFC-Qn!WUzrI+r;KQv_LafZ?4hAXGY9`6-52ZaW0U zRp9RVv=K>*aT-WBQ^iwY>32j=@zryDwFU+sp2@qdhIi2j!H3g@MOP2g zF#@78A!RmKjsY6#d#S|eK(F4esQ#Df1U74MisAJbBZKqX9oMCZcOiNAjti4~K{oC` zFAEt;^Z}p>V5{9- z=xxJ(A6ho*87IMKThv=td)C1?qSilS)k21icw+5CEi+8~uNDh`NF-@0+#93MKaw{K zxW=pf^whJrCu(wJWny5N8kQk4_noSj4l9n6BkhuQ5Hql@=jR5~v(T@+E!YW!;8k=4 zO|DT+EM8EBE#z!B+B@5S<~dMEPmceylS2x|jOQ@Bg6>1Y&%hz`S_{APITK9Vx3Z1w zny^};^2e*L*O#r%_q-usngkU+xfI^%n_zoSiAw3VNd;G9Yc)o^5E?h7FZ>p?2_He9 z%CjiJtmD<`Z{>TG7Cw$eT10W5mKcS>r1;GoG7dCz8rxQjit$-~AE#@2SobAkk?5WA z%kO>qD4PRnV7$XxaVq=S;qTFxaBLzLo=92U@L~-8%czN>!e$uMW(7>)JN_mZOlC|2 z`*_ZjwWR1!0Vuk2GxhecS0IOc^_}snAI4&(wpc6#Icnnz#@=$pNu>dv*ED_q% zp!R)at{{~R^l2&wVLE(a=1Y-rqZ*Syx*e403TGJcPl;)Q5s2p84S{|AG69|rRHRvt zYM#H5zqv*Aa8gIIEldzhgsVz5Q3Ogt6ovOhuGojL6>dkrb&L!yiiYpBi7I?YWu$o8 z4#9-x2?ZRi%Oe`i$T$#uw_A~&g(id9JZyiu2RXq*Fs?^Wj2$IJTu_^e`XvF97z%=m z6j4W(ubhratIZ7MrX?&7nj0;%HCh5sN-rnOy*nFY&st55>;1NA1l8!uarhNBGh}lf z20A&E;|%trLP`SWeg2Sf<4?#;N%Y?IeqA)A@Rb_&8ncxAsScQrIn5nZm_>~;c%HMo z5M*cyzRX4)AO;y`bCkm1-rAa^L^;L-#9d@OgWi~LonIMwz&7OgW!bBc{+lnBV+3w_ zUV8xpuC(Yu3Fw3v-_nQdln)h`Y2l>F;L*C>C;7rWztg?OIgg7AA?vD5n? zkG-S8bh++7xVa~#@1A97_^t6>&Q(@2>h2(y+b19tNzNMTEtAWA>yD_YH31Jv<#5=&h!yAOO`n{a znv&BP9$}Lk*>nbHM5}^g&4qa^JV61cbVDtpyI1)|LT9(U0H!4+S-I~X-3r;G81?Rv z1A4!{9`<4b?4{=2evV}&A2IzXBC5K<=p5Ib!X#zhh3CYO z>U`37TlKlosm|BOq^a7ZHW#ke;xm4)`@bB=%8zzr`WFoir_Y$TbT!W|n}a_%ybni} zyL@&AmW+M(NYrZW(Z-B{;w$0UvHHi1Ieszu%x`Za_=n@!J)7LKcdkXTel9D`I*TW! zzvtWUt`QpF2Q$Z>cKDw{B;V}Js4Wj~g(VUl?V>2Bw-APvp~Ow$Mnhaxwuy6j3Nrg9 zpbnA*u}pO<1p9b}UMEC;)N(nyP{dIS3!tD`Tkc?|#hP>bat{$4BH?e_)}I zrN&*QCQn3ILw6FDAeg}J<_M@qppp#|dm8=W@IwZx-+~csu!#4qCLTIT0%Zu+c`FY_ z8lxek!&6pPL)F6bgZ&g8X7wdzgd2CGHCtpyxGORkDT+2^J@0$CYt49^W^VK;=62#~ zy2t)_P3F3*OV8-^V8EIY9U^v|Hp~z?qU&!F1RCZtPzyAd*(GWj1P8{wQ22w^7%x;^ zk0*&>hbinHO}8iugbdcbFHp&tdzB8;+1xFv6(aI5;aLwQ2e$(XylcAsgC0ko6Hx}b z`z(U{dptyx9tg_vhy(P4UWhLUCEu%GCy-uz*leWjV#4qh1LjZupvj@WDH)&O^P1IDW*h)pjn4S9j5&X5jiN%U;HPpA3@xN zErL2+l~SfIr1jP-KE$)rBEU*uwt~MTXiVmqb~aPB+EZ-0 zQ_0H=G6eOzbxP#q?eD>eNyKhSC|7?u0|?RKE{+~Gq8v3i*Q(fOAQ1b7Bf1ibQ{cuw zpS;Peu$%`QyEdka}Q^?RW@9&O{}%`u*x&i)2_d8Gj<)`S4iY`jRDW2Jz>90d8_; zx?%Hj1>!hQd~^X5IR3Q5Lp>lvq_D6qY2wj!JP+;2`L+a#yE22%aTIZgRLHU#ku*Vs z_JlTxrJ-*KA<)7HkJ4+jL6v7+@YP#bF`KtAKToKr@>>qx%&>mq_DNNx?8r!#Uymd$ ztDoBz$E2dnode|`#wqCgoLZZnJY=*?kABk|JYv7gpgy!iHx@F)0(&I|fz$X_Y1WTE z>%1oqdNsZ5E^GF)AuoO&H$XXsU(P?=PEU0?fZcvakLGxA>nk&?{)e@7-Aw)WRHx37@&^$CbJp|gGgde? zfzZ?FDVlsiG$@qx@DM_<^;F^h05jZeKgT7O=_E=`cwSd`5-_*7z_l(K-4U_BU2j`p z_89IV|LVYtp`$W%oa)H!X^DWICxgXCP>b6if`}bHwAat?){>qcZf1k#-d)_^O(Kg1 zcTgvU@y$SuGRH+$Hjfjx)j6s;ds+W zGxTZ_H+rd$pX3C-i$=f&2&7CNUDJISH%N`U*2V0Hagj-g*OZ~>4qleqh6?nv~{`(YScACmC< zAy^$rwU=iLWs0UH&46NV zDCrVE6?aQ@H+Nv{2x-%y2|1W%A9orkM^=skqw-4%f-VeRI_&Lq&x4@(*}$y<1Qk6+ zZ`018eaYqGY*Oi;KWMxTa2NTzztTbZhC!UO-{Yc{jy=CQx*A`!^VK4 z>El;Z(?aZ#Wjv9)16fenRP^WffIch#S$Wc-L%YcghxDYE%i{h3ypr!tns%B`lbiN6 zM($0fDgsaXR4+2VPW*YSvWJG|%lSD}@tsH7huZuaoGu!u*u$Q&!mgI9*rUI9e-_X9 zc^p>TyvDvftuQMrXk`+CuyV*pGsP<&2vKLif&_mqIzGHS-A?53j{YP;G%HmB-T2iHQJ( zp!y>iVfM0NbC>r+%!Gm@pq5~o>FCQsw%A*S>;pki78A&}VnUd~dfl2R0Vs_G{ZxZH zhAh~9-GnzXlYvW7(H0|=Y4PF&>tikk+{@h{4h0@LV#pE9pN#8p-zC6>oi@;HkbyxQ zh`EaHK;ieSuI3!BD^l~VUvUn?Qh^3*7l;lXI z$Q&C1Nnv=DvgE=VoMy4RF7z=fASj<;efTTUDrXRnS($q zO{R^Iy2OO>8a8M%PkJsEW&&AuDEADp-9(x+2&Gsk{*KsT!H0EhW8w@f`YW-BR#*uO zQ7_9*h*^-Q859li!op|%sHe#2h0Rxzq5y>HdmXLr@o)s>0;s62eZg33rbb_DdZMid zo!}1|Fi=z?bA?_+1igO=>-*ie5cr`G!cneZUS+>JCB$t-6Wc)>5RzyOX7^R4lJPJS z9hr{6PTM+}8r$k>+7hrv!Y5ydP5t_8{u{b$QArcke#D;Krvq8;xcgbfj7jN*Sm7!t_4=Q%P_Hqu^Lru)39iZXc}w!Xh!mYr0$ z9Tt(j-D>~+TlKn9{nWnuhe*4^;2ZdURh40RY;|T|H3)#!MDd=(f+l0LX#R24K4JuO z8wm1>496NN3_7Y1QW6xf!%!QMdQYk#@2)6|IV2gO7sIq$4$CCxk-`V-4W9RBQij6; zBa7}peF!L6>{X)$IoveYmK}?zCWeIFvg3Hh?uz(TwPhok+V-30Whcg4#mFQkxd+;O zq5<4GoyGPf-J&VA8BXL*g{gXnamdLaMX2An3DTJQH5sNLWi^!6zw1%bexA)DP*~;`PduATY6L15Ygm_z__Tu8n>_`X)_rp-J(`s&xk$bfu7|*M?b{F|kgaoS`g;ztHZbo^5L={Vh z4F#!i0&*tcsX3+}ggEf)vKLu)lHe^)=$Y@pDOA$jYq|}p#SVShXep9*(=XCl`Ery}# z2DLQ{1Z^uYA&ENt!hdXz95qHB@)t%yKoK?x00UR8li1jQA?D^J&>0`2>4j|W|Fp%0 z6eZ{M!y#tAz1=4Z3fBt*zt|qUjxHy)0v5$nU#TsnT4K_<^D!nEfj)(8YpK+C`JVmG4 z_}r#W(=J22>I~VqC_;tq0TpAb`F_4G*u^g<%xP|zm{3(L6(a#&L}&|d1sU`L3~R*nSF zEdrjw2fPH)#IWB@UD*TtLdhWW1e}1m17X*SyImz{j4hSz zGQIwyt5Anc6})OSx(mpu!TW-X+bVQv$J7`a*3?l?Olc9f)_hmQq`bcG-L5=1779|E zkjiL*b>y-T84%4KH%Nq31ENeKh7Us0BQzp-e@`uFu=vS1xf*MpD2#Z8v<97uDpW6N zxA3pnMC?%+w$NO;c$%BUQ#%)ubzumJuT>2r`$t!E$&O5b%|u-X9zIgKQ2nXW7$8>4o+Z+Xh0)Y)nHK@oVEh(Nk?wrD?I86&xgSvc6)E6MTslfquML!$JBSrkFAX3Vl9qdXxbT@*IveFQ8@^|tvv+)s@f96ur^YQUB#>A=JV%m+H zYHH{bsT>_xaYW!{8+qdt$r>yK(+$we z)M5WO4-~EgvpgDF`iZ;Wnmdc++hyi`TB*s4spJ3h zPzmB%TTR`fv}wq}!W*XC<@ET9!oEy>o%sLVFY+?Xcpfaiz5}K(3uW3Fm*!scy}dop zN=ydOLi(9ji_pr5kjCLed11up>ZuVVI69QG^Te05nY*LKu8zxtAW}K zHqzpK|KqRN(lG)s?)$#Z_k_p{Hvw@=z<~R2PYWk7T%x=F<+C#Lo_@5T;>VIXdR*f9 zh2p{gf{y+#7Y97ct6{Xoh;;e2maHw>Lj8Z})_)OgfkGkxo<$Z=#_*r?HHUxnwJ&ca zpZ~{|<|5oe0&Lq0gobA6zp$=_*!5t@Tz z78`pvOlPLC<=-by{->iy8CxC(P6E)rlxf#Ih?K$ot6#~3{!!gTQ!s0b`UiK(=f(eZ zSt>9;;UVwtC6o>AWdFNY{vW{)0TZ|-6i|g`|2p?ScR&Ii2!i}#mPYFTa4;ksxLp{5 zYT$oe&;R!M;DHaFQBTvu`VR-i{xSH%MM~5Eb&vmH*=B-)4?QlGXO8#}2Pr6l-jK-E z_EnsR47apD)WqWUH1?#EEYz(rTbWl#>kK`YT#P~#O z95lh;6t$in9eca_|I<*BPn6S}Gc!H!S%Sn9wVE9x^;=?>x8l4&_&Etk)eVKyHVlpu*m0AU6+Gf1$pX8`rj2C*F_@H< zkiP7$xdH2Fau6oR2ajhtVff!2gb)A$-7#2nsP&<7ZB^jC86E@eqH=k8IRvJt51W^k zmZq@qLzs3KpbZ179puxCo16P~Mb0*k_RSx|?qfG?PEH!|X!3<+Wn=&ZQ%O!%_D?(U zPT<{TciZ6YZSw8ye1Ay4(d^Xy{g>u;pp<%TS&@)1PgKxt)>T5jZmG%s{5QS->T@<6 zOcPsmLtiTFS+T)c^&|qBJ0c8Ta(#XMkzb`FXKn!YLt|-a$sT>e1Iu#6aDQ@ZOGQ~#cgaPin}X?_vf#LtMkx)j0+loD1l z`n~lc+H{9tZ?Y|PaItK)V|~1!zaJy1{GX`5g@t--*LfcPIRqd!c|p*#e76M#^dqmF zHHo;{i^q0$j*i8`ftGg?n%$hAvahaO2?$E>jpj(d6bv$$1@(q=Z~%KEHrAmENsu;j z)kLNFXU2pi0wmECc08@S`$m#7zGFF-CkhyvN~5<4|E`2!un9XrJ)DmI4W1XruBu7? z{*ALixewxi9xdvSqOhQ~wXvxxa7Ky@HKh&G$^qr#9?(i@XlMbat5+%P;PA3nnTNBa zgmRh$W}G!kQ1!z9PQWg>JYhX}Y$aw*(4l!tKklt6Wm4bF!*NsllGu~nsF8T=fD3{{ zV)Mf{(%_z}tJ+q+4Em7>&4oLzMpUm>-5q9Fg!c9GhGd-7rZ5|wW4MCiLOtpxSZfE$ zTi5tJGD>1Yv%KCVn(B83C_#yU1Ar=$V=T9G+0aqeT*^ie$jDw;uBDRRk7EKYik~H< zMZ3A)j=n|~V1%@eed?kzUW^T`~Z)*}-Ik2L!zQA4O`t;?~sFn8SSB<+U` zkbmG~2=6C19jKW}*c7B^dpJTeZWzzx{_%5Da+FtAwb=;|2_ z>~Y)0*NgtVX#&># ztz0{^)0K>22(!H5Tn-7a_qi{Q6&tF1RY^$;AVp|b=%^YRX8ri#11K(UK$>i^Dgi%9 zYvv<_N{G?8N&^9bd4xx_;83^O;duZI%uSr|-^Zd%r_~NG4rXp)VX{|cfh&oeZ`a)H z65xvvRlGerJ11KwK>x!tB5C-vZ3R$>Twqg_f)BJDHs;2ixAVz)^)W^uNDWOeQEe6nMO~M}7UDfeX8IRj^38yt9PBTxTwPJv1nLCmUbG`(GDevEfX_1kU zxj`FcNr#7r#o1pYsijH1s5Pc%rk*uu1oaFJNo=pC$i<^QlxfZG?(UqLy_aiD4cwLo z6w>8ND=Gl=Bflv=L1^CM3`1T~aRBp(*=k?0o0=2+3ng4^nO$H%&i^%kMoUw3z*&G1 zHx}x3BBE_kC*{cJ3W2Rk#3A&AhKouPkx9*2hE{B`MLb3 zs;yA`s0$`y^+&U$=1xy2xjohbpLgm%&d#ctl)YM_XOJxT?&c?$N{f6f2NHK)+|J;P zlm}|5bf>B6#1EHeV8h7Ri><7NAZ+t=3|%|laba2?_LG!Z^crjmj(E2Npj~7}y$uq_!v)`N=K!juA$%fR@MV(! z!h+Mb%>h`8jj8JT`wZYLF(9zndW$RV50~Edi_S5Zxo9d$lu5*w??S$9;UpgGuFk*i zQVqwQY772}KvnN)QFy==vR{{sGc2+;1zT;jeU8Fs4GmCj1Msk2qK@lPMtbD1X1(Yh zrHTP7r0}<7s5Ue~sZAXkvt$w&9j8dppfR9jZ0|8Q`8-`GDSfmB6g!Ms`|4kKN;UxU zp4)j+)qXgh5sacK;)}R*#1aM}hQ)CHk8*a|4PE#hkHz2-h*I%$z_k$2Ofn)drxDT<+)s-T&lrrC%b&PbE}0`fC=tUR0D+}(w*hzkn~Rk)=7 z9OIV*m=Rd(hnQ#l&f~Z{4nF=Cu((%Ig40-0^*RZtKd(1a?ud$J>rcSe(hflA1jY%N zXc?441AYA{>EjIRY)tTLSPg$aDKJ(gDGL!mdgrxRKs@h+ks*mvB98AYo5!UPiwoq3 zKsSDwD4arnf|i$W3(Kk>I0E@bJ>U9w-B>Tkf1|!uED%t|{gL3Wa z(5#?(P_&oE5b(_kB*>SFv9q-`y}AS}3m0l5zz8xU%24CiaRbVjU;$92P5-uh_|7kj z3ATj$U3I);$OfdKHm@72bHBe&M?ZqOi7bYr`o!jzoVlUg$i=%Us1o*N1QgNmPzjSh zDE^Vdp=K^kWifo-xbREEgWWp-a>%ZflsY_6Klbm>7H55th2b6k-bd<;9!6;ntXP_o zCLq8J_fn@pT%S3T_;`~3!;<~y4o3RoZ-#SSU-DBJB-Of?Am zI!V)>+iA1&>swK+xzx0`HzRH6P!iL2pje{ss7}JL=#3}h&tDUFGj)IPNj<9H1Xw26 ziw=#IIi3I7^&fOaPmSzNQH{f)PMXR24{YSTU8gp0dcD88ce<`AF0ngc7ExOzYinfP zOpskFA}IXXHV?%+i8lXgb6@e>|M|^%c5e3X)2lz}%aFU|Kq1CP!G3}7LpD&JwV1?p zSwQRP<1==-cVtw45R*6DdG37CHng>5|G}vPF{X7cAtwyR_DF$#asqrg6B5$(*l3yD zD5JOfr>PsKm~rX=H#BVetNv*ozTLBo?+uh6N>8CaL{pop(wSkI!zgqw2e5^;azatS zSzJB<6?}G61L5nEZ?^yya*VZWoSex1pI<-_N#MKnh0yjJLzzH8@xvyzo)LgUUr1t{ zR8=t|{h^$lH>@4g)52_TxsRk&WU8S1NlqC)TZpY!5_r4mZS?j2VMP5=J7@Vx+VBWTFv1=f2? z-15%rlxmgFdoes_S}ho+K5?cMG*$wU@K`Uq_+6ei6Fk^^r@a;7rg0Rha-s^I|Kug} zjY_mtUMQQ$si6R68b_9@rOgrpK+T{E5Y=Ixco^ZfSP@-Ph-6|8pgV*K-BK(v5RT!h zn+`ByHjQ%=uJ6wFC%aP(rcuVu}?G8~N1=4dpOe7LfqZVb;^7;rl4%ooMI`GSQVyGs`(UpD1??yi$j zr%w8VBBj}m919CgU~kbGi)9eQ-76UehWZv+q)Xt3XYODUOri_e@hgMcvYeINrOQr> z+WEk~#cryCYA%RiB+5%Og<6Hl*^mH?I;LSk*ek27qr;hBH6hCjn>;#ORkp6-wi-R6 zfCq46U3@p=9L6uJOMn`h(+_R@{=Agw%0||M=?fT#-}w>TKrS=~xRAv^m49#^6sCRX z;LH}b`03AKHSzZ;Wy$9;Xe!Rn2gQ_S)GQJ8zP#GZ0nHkw2?pv)jGsS^{jJT`) zG(2Q3A?#9vS7*=dvXt^zfm0EInu?0e)R)VAA&9Sq|<3U)e;?q9wXtoBmqf9 z<25BO)4}%x(06`yYmUR+!S`gXQ^Tf)ZOLto1*}Y}AI09D?^iu8OnuA^tZ}RL6aD#A z78CCYI*G|BblKV+*H*1_(%*tvr(P!|8I8YIIX;k{Xjfi#Jp4Y|4I~MT)#-9NyPiR> zFwT#5{Ir`Q$GzUn9kEhqkB>d{8TM8ztDSc%*J{IzN z@i(gjnA4WWHda%OfBZ@}M=O=)a>82O@lj5fh8%A$Wf7^^KL63!s7>)}KHwS2 zkY%W}nN9BtN1K*4r@|4L6E@sYD47#p_8lq`F+jO{?Hm4U`(^3uJPH)|SLoQ4$CKIf!I`Zfs3>t17c25D)#PvBD7X z@Y9PLSN*01c*5fo2Heh5ow=7G;lDFqcn`3gCmh2gcK8~|)mH?uB51D1WW~l;wZXg; zEqjBnvHF0-<pb5y)7$V8tnnM&qe93~c3b31QDKDk=gIa1l*RX{q&M z;?Zdly!qq-Hj}c3^yv4pZ^-gP2vI|D8*OT_P_D>!F#3H}ZLnaPhyaFwO#o(A^HBqV zLG5xg*NsQFGPyQv5C;hH%Z~auQ7&pM-C0V-#9VyJdkIZv%;xU%hK8syad|3hmra{r zPbxnkQ_pV--DFI4+*LXQB?psR(dE;)Oo$QtTHssLwt#^mUO`3ri|Lw^IGpcc z5IP=e%h|nmw(pnI5n59Htrs<%$(`tKJV^*Dn_qPuW0S`INj@}YRAsteHakrulNrq> z5)-6hqz&jKgXnqmAoY-1_N%=Q3IxKot4_giDAg~gC3j{%9V~4a7y^d*$&u?e(dKMpr zkHNI&R5+6G_~tg^YS2#jA9CwclPo_2EB8aBQHVbPb9V9WH`A=i$6l!$nCM^8z2eYy z$rimLiJ=|Ff3MSGkP{#kAevuy(EZbic}s~Ms2rB<@(!o-#L@ZoF=Urb>~ly{b$oqY zXXBw}{IBLsp+32xZ3ns)z|Yk@C+tI`3tR%I^P~8Hx-|y{m~k{ZR2=02p%&0!N@x(3PIIQ-T@PCm=6a$&AYqNg-cTm|=|~bakZSH3Mg^WuszW(zfMS z_(r5i_!TG%AytB80=#d%xE)+=z$R+s9k1tI;%FiNnNbmP%D{y4-p8^+L5``!4I@%- zeLOHA4eVtl#))Bj>laAO`k7Lwr8>+fK8|D{^wG?$1yWEmWgI(~uRmFv3dIDL%^h*- z;|Wt^AKdeV5NHuw{IC{h0p`^o_##q-%7=hHJOTK8mJYEfPIn?^aOW1%d_@(vgXt*cYr)km1qKS0gc-3wv+j38JXp9;=)xzaP5D_8t zM;IyUq;OTYRoi0Q8$Gvm2#Z_=ayZ?2v5{ zBmJVyGN-WUHiko+oYp+@8oS!SOB~Dog&`N6+e9bWYj7PKP1-o-^IpgSIeq5+RP+gVHZCi z_fo*0nMjBlA#B9YS)4eS>qnml^ICqGL4P(tEJEaS6>N~T4{`Dc)eNid{&+{rSyvU?;yTw4=ldU!b)@^VI$**2{inx@~*xwFcuUh@EecU zc#}+3|qQ%QiZz#}icrGrKTu`uSK z=n>)Zv4$Dx*ezf*l8m!?xTTAruFDug5ko+-rJMp|uqm*gXFNG2uTTvV={h5|GU-C( zq@hGd{S=q34Lz~gD0&f}fZ15Gt+~?#vWswo3KcHX9yeV!@pARzUNuquMyWNIlZjnH zEjsGOQ(*5er(I5fCA4N;xK-{sr{DH8IqNEf%}A|RjHl^BxLKXV+;u4X6f<~%nV$$v zQEItV#`NKQ!;A1&z;Qf&Dz(V#b<1i;WSh4s< zEoF*jUzu9$E~0m)McGpz$>)M;b<6qMTt~wFkZXc62dx>0#cqY55V#Q{FG(X4n1oo# z^V`fG(dYwQ#|6&Rb$SguKSO7{$x|)GQo;b_uHVgI(@&f2fL9>%ZhLvezu zoI`dZ3n}AZg22kKfDiaAbgaW~=)!&zpZ#44j|3kC-R3GiHom?H`{s!*7B*M8ra5|d zN-PvBS9_lm`JUZ13%oARd2Bo!qw;LI4*iPbeNRQJ4nBPiqnu3V^XO9Tb6CLM>PR)# z*IyNE-*LyswWw=waNk>N=uEM7A`5BRI}f{PIoj#kdfLu$YJ2;JPa8`J{?z3gn#P2L zK&U;KRybN}m*Zu3cS=ynuctXP=+wRG%x~Z9{Mzv%>uk{FHtZjA3vS9m0jez+ONfRo}6pS%V$B88!av#PfOVwE3H)-zL@si-+D!NZTUSs_GYa#VCT0-iFwE8pT@YoFieCe<_XwoJ>YmWH*Tx6zaKPo!&Nh)%(0tD-zUb6&X0+cj$Q zh+QZxw{!_y9huT14agD3m z09Nd50%BP(WoWvw@`6T12@1P7Q!a9?%E3}GS3FW6^#_#N$Nf+j9E|y140V2}Jh=@` zTya$w2s1kaO*`A~g;G^-B||bNw!YfrQ?11T`*}y4gv`_BfwtRzc0*)ty1|&=)Qt(L z$N2{B?d))J^qF|Mgt@aTcNhMP6^2DsG?XWoI#j$zAYKnSqM2A^6 zSUzFz2WdI9&7tRh=7COqrTP_RZ+9iug!DtC) zkmL-Th~{|2C~0zZ4tBw7;G|J+7++-mPjPW`zuyTGhQpEzHR>p3wop(XxKaOf3P>K| zNOv^$Gd)Ds5->`MwqoSle6G)5Ih~C0e|kC4HoQrRVsX(^DAjZzZdpv3`sZ62D8h#6 zZ~fU;>E(y8f331~$Dids$D5Ox{I&j(xeM>hh=uZ{WTL~d{(MMz2ONi`&a)qU$fsNU zzaRUu#NEet;6GigTAoH>lM%cnOR-3^^sKlax(lqDf>1O0kI!b!*}gS^#comP$4cGw(s* zLZC1o2F1C#n6eTxtt$q$f?3kCu1X9o7C&;ALIWjd3G<7Km?NpW-CA8RVHlCY=di)U z!<>Ex--z8DHNy+fluZ?C-L(R*(iew+^D4f%-G%`HB-F`_X#=${s)=$^5oQU*<9q}j-B%YbU_m5Mgkk!HX z3`N46_2fq(UW5}(-APFkt`cSq;zWwJ;y5#GjYZt_x1*kzHI|$o+z?F#6y2)f@L z-zZjwx8fh%N(u4d=`VhMhFSD%r#p&Sy6pGv;eun7WTudUJ#q46IUL4H-wsSOjDYvM z4(~%_w>SEzAnm8fQB%Sd#sQ@$MZTJ8biSOAR@+`-HAMk3vH88#{IKy5m(6Lp+AA6L z@^Df=(k#7pC-+P)QU3V^{$la{GcB3b%+{quk>_*{VQVDM0!dC~R|=#h1w`jB-)=Bdr` zwd*VL+AaL0_IjT8+22y-*Ch{s&!-ga_w$Z=jB&PRB+`l3YaQLC1z(V!u(=`h<V*m^}_z6qi~kX!gwTOA&UQM4oumE)1IfpC>zA?1Xy< zY|EC)k#UgVH00X&hmsw}{vZJm7aXpF>^@dB&9J@;$9_STPP9$%R0wF1!V5&)3Zje> zl|&J-Okmgi1Hl&&Q7ae2eRYhg5Q5szCrgg~(H0{>txeHNn!JHv4mx$*d-H}=V7kUA z9qYuId78$SSXnBd%;k0@`CGs@|8Gjo4ma|N-s*U^bYzL~h}T7^;S_sVC5&)nR}^qb z!);46BMp96z@q1GRDhl(6<}*9RjC%?jIg0~OO~g`4`*w9^#qF1?U)y6DbR8D z?BJ?PS*TgKIhPfcu$JwA8!lDHHT`9ODdM8)WcE!K#_Wt`V*Ytxh0XNpbVdLsJOW>f zbyQxKX|6FR)ltLikJ_dUg^?+1mcG3Zms`TPc{K5iIg+Z=n}~&U9_UrQT#KY%Y|}_C zfixqthLktt;OC+Q-06Ws=8br8n~5eT_3C^U0gXKRPgsWJfT|CkxdLT}HZlxM9kxth@Tz?sYXZVFyCR`$v<+DKz5H#9JE`uJtNAR?*w zto~s-&pY}4E6&}tN?!erzHc%gju%$`kSLdR=ibDH(3@fySzXSYXjHvh^g!0B|L?hS z>l`6L4>I=4bSgAbp-;`QQFzK3THjSS){(KBw0=5fer_DY+xm>`=-B>cNtO9x<^6Zc zS+#ObDYL)Ir>U#~EQ_NJG;u`loPqy2vE582|M^5YZYn0GUZJhtHfb`*X?mORU zJapIH9kTy@Uv-Y#HR9qii?PwmO?g%8PLaM~i>$>L_}(2JHUBU6-YTH3AWG9k6WrZBxO*U2aCdii zg1ZC>?ykW(xVyW%y9S5g4tJ-gXYR~&zu%Yp%C587Wvf=z`qx+F*28)|ge~{A=NZD? z!!kg8u+9Mig!qmC1PUQM>GH`$RMUUjAN@OW3Ev5hDT5AqX`yxxd5e1I&)L^3s|8UB=l`;32v-2U98-~M+`oXC~AZb2mU z?tHD-%fmH4l@?gj7()Cdh4_D-vCq3yWK~+~=8BLtcHOY{sK7f&*rs!afA^tzUemG~fhbB%bR51Ie(bl^kRUgb;IRhBdsgRbM@1~Q7BR#;;+``*g2Ha79c zUyrS~9ZYqdx34!w%EeQ!BZZVi?wgv$UmHRNUqzRBsm^e;0A?P`361s&AJnZ5i&4b> zRj;^PQ49(SQ!FopAq&eMqd%D^h7qY{khx+Z%a2C~Bg-5`P>-e(QGBk{@&RfVi=zvH zQD!Jk%q)`~x($UXcFYmWpE?spNx1EjPgkFcdTmN1s3p16f9?*ZOA@7`NPEVoqJ<%_ zm~9)pH;2Ri&MP>QLTE5pKzh>NC={s~t7K!;CCU8e9qQmQ#ljjE95*j<#MCt8oOs808p-_aiC@iEg8FPDTqC z7RSQom?uU@O3MUm=+(e2tzjq>;_RTy{z(bEHV z3R2d41;-MD{7WF8W=KiFZ;??L_PbGoFcnh4#tXgPAhhE+Ad_#{^a&*S?K3Nc zHa_^~Q4N#q*)pgvG@btdlYWwU<3gy3pNkP!*6eTZ6M_CvG@<7mJ;RJVw%AggmwHzC zrQ-R>p1SWba zVQ*MT1)^!QSd8!Wybxe7RGzioeu^Gc+VsEO{56qhU2$8P%%2o{ab2o3f>HHua@a7t3qj#5GC_8aapF(v`USIE`Y(Gp_J> z+Tn`cqAvQ;yY0T1Fup$}eXp(jSC9F=cWbAI{2l(d1NCdP?#}D^%VoLP$P(kE@b7 zUL)SYP$=hc{FPDv(6|q+yeRDzh4IC-U7rSXTC0jYXn_BxK|vRRWCmqiJMJ#XGzp0@ z`R@*x-6?Ms+5)K{tbEG5?iZqda*%54HuFxGVodP=WC6gz(~^Z~qmmN!($TRB=r5Zfr1wEw5^XSM#`NjMfA-hP5< zGTUK)MTRz6bre38eoEGDhVT(ZJ{FG3E5Wt+0q#1CmBw-wP&n994SZW&fWm3GKoC(d zQI}D*&_4K9omu>^X*fwbq8~)BvABGSCESZr<2{X9S*j^2xha?)3mu64GVg z5iMm*>X{ZwHus{GicE>zP!bpWa}mGO*&RuO2oMJ;S=LwE z**@59Ab&OaIMSvyfL$hbKdJ#7+Edz#3VV;fp3jMLa_@(4G2F4}ExfzakRZmm1u4B( z$7lu=DZ>0G2sK0^_HFXRoVSFW533l`z)8M6mAeuWqMGmEO`t*sutBQYVHF2-u!EsC z42afG2T^nSL9#ba4+G0|APSlCSl6KRyayDk8UV-cUsKxZ{l{<6a@?zI*aw_z`ox&2>CtN{d% zma(5dX+iV^DKR$N%u+i_;mh$~s0?6P^kCoyhU0?!e_n$J3BSV`KL{dVK)vp2HUAQ= z4uvZ}+lwb0X+1z=19oYCnJ98)F9f@IrLVL3N33l;DNol(*Mb zT*`d&ylz993R)PWQE*seYyQ~mT*L&&q}Q&p<wFG149==aWhdbCB^O*({>ezObgB3A)b13~Mk|r)UthwniLU3n zHakT0SQEs@s%2_!C$k`N!Y?L@<#vy;km5Ws7@)TE`aDdcQ>m&_$1+`qy`dcMKcHgiwU#>~B)1K> z`A}l-7wOjN>y?FXg&I1uN+%lSU4(oN-4T0KykVv4RoUG-%@$#dhr@kHo^xI8MOI=A zxR%hDIri^E0WVQ_EpD!MIYN#o81fxH*2jVn9c?EctQBL%u5i3R zxDWSSI61?i_BE!4^&y{+^gKd3e-ed~BBSA}Ej&iQxMPg7hNwm8Q(qsKDDn_q)+^d` z)6o(^sC+8=peQ8E+1O9Xp@XmQM4|<;At-g3<3wMshYNPfN$JyrJfuo2Qt{1t(Z+?M z`!RGtfYQ-bqIN}jf2jBn+S@+Rj6RPMJs-~WI)e7Mp(4R8%_vTjZ@xuwrX$>BBtO}g zMhRR3bYp*ga)Wy|j&V8I2ypILnn49RhS0}Qgx^*MuxD&=Lt|pSf#v*;C z!trW+dETUhX@9Ez02vv9Y1W?_(+1?^=usJDjMAZ^e`U;s+#2C+@4_FzBxOl7#L&hb zHj>e78_0`ztd?6k@0*3vh=%^iqpg*I|4u2dzAy0!(}b^-JWPrdOj}N1oEhu4R0QX5 z?e3tMVjzUBLQ{&o)t8`1VyRk#NAX*DV^NnP%`rwW)hd}(>nC*D$uN1W6wyRT7GqP; z&L5h<0FW3vThj^jWcsiDVF7Z=FXP+8qAg_k%^Ukn#xj&aTsW(>pE^As$WrxV*Vn*t zp~VYSQ9Q6l)yw*VHavgSv&11DN`FH!W3HbdEJ1==+y9AESSBCn3Kl&pmdA^2UBTp{ z**P{;V$Tw@-As}*G>w)SkN4}y!V$PYzVfbviV$rQa}f`BF96&(%lm{fAh#f~bY16^ z>R(9BjQqC?PwSrQ5bC?} z8^%q~iN)@mr^L+loE1`iG2vyhhn2c9J}!rWF{T>R52-Rin( z)K}28IpO{kYVu`3GIi}|EWN%D{v!4$cA*)V<}{YB`xy>j`jLVs%$N|>89T<<$W zlZKyB+c9|zKZk|jha&ke8fdm%H-nE^CBhV=XGHQL`P4JP7Tq+sP9r_Fg9AG9Sd?i( znM^+8fzer5x(ufu6K%7Om%tnTcexSw*U}a%3Q{gc!=5A}GDhj$lu> zbs$)TId&s4ed1EYuX+R%o{h+69}Zm5#nh2dMZFZh=3BKzO6f|JsmX1`#*LknZwI-< zbB5}M?wD|&*Ut%~X~x?hvHM5xcy&TpkI=u~gyQ#;Oicne0vK6@HvE*+`;oC59FelV zfj1-`V-}DjRFQiK$mlVx&k5v2{11Gi{PT(H-KQQ?<-RBdv5kgdLs6JekoDL+YAA>g zHm_UpvJB4c4Mx!kDOYmKA;*pOa*&c@nFtl(JmHzlVOhZUV>X4nhcxzkZ;E)AtWsZx ze+=cXo{avldM4QEIfNMZw z_T6znBjn4hr@-)Vx}l^@G-{ZQHEF*>rvsNI&#l4NAmcrs*b1(Q+fGLeAM^u2aF1o%W`YH&Z%OjQAWSk*~S<{Q1DtVQuu?HU=*O72^H zb!9G@(I;qHQZC)R*o&#zzD|Vp~*z!KHFv<-Kv+w$7Knmzqys(mCM!+mJ$ipWHHiy>rv#rOsMTB&f&=y&I1p3DYw z@V)*-_KhQ!i|Ie-_Xf2(I|02auV+csT2P6rmTa&XDVbh#vHg>NZYfKXj{GFajbYdz z440(Y(={gVFH;G}g|UpzD^qhCZ_Pg3fTVjDLBM4)7~Qh1V+i>iJmJb!;thEwPcE26Wu5qwxku^3dJ<4P@ zg!mQjZXy}2b_?4^z}4GJZ8@U?TF0Zi5u{By2vr>jmkd%gbx+1jDFs`K=d<&ON{n7* z#>Xs3_5&Hxrm~uvAN81Euu&7k1$<)8o@7q-qU~>v357lo*~?YIh_FA3n`izzSmJX_ z)0t(^ERP?X&ja#--XTVsI3jySg}y3N@cu^hBiPbWO9IFrs1d1UY!q^2XM@owD2*hD z_eI5zUfKRw6bWZ}5eyOT&t95Q62ZiF${V&-P{Uw%biSpVP1OSrmcqnViPcv}Kn$?L zDN41XWe1yBf{7FHmWgcqPNj#7pb%N!vc;gZpw{vI)}M?N(wr|!gj)-{2_K3K8$)`L z+722MAp~UVxIcT}_3_eWC@Cu&g^RdZr?EyqYmH+@o;hL=)iYLt`$HS2ERn^fKpTKH+BIWa&j3T_)xbRfEc;h<0pc#hmk@d%x z8}yuCid+Za`slX?HVG(sY{hP zpNBDkb8UC3=W?eKbtP^AXtwGOC##-s4~C;c>AhZiEcHL;`{$xR?sEGoeb(@^-@EjA zKhjebg+ZB!dXe){DmrB3aY%#t&f2-4OxM7I>M~)H4aZMM^7LG?EE&YHx;`q?r5j~_ zJ{!Nj`OXvzd_;Y`=TyBtn0M*jZG91Vxs&&~*u4?Be`>q%3FQ}k0ZtxX;YRUXKS4Eq z!esZF$Fbm?qqC?nYHlxYNXUw&AGBqe743*@85pv@ZoHe+r!XMRC7726t>pQnZ4>C05ErptqqK3gYHt zI4EJTbled{N*KnjvLF(QH)e!XB@Y{fl2+tMew4}s*JwyRGBp8bhq)PJQKG%KprCOB zNB^8WkiLx)y%PRA-696ZJLZqki_nzK*T@l6h#vtDEhIsfMr2Uj0>@WBHEc#= zu$e9g=PRb!7~KxyGd0|yCp3e;B=mxjt`AGtxI@1^Z;GMJ6pV(*u%X5`3$Hz1?LE~8 z0t&H1PJ9UTmAKonp$kx9I~9Zy1)>(mq~*EYsIXwsZ#g4~J6%o=9)!MMji=}mSL4*r zL8cY+ia`{0(2(p?X>A72<1I7Rwn7t{s$#z(!0qp_Y)0-9;=*m3#%)2;tGp?8J?R>s zHEe}DAW~AK%PVv{_lddqeGcTYKDYJ zL0YvY$8%GifA>=lv+-`cJ3g{~v%BGqZISdDBx|nDlvJwg$6Ft{=z$Y-Sa9u3^T})h z3j=-pDrFNki1gH?U95n|#+cyijWK-nUP49nj8Dfi^OkkqA$LF!YXQ?jCQ=9u%kZzs zzB<8`_Aj$suoKf5*&lz+=PmrL*@?eKnB+!~QWN!e&5J9{D?n+9v>B$@@R~fV48YYg}m8+mf z9kia*%RK)?3?s-#`)BM7r4PVEa}U&h$026!Zm5{y<$|Fx3_J;v%rhRGCE@R!zm7+Myo{_F=jR_^(e2 zv6aQ>4Urv82(RPX4$8wL*GCx#5==_bo({lNH006o44E7I461kxA@H2a`(#pM)iv;_ zQ!HyvVN~%NrIRdu9cPegR*bF_jW_(sFRiWB@6YG*m4{|W4?~z7YBaI?L!MiwLpa#4 zLC8+PzPsG5>O!|oMWDDT&a+lSjD&oe4{AD|=Z1djo-&f?e9eAW2U%H-H#jj)`vtDi z3Onw4)kILf*#WiK!1klFgV&Y*9AQfddm?$p^>ndNJ!aPMk9K1_ueOt6hnH)yz^5AR z(X(>}c|d_?SZKpUWlKYgb8wZF1DjBUd#mCd*dS%8IK){&h@e|{SmdUzF4-BJXOya; zuDS8p%HpD2nED)a8X*sR3a@ewerjc1ps^8jU6IikXLPr+KKxkPBB8DCO}?WSJ8Yrjhjc-Q# zA|o#fS1cqlQ%ylGNI!zH2mBSijBaIkzrI4Z&hFx8@abZu)A0AJ*uL5{1`b7SYJW_x zZw###GqaVtymuEX`KrgU`QD~btPpEp0jgz-<5^ABwr!kLrrrqrnkZQstqh-~ipP(Z zrFLv(b>KE6>73N10k^-kpwT?0a-*%Qz5h?rAlD5N65HYU<#c7%o?m!c2(P?ul#1>H z(?B?8*&jxY`C{`2MqS>SS|_)lG&s)Mtg~<3W5;o(-t<$pxFGNfR0G7kBjD zJHBYf-&U6-qN3g>Otia_MdEStx&fh8w2Rez)0otwi@Rxbx~uN1CA<4ffu;!J>r#XH=lof31UFetaQq?$wx6cfqQTyt|O&*JCVF-0Lr&;X)(v6Ci zIG@iAtjt~KsjtSAotz{{!$bhW1e&$=i?j^ZJ)X{&N)0zQwI}LS8to=6eF85J#|wrL z&-kW4gXo02It!;*yI7~P{xT}}WK}CQsl+vMW3DTOnDXNWL<))-mc{>x&g_DN9woneFj-1=9J~+atpe>$$S6jCASY(?+u?ZLs8PH=#xx^EzSQk@$me zvS9&SQ`LlYszpD)s`)gOG-zkSaaxWj6NT1Wm>!O^Lf{72Mu8P);yQoc{o2s*6 zR-`(k7o*t@WY(Pk&HDfT=s({vtpmjk`Qm0_&kwx?3T)=|;WXsO)lAu%d=8!UJ=cvO zIM*tyD86@c{;xl+-XrbK0+LNWf4K6^H95 z*hoUv095sBjkbT7H0t%J_5Y97{g3YGmJ1RDW{RSzU2q;rz_P)q+6qfEX7>aw+yC8J zYe)ey&`pH*Yd({YTdgjqo_A-Pow{~k)_iPDA^%HhSs@bStBjwEj5KX_x_ZQ|(rVs} z|JU&AmW%rLuU+o!a(fRBaC_uc#3lwG}{@*>eV1Z)Av|%Z668o?10vInZAz*Bt7gE&{{&$am4*>73 z9{4?p`)_UfFD1^Q|Fu?1`mpPNJMOVWfOlgB@|q|9ceQH720m>4|D#I(-nciCXe9r? zd*Yu(V22K9I9;sJsIgio1s0Za-7PC0<@iqewMN;D`|~xIv*k{aUq}`D(VI^qKr+j# zOQv+{f7Wk8%;4?$?sO4Xo88HMrrlfHzCAsKuXyx7Y+oA$#Ph-GK|@3`z4G#6#abe@ zj+5p335WHMkwWPdx0CrA^B6nZwF_GKFMxf*+)UIN{Levefd>KL19WQ~7fu$e5s+Rh zb=xW}kGVvE@u_yc+Q=cWk=yZPSvOCJ(l;@kl3?~9U6q9%U^-@n&8RgCG6s$@zq%bw zXPZ~89O!g)CQbT^o{v(FS+(H4++WO?vDI^|MJCIqf(*utvz^BO<7;sP&hda3SOa3R zhV>GkW7@cy30y<={F0Kx^abnWg+)fa)<&n|=#4fPj#0~y>`aRPXv?#5nA8I8c6Sg_ zts7Ilt<&L{OmmXi*6ri@8iPi&39xqYd%3P@=F%n8Dff`!@noVd$HQo-FP^o$PDrk` zvkoiv4mYisaWS1{meN8nY<((CuPGD#>2not3OGFC9nTM!8=baa(fii=zcfYDaCVpL zG+12Ks@`A&UQ~zjVRW08%l`HMXn|)GjYjX_gWvqEZk80tHv9Suj9Fh-JT%&jRF^8P zR~y!z!e%G+d!O+XqU}APMjc(sMu81Z`rmWDBd`tjgaYgq}6p(4?D>G|HN`#EooUY_#1TZ8khY zBo|9*4c8lFQ$nmZ+Ac8`s>;J-(Q8&)NNzRS*4kGGpOBaXnm;>~X`Z^mWkSk1BRID` zTh%y3WD!YzD^A+wvyZOX?23#GGzR^eyaZ#tRW8?o?;>CwwP(DW2MzExsDjYBT`fpM z>a<&%7r=ztv@$7f7?R7ttpjdM!S-_snpfJJdEvm1G-z4H{Gz<{SX19uQ*(^ zZX>r^v|lHr=7-0_sMG8KxQhH<_kcXtiS70*T@uX`|6|$?7pEPyk|DqBhn5NU92VSc zuO`=IcXDhkne$A1p>C^l>2|=i-O8%FsNq_Z^-{6h1J4Z>gHC(B{+8a_p!gG9#5JHK zCZBeMnO`Esmrd$0Wcny%rqgWl3mq;nF)pQAyL6bm&It<{nRL<$o z-Tr|~a$r*qbaf~V=$l^%~fzmR#4em`_RbjZ>w+S;C{ zbiJB(UAhl_jTU%!H&AiVzN%c>}t?1|qz<45<0ed}=%00uh#CGcu^x!HlQ`4Tl{ zWG#UQHjq+q%W%4wl5FCoi4YQ*#_M`CJ-k6aQk`^gBi!b8UOO^y()}ZtiQlRiCjo;K zg+zaA?zwdB9`nn}a*xr059ryAIDDYmt|m6F9U1+k;(r5jQ>bL;JeD1Q zKj2JvUGDt4*y_|>qnLI;?TsIZNNH#HjnBryETu)n8We7&4^pl_v0s` z?nVwn-{q|Gn6RdhcbWI6MXCXqVI}}{X&LCRMj}(&ZhzZH>i6VGBL4K3Xk3HuFHI^H znnh(|zzVp;3eA4{dXw{^U9(pBEsLwVYZms@ipc$O z#5~()1RWt(z~L(X03M4mo5woa_*cx=li0_b{fC*M9FMaRbJSct{zu5QbdMZP1f|B}UXH zttp;Hm~#>WZH5Z5e-uW7&gH9P*kUC|`y!i~>RPoNtZr_F)wPjHUuisrt&21*LjZv9 z-z3lX+Z^6-U1Ok>3>ePqMI}`#?Rq=6zx0^$+PrTBZa33V_+Au*8oKY|RD}|;R_cmH zS%#;Kfnui=t%}Jh7Kk~dC-zfk_%;xH^ss6-ai4l@h8HqHZZpoq*8 zz--2tPi8(nFLk}YUHW~n3=!15T(o(WTAVP>43w@8M&+>E^_t``GfsGbp6?1>RySH* zd#;dp9kw_txRQ^O!en39@Ri-#?&u1yb%9l`sb=zR@lNC)3Y}*ges7NAUBC2@fa%RP*RAuxGHEx-D zz#}2b;sX*3$WCiT7sP#OBo_ozQ{vS*VhkDYe@SAp?hbG>gp>wAen!G&H8~hfVp)zN zD7>TYhLupBA$#z+`4dMW79(x)1B!rk#86nIL~0=JJGZkE+KxmLt5oj@n3=f7O1@RR z%#1<>p#k)ESD@slL z?#~PFH3;%DzGdB?p-rDDZDx|r7oyD6PhaxMHp4Rq@Dm&=t-mdUM~bc5_C zzv&fuSi>P|*ExKiZ-ETs!zsV_8+*SO?Op&jx90cpW>2RNh+2w7-RA+~SAVLznLmZ9 zlh4WVq1`i`y7#dS@1u*wQm5;|=w~84ehY(-HM`a#*|w8~f_A<5<#2RPb=7S0f15x; zLhz00qc7Q4h8e)ss*aQG!A?0|pz(H2j4?;q&Bl!f0&c7Cvg6d6i@w9)`FeN)9y58A z_k5o*y)AVtB@cT(U!e0TsB>B;hZag2S`9 zG1lZ+YH9B$Maj&E+t4s=BCqM^dSOgxmwWvW&q_%Q%&gyg2Y@Cy z7K%L2C$5Rim9HXdji-g>#8f4Mx3gG7T`k>caYE;P^Z>KtT{3kU0 zBff~XV)?L!(X>s8ZHw*#xbyebl~9$EhP-TyXnOjp0sFW$YJ+aQM9Ji7kni#LK-ll9 zo+o>?iPaR6Mf<9dn0FEtbQI*>Zu4D!A4VP#o@Cf%wxn1Q!q71N_VM(M#BdAl&_MnT zU5og8lACX1x|7&!b!cfCQ4P1kgqI63+md;idQC*C)t|Xy3m* zpU=;ol`wb#o9l;<@4bh=jiJ$>Fg7-U{6jF`w`#pZ|g#B7((xviKUe>@Ftlub8e$W=d?ZE}zaj zN3=%@`7#3l9<=JJUt$BIv$g!081EJ`&4|zjiM+}H5#|+aLh;2*NwyC50ix7$W_`NcATM1>dF(T2jy~nwf^!VYpF0=LhYcM=^pYWGigzu3^ zg|)j^0I{ISMt+$k8-Ts78vY_Qds@`qdRAL&mD05;Q~L+2$vk%Kh5bVCqaW2g%OzQ{ zd02uGWeI2+>zB;>X#1;|Lj5`qSu^aZ~{XqX6TFUS3RDa_WMts+WOO-JLK=<^0gvQqKC-*pM zeLgAM0>D>v&3^%a-W4E~ScJjpy$#)2y@vO;7Dwj4RYvAa1qdqG4s z3XLO1qlq-7F9|uX1mGAIjHzKC6pEq^`*)#tpdA3}IA1TDiewQnmBa1S5(Ln3uM z5~S^KJOM1-DVzb5Bz+Jtjm1#?5iL?ywj2-kWob_nw0ZR_I=VEih;MTt8R@;V>2ZQfOmF}9t7 zinE_D+`6ovyWtPONJHb~&E_Y4`7KB1k^Q#mm0wO5ZLoe+WGF_D{F@RO zP|onJ^bu>~0$1fpSicC{t2QPH{PE@dS2QB^_`UvKUNv@b+r>9K{#vpcXJeYoXeu#} zxj-7h04X=*wi*{lcSehRp%Rnn{^`Lh8>eYlqo|MmJ|$eLR<7f5*nuqYw%)wi;Yr}L zFGO`4LHqM=M7n@M0Evl^`*&4LLu=GX%kt%RU3WO?}hbNTH0+p0^^U)=;Mc|W7^8%)-ymIEd-*Daq1 zX!IALdA9yO7H$qTPn66bweKQwwZ*_n(78#y`hL7@$vZPXjY36!EbG_SdgETNQY99( z#jkydC$V&ncY`Gn9*mCT5KtPfKJSHRLSX6QQ`zZw1sv&SI)5NjOXh#xbUT{U@5Bh=tgCUTor1MjCcu%Dl<}A8znOBr+~n^iun0e< z`Ex8xm2etHNeVd*pzeM~qKV#2)AIFNC9<2m=E<9=@V=p@g_k?#E3n6cnjfY6h|3K7 z$uS7w3%?`uCe((4xsIv~AbSO&o+VmZ2^b38W2Sx5kyygt{2MY&KBqDm>@4 z|8eN|{)=VCJrf2m4l(g(H4UVZ1v9oC#y-Gr!vE&&Ua+8}j2<4Sq$!8+VQ4bU{0(z| zp~{z|iz@)dDV;GbRTgIFR+_;xje`92lkefA{t;itK?7Z+@yR>7y=0OIs#J`XHFRdc zMG<%~k*FWL*Tv6cwnG3X-?Q$rqzht6m3cZ8_`nS&3rn*GBmJWWZ7DociaE_LrKl~c zkpxonnZWLR-l*IBfYm!X)N{WYfAF<=x?Y7FakkFc@26qIl$9|hP7iYY7sFUKM5TU! zJ0sN2tBVi2KD#nm=EvmtItHfyPdy? zx{j0ERvacS?FY3e5mfI?=jYv5i)1ny92AojDRC^l?w_zfoaeS|&`}x{7B9IvZ|1gi z{ptj9_^j{Ot$p1BIRuSpZRXf+K+o1nzWe6powe%Bh!Tzl+GGaxwu7U<@m0a>x~e5B zXayTl%-9z(^yU714Ex6zLlaIHUMcTpAIX9Ze+*Pi@`_KiP<+#tua|pYWHzEXPjs_5 z!u9=%IdS6P1<6nz&Z72$DV`*wg>vKF@34YKcF$Q-Ue87DiY~$ zXxrQ<7YcU7)>Mev9?b6H)P{$JDSF{maQL@$N3h}01Jb1;rB6@hybg?ZxgJAjl-LKO+1>ut%`vMxMeEm(6s76iLo^Mfl_MJYnyfKg#V3U_c_PJzl%B zw^h4XRjKFWBH&}-OsC3g~BZ%LuTCoth&O`Kb{xLG}VttBdp&2f1b_P;?EXXWLA77c+9Fz zebN*ZfzzXMjOw~BNi@jS8hiv>R6^V==!p59N+ii=xeQPe>5<*_4DLnxdKhH;D0_&> z*oI$4Jv?)KpDo*Om2;jGrPRL~@wMJee&qn$?u%m1B2zsAPTTkyzN3?kq=XuJ?)mSX zlc4nOXE-N)jhtOMfS3$pSxnWu}J_$fbhprh-PEB(GjH%R;VLbI+3HR#kk+G^e+S=15u0NDy;*5Q)vVs zHFF!eXZ#0xF=+w-)ru(Q%}F{K_!_0}S2)JHW=?yu`1Qi+cV8#@ua4&+@;2w=ahv-f zEpT?5vu4=naNL*OeoHnp2Hn>25w0tac?vbfJOdd$(1ZLNXxEXtx?DK;j?FJ>$ zq&7-+)CBx-A|J*Nzr{bb?F6&6yOxb@W5RA_>B@(+(6q!67h@r65U-MC`_Zpcxu)mR zNyeK`xyOOuhO#T&S}&B~Kz8cCov2GHf#sm)WEy1L!i9)K7#fF>S}J0W?OEh{KqFV18~UULVPH714jDG;zs%vhWdQ?sV`fGi-hU)zcqED8Cu zoiu*E{Mad6;(HK2jyX%`pnOofhP$2;bu$TA_|SE`p-lO>-+U2wYMw~vm;Xmwi_vp@ zY5_cb@|JTUaqs|29~HoRQZ9#^eO7L4oxKB#Z61 zDfnY}1UCja{HIj6p7v`=>gZL(efIH;7Q4^7rKwv$Iy+zh)_5sciIGWmU(+|#?$ zjbYHvY~7OTp5Ttb!<)#6@htj^fw<5U8)hi9)2?K)&aE<%(gzpvX~BFPSFY%z4w+H{aLC5}3XpapUA+ zu}+DojRRwiO$J8bzY7bn_1sCdr0Mrj<8h_~+2z3EQGNuMmXjx?4(i0AdfHdf90XKg z{bZjN;h{8fFn*y37P!qt{^1xOd2ifFqNXcWL^c9N+$b4uUu^+NRJpwxv+qoc>c$8@ zsXZHs|*7mC|@XxD}aq+7;v{p2Q$h-QcUcV$1Mi-W0F|m=@LHo zvuGN9l5=h7dKgPcSpb*^Mg*k^9wUb#1P;<6nv&rqu6*=>yG8a+n>c%cudGn4Y8f2E za$VzGtAZF_BWoaqD)>)_L>98W63lo$MG0L81gilkPnjevDnprqLfTu6&oE@$+Dg>< z5=hkWGyyUQb`;8z6}5pxlthXp=|&W?sW^%B6)L=p-+YBDRI@yee z#dAyIMY)giMjsLSgs*V&S1HOBD4-SeM{-Al3zZO_J(_^vP9xN7H%^iKJOx$bw;>nEy91_uI$1*-wp>dj42w?u`N~!``78zNmx?J&0p%#3po1ld z1Vup{y5#@~vr6I2zQFcdOOxGs$RK&uWEL;d(GotsPQ68qc3Ej;I7A=Xt{&BPikNbS z3k|)NHEJz-yh>V=;2sYUB_uWA0mEjboiwi$=CPDzDS?2`p7_iXhB?r_cR!~8@r~&5 z&EB^LNY-Pnr;Ut!0D%JQ^8Wmf10iQ9Zw0+yhLD>b4+2)MYE>X7hdjNu#&aUClNBKQ zUa)eShi_VcE8y+r#PGJ``QDLh*6i>0yYY~`UTgX7Vc!xV0rFnv)c-@?IsRwXbcwoS z+g8VR(y?vZwr$(CZFX#19otEV9qZiv%=^yFnK}Q#`Lyp}_Dx1mrWIu~O_cU26n}m6W4T`Aw;&MOUs}C#AP(LhMhTRD&gRLY0s`X}sAjd= z0PbySP;@n4;Sr%!+XR%xT9dknZC}QK&Ua;RW?qG`1xAOX;CwjFN+Opxh#C+|Ui; z11Kgpk>f*8e+Jr2wa3KNlx54P*NCUCkq!})97@C}{c*8b*h8U1ZMuYHCz~0eY?!ay z9A+?sH2ulB*iZ(|;sc()W%B6vfcM{;98xVRmmBjl`qv`bUxvkm4?m-t*4XQiA`NDd z>wDWx8v|hP-`(_tMOk0G(4U2vfNzz#H3x1D7fIwWEGO2izQ|9~0f1VvN)V|%SE(_j zeZiZhOwu~Qz7&Uu#tjS1{4u0b?;Rp+@3#YC@g6VY0H3H<18NrRsNU&!GMKcDV)7|d ziRQdu@__j0X4?RZ?jo(_7;y3&%6xyiHXt?Yu3}+Lt-C_3GH<2ISK#joqv}tS!)E(~ zl4@x#t?_&>he@Q)KVi5yA1$KMP5@zRB#SdtVBR zR)}OfwEug7azW@Xa5ifHlT=CyuLmZAO75^l17Pg9`4a{L0hds5wqmiKMr>!lF2uGL z)qKlytr2ZshAJhg^==Dt({W_#C$J~;R_W(rhm3NQAHiq+0+6+0i2Y!1G2&RaD*?CT zv+nmezVrCrosWF57az~3!#|6pZ%sKY&rv9T1BSTG-X0bgorS~MYRDUvKz?M#qlf|B zVF({LFJ^ka%XUO;2K$FdgI3#aGh8(Uw*Z<64|GslrNl^Z47Mo=>1^~l2fx1iRsY$y@oYWvQ=gYZLcg+iVuZ) z$jwQk$k+^S@o^fkR{|#>6&hF}YlyM@3UbfO2D;5j{5s=?l_SAEC6%rs7No9PJ7)C1 zh;jg!0+vCW^^J0h6y?5VPp`bf9S#P|=+8pwawah&eDZjEO-xg>#_a+)v zx1riLWIG67X|xhz9M@*6Uyrq3_~|@e0JfxWK>}vZe4(JIS2>m@w&`DgMx<3%q+O?j z7O9+2)&oz?vO2j8-uwEXCmim4yEfaB7CzqoBW7f_HwV}sRAlv;j7~Gb5=!yiJ>6cS zQ`>`2j=(WCzPt}TzjB45pMOjLCCI@q`s37#x{n~hWXbDl=XR`5Atq*BNq~$Rk>nacf=f7 z%eobDd@IE?OApT&+8-1z?{y;WjKQdJZ`PJChBM0hxHMZVItCl~-uo$@t<}>`(hjPC zaRrGFTfM+b$1Sh%+@4K`f;H?rCS!_S3|! zQ5h*>>>qK%F*!fn5K&K=uS9_dYKccuD8BH#@q=7yUBI4d(kcXS27ztPH_vd1 z;QVI>KtOMiz^4k^)a(!U1(6T3DE3egXl%VeK!qn84*Sj7th+Q`I8$ukT`vHMf=;(S zn?OPGfNZ-#3#nrvV5hZsNbeWMBe~V>rM5zbb9m+YklC{J!%b43t?^QVM$_GWx7(16 z-IyWt~6dKDeoXw_j#A*|=M_(1v{gP;VrO0=|$ z^RMCFOw9X<#|>gU4GP=t+x5-6qTr>m|fB&q!xFr9A8>F6~l zAAZ9)B9JG1Mix)xIQUh&AtkvumyRhXqtUEL!71D;ChNMbCw$$)K$-K5Gv}0hdWSIp z9pCGUo7^R)_p4x3?<*5!|4*COotIk>)z)4|!i6NRL32px4u-0rt!ebb>;58Nk_QS`61!pP6Z$Gtt_f_!>84 zyaZrsLIylfi+!@kH^423D(Z*iTgHK;zyS|A?74iaea&uPlO+V>F3%M})_j+n0Z2`NFz=m#I&?DXNPu}WQfS&7>tH|}`;cKRVP z*N2Z#)6PjjGyo=yCOs22|N9?Ac(tsSK+tpy=|LvJp^;2cZmM7sk5zse;DSdAIa-17zQ(Dc^jJBGCq+a1FG&#~c%JEH zs5Zn!w)WP*yd<@$rj$VncJ=+HjN3JY#3j%3yBL?~1h&}I z)kn(4a&M;rWtZPe5*`ixh4?}x`!zRIrfn9B=?|#}+SMFjav}Iff|Yn!M+d~F1mCQ7 z>Qawc_n2BF*M@-9xJN5m8herquKJi1q*_wC)o&mH0YB<^@hNr+y+g2UY2B7N_EEEG z!Ogd<0p75H`7s7@ZUdFxk9zAIkLPO#rzG${j*5$Jmq*OnG7_y+>t5b| zKgs4Y>GZD&X}zzrueRA;-`P|^JS;sND$Vk|pQ2=C9Kh#vfy(#%wddvMH!G-enJ6*H z{uT#)oy6G8ptuL(-1Ab(b?nMKfX;E0tYnRKx0J|v0R>kVBNsCq1MhBX8#{ehBL;x9=@W|sHmUKEj#uB?{NC6hk{0^Z z&f@Y0*5Ar#T-Op#sD}864#e@*iB6)QTA-HAF6p)_YV?sb3b<{dAURS;&c%jC9$R-2 zHsii26XUn`%{gkn!*MidHCDEYiZX%E9#*5|qpNacbtS{3fpGEIeu#SQeu&hg{f^Rm zC#L8_F5llfrDOEemW5Lh0EAju6l4MC!e5RO3UcQVN_nhsU79yZ(?PFzN2izE%#`1K zXoODVnV{nmi6x0Ghx02Pj2zMeKLg2SN|9V96m(U~gp3c-gs~C%$Rb$5Q3x{>;%FC#6lFO9*%#p z_W10cX{^(9O`!Wh?PxJo^Er$Kn_9#s$}#zm0|PFwXXt*3lB3rhlo;T6)~;^NDcs)k zqto@uc~b$R_YKePGj8$oVTrx_IC>Ef(yOfZ1)y#K<4~hpgdCvZy|BycGcH-5K`uET zl9!}dK4KrQddBCm6I?jqe8Rd!u%CRg}57%Fw6K^xVZwX;(yaB77W>0P1 zFUd|b&reRX+z;K?9$9{gT_3b{Jo|GWT+S3W`{v_O2|sL50(&@(S6DtX0_f!i?7C=W z%2Ijx58TaO1SABE+rb3+ZyG#lSy z#(`qYF>`|r|82(~mf<$Ka=tH;6h1p`@{ zPb`gfh=&qOR3-^DW^;_NW+acd1#z+$z%6*)?maFd-~{WPnH=#Now{A!1&T>OoIczW ziUf)vpSY}LMisJn2DhxZ8Rpy1mFLWM(R#`;swV_D)HVnXBYeZw1k_O&=%y0InFY%$ zg7`y4U@&(>?s;d1&j|9|d_o|pQt6C=p!f}BF%~7!HIVp+!u+sXK|F&9;WVVd>^xw| z%69T`F}^x(`>ICJSZQa)&T2x^FMvI8ztH5zx2R+47+`NYM0)`e4~2nQ;cZEO&WxEz zaS)^z3Atg3AH9lH^z>~VrJ>~9MC>Tbg1{aFMXgFCx!Kv$kALppKqBnwjuO|N5OTi# z3IyMu8UC1wgU_q6A@YGFiPyadwj)_AH-Ck}|IuAv)zlvYN& z2aONe1VLr6E$K34S;x;|ZX;E!um=AyP0X$ZjCg%&U|-EY+OR4%%p5QZL&pzloHjl@!* z>R=E+M(y;$VbbU%M16^T<%9>996Zr)w~UG~5M5x%r>OdBERy3(s%zaj2K4KY-f@^_ z5=wKQ!B9s2N+1?p_VvO;#hf>!R4x;{^T=@VW`r)e#4%EEK%2ITNL|U&?l2-Z5fWrw z;o&)lO?5?w2TPGvAx-jD>TOG;Uzp*>8I!=8_=LItfaJyVBAZj+=8kU zZ%QJR&X8=+zO<=t9mV$@Uczihd3pN-bVD2=k2n3vU1q=C00?I^ZV zFk6!M*Kbwm5F?b~4f}5)$;&MOB(^{G6wnWoIwyPv+{i$;OpgY(3_@Sz2CrX*W?Rg92jH=kE~l^KH(c-@iQ)xbKyOzJefd1JgxD>1E_kD^6CyfVU&Qh(?EGl_eGfu4Z^@YWLHH|}_#3eM%+qBR zE~7Uj(OU;dc$fKJn&>8#tBi@LK-juEtWSR9c{-Nqm0lUVZw>=b=p{!2W)U-Sn)p-V zEM`suMGx>k|BZ!}D5m3ZeHKx*@dC+>W0V11!<&N*qXzr}u!bRnOqV1lbz&jbcDQMm zB1J%9{Rt*6l%>EFY|nb)cLs#mA<9h|^)Y!IPk+mx_#?+#acg+t6xIsl^|CP{nE$n@qMO~UdeL8O_1DhnhFNMsVPNDz7&un&7oS*XtV zMrr5Z=6DTVT{=$lQdDMH1sJ#{3ua*?9(PKh3@d075rixG&SXqjsl7x-v4ph(mUwiB zad&z`vtetE^AQ(N*9=I@woFiy{xCU7B7Y!#sUGEEpeX?`AY1`$$?>p}-8W?S@qXVc z2$})~mKa>leCTEvampE@j6(F*P_sD6J!J-Ocvw+dQAc`KY6b`Sb}aqYO+Ibb{DVk& z81rxRb+F&UG}Vv9OLZct+BrQ~`GHT+j{9+b+PE(&C@2_2A>o8Rp~?%KW(;}mp|+tf z&*Mhk>w+{jO#5C_kC24kYD;EtPSu1=o1_3$W8jPXnfx0=j(gI$-~*ycJxoc_I`GeTXbz`( z^kxgP-~0Bt=eI1EfB27xxW5U7p|aOmT;5757XZT1QviQ}19|S|bUvM&oSRLC+^jUlBKTqATyz2NKa&XG)| z8Ma77LeF;r={w#w=`s0nvk5s*8Z0gf*t$X@2X884xjXNH^7wO>G}j)gc^P_HdhrIb ztEFu7^vVHh6SG8f%5`#tRGA@^h$3o~lC56}$9M9CZ>!*Ki|Seu7|1pmIm#+p0}d}0 zS&|nVGddnGRD~3G#6ANQ)qa=*eO<8X?Dl0WLCFdS{^c)whLq6*O z)e^~%P1ILp_S6%!mXT3H8WLgZpo(u3QHJb-T#*P;Im))+47_^BCc* z0B=}MEOivtp?I8-!;p9sJ~FEF`C*k3AY?HslWYsc9_qk?C~{$>_q-@P;+n~eGGx`++Y}*OT?_ZoQ8Hmd$ zN7XnC7udJ*b5Yf;~-Fwt< z&|>~_wn;)iPm#7I?lX?`m9fEFW~NZUU!~@+QWW7wIt_!IqehnL*uFNyi*d?LV+m<; zh|7{?O<<3XZiGw|tKhwU<1(>8fTTvd1VU61iG2>Ohh)aA><5pVHm%VKmY3LWN@>oD%B&EV*;$wpX!+RLAHFT=R8h!CQld#G!7x%x zG317ux(SEDrOsQ18B+Zn0d)BJaf7_K9Y*m0@woHM2|J>LC5d|>2&!}#uIn+zr%mvM z$dV2{c5q=u&M&a{Onpqz=y5RvZ*p-kNTphc0@ev0v7R*(0Hett3{ZCkHRWVI9UZTF zs?@wRA+>RuSUQ~9XBJfW7aU+0IA3Kn)I(lbsy$nYoHwHz0zL%uLd_WEbKU%CnFeErFpQlpvQ_ z+$vFktYPIr)h9>K1BF;0hGJ7=o4DXub$nPRLYs$0UV=`1%0pw$W_XzUCLgU=(}Cbq z+2sD#8W2Fe{(0ojA`vD#8IkM_rYjM|X3PcaaIQ|J!4+g6EGSLFbr=rqQ55O+4+eCs zRQxkGyb|@j$-4-Z?3gG-{VSDqA%Bm8YMiMt8#&S0mJt45Crhm*3ioLLk;f|u@C3-) zfr?Vrzy5$4Ku40VTt7KyA@FY@{w2%b$163kB-q-zpIX3VnC|Bx|}Uq|5pM3`Rf0#Ea6fEZPTpkSyVRuX*2(SX1mD04_Iu_uAtJA z^*LD*#K|%H&MlQNOPvXDF1TFu|E?5&>udoVs3g$B#{<68!M@L&#dCF99^}8&=3jN6 zZ-yjKwc6wkPhles8zWpeI3daG7yn|W~{;H9#>BG&(sR2~!@6`+O?OGis1|C!H!r4SDd zc&Q{HVe-E!iy!I#QAp=#ewx&Ny&t$|MiM{r3)uciajp&0{jK#!jm&>Z4bKit_iL{; z|L5nQhT=8i552#y0{VjhQuaI5w3PfyfFK+oZF@Sk<~#nI7BI%%9ricW>mduQ2J_I1 z8)+oy8ck%~_f#u!jg_-&x1&RBu@a`iexESQ3qWVfkNew}qyAPd|1M0QfneXhW7=Fz&9Rkx7hv{GRX>k zw<|2#s(-iUA9he}IyF;OaKEd;!EahOckbF6Yqe9-F5s>Dg_M9=KBWQjs&60IT+Ktw<#4H-7@GoV6;4kkeAX#JvkYTx3m zV`pBRoM(mmXkOa2wBhR(w`~0nwc!5}B1C30Lgu!6XM{qT5ef|PAltLJ4a!hjvE*COhg!kAM;@Y$&O`;6$Bk*Mh<7C(%=iw+knAnenJ4i zTnQs$0D_aDG)g&s4H@-V&(_w}Kp@wfXUgUx={yDc5o$^^;$ISL67|LRzeAzeBR1i) zEG)hUhs3Snjx0-?mggiuIM0W(E;7ftupy7f2QOlZ!zfQ=(7(=;lw}VTmjuv0VA~oC z%WuyRscc9=*jOcHE~VlJRaUiBOxMU-S{CMVs7JF&h41^LkCKCJtB$EJL=0@P)66SR z|08Oc{|-TUW$mpy4`8m<(8$n@>zqWTLyid*EC>zd*XicNjYvhuaMoDf8!Vz`W#=pJ zP*k1#&(81x4MZgFEm#$OUy~8^;7no|FtV#vH-vreqZIc?SsKc216A`*)%gow%{JG} zT4W{rC6uyXg?oFfXRDyZPVDI_nvKmpH1-CMXM@Y2lE_hZfCPNhu>Wa0{Bl(qQhugG zD$H|4b}ZW^F+n=HjY7%%Vd17DWj^RFeD(~B5db6tl{66^;z^$~HHstGRKnAIFKsA0 zj;HROWA;wIZs)ujNF@7ZB3SSYYZM`nA{x?W8?K5s1{tmD8;&SG9#0yT~YL|~w^CMz> z$7HA%Il)<$XENzrv~iO`KcGZxby4`6at3Y=6=v_nQ${E0SwJdbl!=wg$RB@u@eEO) z_uP4=X5OrpqT;Db4*#Q0tVw>&14b zpfVGu7fG{e)8tl8%~#=(lVJR9-_1ZH=Ic#uFsK)v&I6{$xUkHfn<1%d)`$9}ORjm< zh!VC=PsQ&mr{7I4?rPPtNm#wa?(yxe_Py2ZeTc(jRbk7=4K&w_snLa`%y2KCntjJ@ zD&i%tcb&&uIf-k@HuBq%D}=6Z>)z~BXewQ#iW(G`BU);otP`J;%1VV9j>9Bor?&UN z51h5`qR;J(K)W6Q)xSaF+x`!47B96!y+_vT<{~cneb=`<->%znz7W+ejG%QaPxQ$f zQ_m3<<7qy=FBr8f8}cy=;y&AQoe!V3u+(n6t+tIrArv2t+&c-*>t4dACOwBEwDj|5 zr%nb~X@SJT>^-Tkv9Dy?&ICGTN z-xjtk9Iv&0b+Uf%HtyCcv9yB$!g|sY^2=h1u%x>Vc<_)M6!@cKhj2@l!0ibfmAzZ*(x#?SW?FY_b>OQqk9iB=>VXPuw)T8 z8-%WB*kaVc!ZaaNwP)M}TpEhe| zzGIIgEy*oP&f2za?)R?%<-^x~gh!I}AN=`wIJU*7M#mlJ8LrhmSz^IkM% zle0vt-JV*_I6u36{ra@u(qSJtx7;KdHO~Do_*I{9voJR?oKSP|5%fvY zu^h5O$M@%hUfAuruORc|jNW5ZZX!Gs#I6U_@Mh4wi0{kl7mnKZPD=buHLnYo0E}E$ zj5OokFW{Z|Q4sia2wflEoA(_DKJ=YW#O=x4E(mL5mI6p;d?1Y{(-fyzupf2i{;A*c zJpGJZT~p~7e;vq7%}(;jtH2RWO-~niS>^`M9Xu3=2m+Q(4p`tpNYN=zc;!Fr>m|Xb zU6aoC>?iIUy_d!s3j@7#uOBbGv`YENI)pZm*GP17jT6N;%mm#sHwow)7LSLdj=8JO zbjAE*2j7wWNJ)Zr?WS|YX;v|-ss=SsHD7e?#|v-HOB#e!8tlW@vc{TX0@mb-Fd=*H zMU78f?Oucj=`tr>H;&b5kVAt(KOfrd9>+`Kx=*nbLp|bWGqvjAdKw9DE|UWM_^xso zN=}cxxJgF!5$@pPHWzPBc^W1@V(57fk}~q_(>pR>y?lYP?xHDYJI~jvJx+c~`*W$$ zpjvxhT|Yc+Vr#Y)e!B5dcRcu}(JMhJ9t8Ta@II7t(bL}-d%Wy)%riT;TVTY_(Oy{k z-q{3Ov^xj_WBan`GQV`)NZ%R3xnz*%=(;~lKjTclijLhMKeV*zj9#vE=OV5rq zlqwTT7E+;Be|^vvm|V*nU6trf{@i6A`CbVb22B51_IZ^T#$Hp^dp)zSbv0=}xjXTa z0fb;oX7eG`8=-nNo9$)0*V5)58WXIG%uKc}ujaOaV6L{id?Gl>3zWdjCQSS}dSygc z<+gwnveUWDe(z3_ose)}G5d?pU2X8gjrINPYD#Hl?8mJ$6G$C5hW9PjQ+>jV-mC2+ z1fNMdpV;?>=Lx44z7N^ct&mUhPx@=89sJ!O-bGWM1w5c1Akxmf7f=xca>E%(S2pIY zb!t0?VgpEWuIP5Xin!RMz#G=4?_XU)QS-kMYa)Zu8(1vKl3)(`Y?NN6PhaC@c*I)b>8H%v!`F`9r4sV!~aB$&ZAnjWN zUTz|6wNQUol`R~NR#|7t?ngt)LPO62M;LuTQ^d`o;7{R<8uz@rlsLB1lExfc49P#k z=0`*U_SSBGwqxlU%YX~tjI;ARRa(E<>K$qyb4fey=1^qk^SGtC`*LfC#&s!6N<(E?2vEOZ{OxgbDV8Gb}xSluN^KsNwS73IWPNLU* zN-N|BCKEZEs+Nw|WJ^E0E@W2+p_MHTo)JSl&)0jj$B9XvS3yCJV|&q6U}1JtZy(Lv zSgsBY42yES6`nb4jqb?8=;hCoSJNDGMlIV2SasWNvtB{p>JG(r3OHCB_2DR%+3o%mnr;wD&Hq%O5N8Ic1wWO0rK1f@Xi5Diy9u<$xeZ;#Lvb88 z0MWjs&Ic}tTrL+DE!m6k$;M+}e+DY)+&_2@ z&7|>DYbLVJ5B8mg4=z2N_dG=-!|Kqhxbn&Jq;$u+wma#}U*6${)V~|j-6miMN`yZi zu=+<})fx=AsyAW4(@`^hw#ldE7ovTtZxPPaX-3liZX$;{uGH}$0DfW{nA`Pa zB&bs#Lr0+y>+|vys1feerN*;w{DXsTo%w-G(SePsYT)QSeW2@?_k2(qOtm6eXksKi zP>W1*JfmN8c0qPs$qq+u<%KEJ-au;L-t*-VTaM)u%m$k9O1ejq5%|}1%kqhGlVM^}Z4wRXua(Z>vGi)c}f5keQLBY^@3 zC&}IichmiEW(}E>+4V7-=m-D-Zq9jFugo-LV^$u@ZsY`DBw)b|Zqk$3FgV9)Q zRaHy&=lbvU@w~r(Kc&iMs{^4)>*o3c;^j=QC8Wxa1$l4deqZLo<18$5Mk>Kl^)iPPOs zL|24DHW;t8v-{Ku;tpW}V+6K+Mz)w>h_3>*EUv(tk&Yz~iDK?)vMs2P@3+AglJWA> z&;j+;O<&J>hmHac z&0W$|{LBd>5p$^3jE;3`9%DcS7;MjLBkb11q?1RMsRi5F$&;S8B?s~Whm|cd633tx z8)k1`134BTiflb>sJ2HLKAJ`wRil$tbLoT*6HAvbD>ov~wna6*^}d2CC-s~;&3O*v zhKhC~5UFh|l%Kx!tn6m=_E{6`fGkC|WH(A#M`Kfw%3DT92F*~cgS#k#%E1(?$IUdmTZ!J#0wre(?7+wZCBC?hoQL5fH1|(u^RtV=a;kfhk#|&xTuj z76i8imp8KBDDQ>%J``wb3Vez5WNS6q-dm6pu={ROoUG2$EvH3XUS8I|sJ$(W_0&mK z==&yVbaZnK;%O$?Vb+51cp(M^xFCG(>F zj|%6Rrjia)D=W)QUdVCgY=#r#Gi=7Q(J0!CZqba9%c+o0HWZg;BEGh+s^>-_s;cwQ zUN`gS!q9pYR-e^y2+`!z&Pd1l#soJ)FM^XhCWVB)_2v&izzw?|2T$NgW>kdz3e}CJ z@mQh3fucs#t{GRQ#zLz`4+<_C_U!qePdX@;wg;()d- zdx8`vm~(6D*!C9Tj3`D6yKhnK6ZVRXd7elF(&IDJJ6ytt7c5#d7fq)jT#K}2i5HV> zZ!!C~j8Z>+wrc6qm_w6Vx#wMW{Qc)+sewuW}e3wrSu`Xpf<@4>@ z8O876bS+~v@0`n!%tp>M=9m(gC+J%@Y!cmS`+GLCoWVH?PtDn(Q3D7>6-HNkJRxbw z%Q1ZMiB5~D(ow-W)PdOa4^K}E38F5C*<8SGrpbBoam$L$8ZgQ(l{Vt7h4|_!#p3F+ zeKn(a)QM60LSJ^Y*&d{dT-Q?Mj{qwfMb2VF8$~G0zuKddPa}7}zi-BQ>*F^g&3`~t zs@83_TU^8rPyPa|5$_fV;AV(xQXXE}N5ms!P9HQEdw>DE8NUeduz*G)*o~jkxeT`V zk2)J}m*e#-aIQ5E?@L&dcM)bmdb}16>!3jbo}zmS1M2Scq`-&du5#Ieuvn4#F?&75 zC^dqx24=JoOs=svLYBhhWCclsyf|t3mNTEb5Pa>2iBh4kScz%Bm~;zh*owl4Pm`N# zxMkz?1xqm{w8oIIDAQNpIp7@h4l*}oQ9wln@bAmp@R~P}lBmH5$HMW~i7_Q9%R*+X z#58MlvJ(O$2x*uiJm)eB)0AbV;tl9e3JB6cgXcpV3~}uL z5vToYY;)Jr+7s}Lt_!^-xh=a4`b98?1L|eaWNcG`?ENM-G|V%~oQhICdr}5biu%-# z3XBHJngMs&8U+F?KE&_YZ}p4T(VifQZ%y+X^4PY+7-rD)g^kr-9wPoCXq3e_B;bdK z5oT4k0d|K>x8qwHzWZs8+F?|sP2#GR*)mc-I3=>Nn4Rcf;r2;mAdIWo6X1;Eb~NT! zN#;U4HO#T2$CMQg(@eTbRdP-DMfT!1O(Uh>tJz=SXRu=SEbpTyfyCzN?IE*-u zl8PqDc^Xw&4s$1D&~rWvFHqDHc{vSAQL*c8E9~efReE^C0kLO>Om9Ni>3kyec?-T{ zz5YIef1{ONkLrOw zskpYHf|7Y_ni8`#vCrrsxZL(Clt8I2H(&D zy_7f<8HcVvetp^-!Ai4KsXz&5(Fz6{`ZCZk!p47hH`(FbLacgFDl}J_KQ&ZbgqWT^ zbTiKo_LuCX7|pGC33?~$6BR4656;`~sbVOsAN6KQ4n(f!45*V^H-|#o5@M$KKG_Ct z!~n;e|ekoYI~ zw*1H8$HyA`#KpVux3K$x<#C#+u>fp$1xW-2J)3HhtVUr~&2&qp!&pS3BPQ^zb%5i<4K%sV#|0l)dg`stxh>34L zlc}`AHUa4$$Lzgp%@!yyzf%(dkA{y=gj|`vC~W zOHW>ABEY6Hh_zCy;{|ZAr`KwQrGm^!|KW0;qNeMyq#!mt|E;6z3qa=qfXnjQY&JrV z0pJ7T&~=XL--=6b0GwG;WyTAq=bzKRl%3{9S!Nr#6|)Gp)4%*Aqc|Q_1qFvi8TJTG z;27T%uATnlYu~3%%o_j?bFZdtTkr(nzM&!gTte?NUaNCn0)@c)ygLvA(`L5=Uq?Yt zUq-{!jxOPMM6K7qTMWMH`JIT#ze&Fz7+Y`%bzx3fvmbf1cp!`2t_Of&Qa^`8!)3Ex zofjkcsIl{jA`M|9EOY{#XM_W`j$Vk&h9$^M5RA^mp_-N+M=5IInfLtw_6C4&bO^ve z2}Y~31QFGH1IlQ2UQLlK2ipmtg+lI2y31v<84Lu=`MoZZ=eidzkut%-!R?BcX9YnZ z+==qr?>^Q$ATmOd8v-~*eh9c6goXf01==wY`Y&A^V&qMzwoS*$NHl6PnM`4!c0cPD zVlq2K0KbbU{(S(Whrw(H*LIzK+Z|+5!%*^4Lox=}LJ+%FqbXEL+kty9_z;QbZd~TG zbLS&g>kkRR3*@cTL1la-G9QxERPxOStkt-|p4fs+N zStT2vy~`#owZIRJafT9mEiQ~4UD>leSc=Woh(95G(NOj`M7oreh%lQ;n*difeG}rizpOvs8eOgJaNS%*lExJg zWy%@oPGiF)brqO?fQt69kaL>0ce^=*+>lDCtv57R)>+1{KG zKvz8n`z4s-BSHaS#l?O7DdP)yBnv=i;OZ#$lFcXXp1{3g2C$v-&puiQzP=v556tU;-M5S^2`uqS8 zTDy~n1u6DVc%L|*i)jy}`|$Z5D&$H8!5FpZpk>Gk&kBUXta(@@$?Dc+ihgtC_dQnW zeNf%3C#u6l1sS0rs@wqC)Q7~{uGaUX7opjIAKEpoC*;n!?s=mlo?mMM#}!Dp0|nzdm=qv>J^gltM2ixwX=n*}-#J%)eiQ>3ZLal)qWXdwq6c9^ZzmUx(J_V(7MlpYyqyT;6Z z6g{%@JtU{LOu?HZ+llZqZVqz#_aCTxFf_ma-o0~!IwIhyk6X4K%46}Is&&(-c4Y?7 z*nd~f@~s)+h#yjQ4T1!n8XGupK|>hNWTtJ5Z@Bn~JYZcQ)VF^tSU}<~^D~D`+Vv=l({$T{`6h}MDO9gz?#yy$bM@LAx!VL<++VI z5jWdGHEXu`Av{qtKcghP=*yHZT8=kTAawL;`SuXrK!da4r$TXYcv>NSM zO)Uw-Y@b zH@`i~rVUD^^VW`8NMsgjLdh%)lQJ1s5qB{rv+4n{-cj1(+w>fIH)IWfvHS^dUovLB zO?P^HV2=P`C?N2w_dACljOm|%xjcg8Vg$!-sK{{$Ni6;g0GSR+U@ww<<#@H?2H->7 zcf*lLFfE0+Z*J=4`f>qCy61l5_+S@#L_ef1Nc`|#m>&=%g0#)hVkm4IHT?KYH!*hJ zL6L;udS9=<^n4yvEMHbfdBF<;^ZIn-EvhmQr9{QbU7&Xh3TL~IGp%+5{`QXBbO7g? z51ZGUBW2rd$7c?F>Yz`S0x>yg|4>M5kMDV|JQh9S9so|nKA7K(1yZEXif!Z+YWdg0J$9nP<8#h-+s3-Hg8*j~u76IqzFHY(b>m$xxF=_Js~Kl_=BuruG7w zF|Z$BFY1KEX8FFn^~Lx+2T42yO1(sT~d4I)i_tPL4%qutKC{clzVevR^kj)U9vw56t>{*gqgQdf?xJt3+m$Z zBla&NaRhz<;8y`C$k4sD4#d6wpd)V3W(mR9^XN-vGxSF55a``;3}c*xP@Q&6j6f(E z1cEBLq-!<`6yq3t*Jm%lF2*<<)Gh?eDajvnfP&EIT|gIJWiQdv`z)iQ)3&6rBwPGbu>+Qymq;Xlm>x8h`fD8H|0C z^>F?R&G#C8-;d>%Umd66ZbUJ{TmG>4g>kP}oR{)#`!Od^C+kkoAn18XZWfvrehg|hWV1uG^LQxrwOW&woEm6a zET{1~+!Ve%_;qcq1{nxvl9op=w+UZZF{QVr<6Iizr9l-B5z7Mw-oOoGg;gKmd82lcKE}4$f#d)VZ5Ml?N=Qa_5=L1 zgnUzi#Xm5?z9RxPWd01d=SGDNOAr-!PgfX6Bp@KvR{k!>d4$@IRc22t(D0r47R3|9 zp-^E)sk|g+j&ni~?GSYd*bH@K5aHG~A>z|4&nFf15L8vV;-}NezJf76Wm$Ty$NTMQ zQUw3o77G(g*h}jFVe6a&>k8JjAKSKV+qN6qX4A%v?WA$i*w|rX+i26Mv8~4VR?ofX z-us<%{(kq~v(~KHvu5Ufp5Oa-i-10y9_bNFpI%(BJZTgfUu)QgK`nP3#fa*9`B^!K z@9L=R+rB>8n4w`#&}M>5q60XyEE3E3sT$O#Bbyj{h;W|wekL0iL2#dCAr6|U)MeE- zV6U?M`E`(1q>suViwO(;z{bjodQO`?5JqBvZnW=fm}8Nq`ML3T$vu$zeS5+3wKkmQDqt~QKA9`Uot5kvpn+)~J zX>?$XnIR0PW&-XS9ckbZ!`{@1)31RN3*klE)!{Fa32jMaL0)b2*Fd_e$HCVNZ8$J} z3i^@hbf(hbl5=y@eV6U61?3T4VE?1Srv}xXv@1ZkV&HqyJX~jv4yJRYwv0j6{smQC zpkJ^>X0ZmYMvuU_V}A11g3OsNy1?EVhc3jO-*Nk(IszOj8A^>@u5{uom+hzEpV3JR zm2tlfJ!OM|7|<;tToL~iYOK;#q)ej7IN0n_5WB##&!yn3Zz5C!ig& zCDbz1a`Iwx5?*?G5#Qq-^-OLW9j;kk8Df~12NRY=0ev9h`xV=15p1m~_*7vxl0J6Id}D-0^i`{?KQ z?`KO>Ps2M2x$0V$I3kHIr!6P};Zk4t0v3~)oCgP2R-XJ}8F*b*_4JH9?mAPHJyyPC zTIwuzLBUzfvvZc21dP$1zxRw(+`f1m>@e&mj&PwK`kwnXH9XDY*0p*Kx^fDpYk#bp z4lvZOFct0gviA{$yByeOLIPg||B_#2CD$8?&x!yAhKenOW-a|iz+sU|`(7MuumXCa zmvJamDVmcg0yY}&H>_4r?jSilUk6`4`(-FlMPcDcOl30&jq@mwfq@}j_)a$)K_ksi z$Vh7hWZXiDyK_ah)gGD#%1R<)H&F3>h*Gh1?dT)ggrO_T1krScq4C&;*qZK9X>FGhlmo~|^^m>q$gSkC-lrC6!%3btn^Y^*i! zgWM+K?6G39ba)#|Op&Wab z%AjdB4=NSu-lWJLPp<>5`5o;&9wOouZ@*(3ct)cF%CFRR$IRByoPHyC)KtN^j!2DG zc{@SFT$v!e6)-5Fw!KT{iP<2=XJE&XPl~9%8X&Dv=L9Q?x)$^{EJI__Na-5j?5yl0 zER;xhP*M_>_8~HZ6kVOa*fINCI9@AqSnJU$NQ9Gn?qVbY)9{wzxif`yxn-H@ef4{` zTg%lA+hb$?(k#5gomtAtev*_TGej19kX#3N|8T3DUmwL1?PmFo{wHgj==_IVG)KY% zoM!*@hS`$<&soRsv=`$Sz23ny?~TBV7y9j^)_|$!?c>R8%1lY|9pY`AevE8$b=p!2V9N3x`!JWKK$IM0 z^9`=&nV^sVs`Iag>|!Ol1C`YoIK-1H9c-T}oLfsU8qdYlYwUarG7+B_q8tY8nAi?& z>*g06mTqi`4OcnV@_mPXJxer${`HcI%qHQG{_E}G#JKb;&Q}@2<46Qd>B*CcuG^|f zB5NN8cF*e`ued0>#()pMb9uSm4&d*wqQXMvey>WN==AXfQQs%mWla23f$aFg{b3+q zR!rd7{~6VJqb>U_?J%E*>SL!@IS#9dfo{gaLBO0ND3m(oV1==+C+0F-*}#U>S4%bl(?IhxCKsB;k}`=jNPO~P6n2Itr2yqZ z>>?ai`@@$<#BWydj1alfMIFN>dfq@r=9Bo391`h3b-|u5_`D1Bphc8F)t!Z`i6}7} zqLePSL%@`S=vSFjDG z=qZ=5!8QImOU<;(YHBPLP9sR&{iggGGe{y>-ol#Y?B}Lz4 zKAy2MgXOde!ZO%EQV)V>C>Dmo`ij=*M2+GMkv4>9xcBD->%svs!LbRa**4+ z)tiLlcKFFxo*tgzL_5C-ZmZ3i1z+1lebpOqu%=T6CMx4miBEBIA4Ui|gzdSV5$e7= z>S;hq0MQK#j$9m$>hY6y(jB}cjLbNG>~I(L<}V>2=}I>h+G=K;JZ{a&>16ezy_m1t zVUAt=RUbHGN*7_~RzHb9g8lmu^nL{k!775e+zmWZH(%2Wy_=btuFFP4=yL*17Xw&@ zR1=4j;nWy8- zNx3_MSGPHcEMo$%Ii)qwnJPk`B(8E39^U<2Hiyavs?tH+B}d9jd{+HUr#B+bNGTtJ zW$H+x#azgHl}9Ko7WRp6+hs}E;q~{bqs}NYS^+3>OZN9Xrbv(M5gKKp`$gZ0PIv0P*R5r{cKi*rj97?8mEkLQV+^xe1S!I1Ks5shvRD`%{b31QKq?K19S!N`d0y z!zGJJ!|YawIBV~oqw=ePt3e`e`=w}ZLfr}=7pVG-n#w2r`&nVY8w!!|WdJe(8yjcV zK5yk-_yHG(Jzy^;y%!R^?MCh6O!X~ejTWFIj?(|(0ms%^N|>A^kP9K{{c*v`?eeQ? z_dKGUh+WmlWT5(}o;3-=23pK0RX(*LVlt%&?-tu71r8Af{kU{lqsJDOUC6^lDe(WjWZC{UxI#3_oC zTWt&pTM0r;1}$hHpoD-qMMO~_lgrA*N2;d{WjaL-p)-6`I7uAAYb8Y{XD-fysuUyS z2)9XQ6IH6Qv}TQYCDX~V5+@5yfm#sZmm4H*Lf1#+JQ$)a%a9sJrybB)8wELJmW}J^ z3M*n3*c3;79XFRQDmpicOrUEVbpoR2K^ukXe>C%V&oufDP* zpuSg*H3^xT!gaRc`imS$;!59L#GJ3n)8B*U@*2z@I%4tQR(<3$rl+8N1_Oi2Wk0!m zKG+)w3k`2Wh>3XHqaJATU|QA1v}NFMv+CTT!SOqwZAH%X#IIQ!AyV(O(fU2utDL@! zuDN|aKG{Eg$|*|3-%8tVgNxbm+iKwILOxJ)Ag)9mAHNOvi^xH+YId*aJc2F}Pb`Va zEhUcFd7R^F;0GviZg3S~wG#cFJ>ubqFc%YpI*H9K_bE|uo z<$}?eMBXrcss6ap3jW>F>(?iJn`34|@zESAyex!U;Mg1*e7)OgsZ*Gjx5NC!w%xq_ z^HE~OWb0v+Uhi-Z{%{w}XNnXlxs@!e5@$b&7$dB|a4847} z_Vv%vsjIT^>wuV)JIAX4OW>-t$i({pFW-w-$vk1P5DUtNqaj{!uJ^AAI?VVI;Ty={h+DbnWrJkwc%;fC#7} z4+&7@LWBQmjsW%VZlRnA{IAvLlrQiTldd-C$xxs19iZu_Aw<&%1mfg>kfu{M;om{u zLpzGtNjbW2UA~ba%Bg<`wymFLue0zCp*&l=&5>L4I%jsvB z3lCWIFm~;OW-}Iw_hK}$siX? z#DUACE2elboy$lDrlS!yJ<0Sfy^WnI^U_DD9PbT@kcqLT?gkfY_bR95+*vc&3^seN z5i|%hQZb@Hc;2Vqkd@s->~n&>B#nuR#UBp6v~b!hPj@mZmb{@(rC=&=78rShTxm?EPqysqFKGn#q)`;dx-1O(+kVZwMdVm? z+8v+zv#&CWCs^<`)h;X+Rheg>8=c2sbg6eB{( zjO|;B|G}OzQSM#zhm+;%1Y$BTn**+>$ZLePhozg$*eNVB!r zQF!PQ@XC_07G7>U7qi9lWHD)Dh1K-_tZVcSRQK+0Q#8iK`GX?Jo7GrZE3Q18f45AW zxuY^OC&M-|FIK@nk3omy`SqV>?g~9zZj6Vg1)R^T#*uL6CmB0XYw!Mr1pv;Y#Yj%-Y`RV`g8KcyvT zMFeU%E3?k@v&(5I zGBECpHeaq{OiaP}*<0(6ey|nnh%-zzYr@ zH7^Was|2E|ib@iDod0XJ;fUD2ovl_q9DaI~OTO%3Je8LwlCKS@Ulm$?9gO zmK*K@xLXK_xF9k?0W=ifmkjh!mx2v!6MSB>E_S2yTRY4RqlV)+X~oZH!fJ-8GchPF z`;@~-5DTSRO9mWdlOC2EZ{r_Mc+ja6kZFhGh{s}O$8ZOZhP}B!EN(OFkduhHBF%q% zGgVqx&g(r-Cn-_1NLD&MM+`2Q1TD^q@+c2OuGQ&>JM{X{;~LP2^(XR>TY;g-iy~j< zom<_5xh=GlR;SBpzk9hg5UxQyu4Rf8kSsLQf}4(&YNtoL}|!!L?Sjt7!OdDdxM$Z23jU>2EHvX zYUXhDaXwmlc=yL84yciQM_nn#8gNE$R7{eabjwqwj$8f#K^Y9!*9<2cRUuUogw_wq z-q83VH9cLEHHCKhD9CyRDKylpo=;H(D5(d>X}64+)Q+}^W-MD}Culxt684cioG`8~ z%3Os?x-HSZJ|}o{3_(Ceytmjsc8e^6p;g8U?yAfU!3U;mq|zE5#>3xLy#>fn zB`d_|Q1GB8If94w7I^WE*^~lj_8_Ch#Z0cC{+v@KeTJKONBq+JB|CYj<=WpnQLB8d z(&37y?myp-X_<^TpFz~uy*TQ!s_@eM-QZhMOD!GM?cCUCRfrXh~zr*ZA@FFwP4e8^iDUuT) z9qjbmYsaqQ-9!zgn9dM+j9pL-j-gm+UJ>MCRS8tnOb~tKTGz4oivK0kr~a|kBA$SE zagrO2bNA*?*u~35Ntlc+h~yZXs9MK(H5Uo|ioU)fzQX^n24(r>bt{5&_%qcjG?NOE z&)s0p-xKCh3KZIB%skIUJuE549Y+a63MGBp*mA3HkWKY!ahoX*6`?W|kf`LBG~^UR z6csUCzteKlG>smL?S(Qg5wzGR)bXloWRXs^q`kPwT|ii$Kw^X`Rak$;OyPCI64}^+MUA;J3iXIr2{mS?*ztY)(=I7J>|z? zg@pi{5vh(|8EG~CQK1#h2aOB~i76da?sem(oh0eD_=kM5T|{q1Hc^$LZV1kwcvv?TSlfVad^$xgm zA;MTmm5)kSqhr&~NTCn~YPBOm8GDh0v5nWEvLY8{d(tkAseeRaBzM>Ho}QguUREKk zlfLfM$TG?w;AZesCCaf4LhKDWYPa|qA&jn=8J?UCF5nI)nvLM=y9(Fa8iwL6qy@UNr+j@xlLF}=a8Nv6{ z1=nu$TSnFyHSr4RK`Q^r*op_p#)ehH1@tn`^C~*>;1PHD|gk-dPV>rT2w{kN$;5T=aOS)zS^ zxfr4z*jNeqw{o_AMYFtF9`_~eM1nR7Te2yDR<#NBo#dCj#wXn!4hpY$g({Sk=jLszLyHkOGn3UCzm3Ycbro>>hf-? zj!jfoT8pAXXMxx>uBh38MTJI7pB^{`b2bT7k>z$1Bz$`^9gytd&i2p7?K>k0hsvT3 zJ4L);V6eKd+(74SoaWmA6v@g-a!Pi0dEX_X=pY?&6@~x8FJ6hkaaL}AnK8P1csvZ? z+bYigoY3r@PJ+sQzQRpEdWjbNTS#+qx|BSqnpTx*PivY}@zAzV*I<}d;Z6=^hm@z? zYrhS5-Kb$LYd<9?^!g#B!g;;H|NO|K{MY5+Zm_Vq))RvrepzW~acY|(B zBb{Znl1Lvzfrn*3@{*V;mDM+1z9^_MDovz7Y&QD{yx#bXX4@)h@^0~N2~jq%!qI)4 zhtx#x-6UDMU_t~$CnvdUxx^1KOn8ldQu5D0A|h_CS6?-lG%clxQju~)7pNAz#h|YY z`@#gIj7cOSjXo_~apC0cJ8~$&q(Kk#F<9C{^MLddyqI-N{JF^u$SjC7xM%Bq(0aGL z@`b&xEdiH+nyp_55rvFPyhGAbB@ZzLX(bZ4Bt}1t`DJZYZF{t|k6+1gimC{HK)hzW zr_XeW&ndnfHk_-N`0}s3l^|T15%S6hs^ZEuctb=upjOQ9Ft(1affGJQcA6r0bv0-v zF7b0Nn%8UWM*5fuEK)KDtb%l=gN~Q$iM?=f$j#BPmgXy!p!{mD9u|T<}1$_FGshy5qr|ATrSihEY5I z-S7BOfICp_cbyBx69qezfGfNN1mSTM*(TuIzmLq!9LbXso4_F@tCTMDCd=l<1&mGr za)NCWJ%2Uk#8A`D-7g|XA=5gu_UDpe`_X$#>fqkZ&m+&2GP%#O37nK)>x6!Z+!fsS zT-}-b$~eqx=ngez&=+~Aaw`USku`G&+hqo2RA{-98UOs~ggKy$dA205(xCo?C~{VB zXOIj`O+S)RA^U!Y!>WE#067Q&H6ig|cc=$6LoVz@EXXkj5`IKOKZy&LkC2(^w>MV^<-;1f|;ljg44m(6vL{WGnhoEE8-D zNwqJC;>R?SJkCVDFwBisUz}Y@3l9_+!rjoe*jfU}gpS>3cKBz+3c1)#1#Z6Ug3-r0 zX*GpWPUVm!bD>H$4EFkOHS`7NVi9<}@Q)PX18EaMB<^tELKraAd_?A0#rpOxX+6GD z6QMw1ucKU}>6}BECAk0W4c-4+9HEo4*OVO+g8oT5T(z8xZb5Z(3>GWkR{zh>kF8hlLvy%}%8n@lt`#-2Lhv{%~2fxxw z@8#m0=%XeQo>$VRJ}zu_&ncEh>Fz@cwSSQXZ&ct z<9_+mIZ~7J@l%|eG~2Lswn`-_?f#EbtQZGR29v$g{-J1Dto}(9QjKwEt(?7G3!$?@ zI7iaT1kQyP{;vd>Ru2U~fAWSFq0@}%;!+J4#&_mGeK9p0vkfKQU9MqiI?dfQU=7n@ zrhZb4eXU!POnAn3A?L#Id_kYFzI{l3em}E*7wGT3HXM3Rr=bjMl{wP+FSt+8TL@T! zMR4`LwM{w}Z6@giUm)!VhMKLY!IU2)gmUN(JiP=&QT_hK?yc%ie1UIuo^z!E$ZF(ER0pcUtE0& z@|(AGPek(`-nR7^C!;Cyz%NBqPA=$N>eb6WUG6uB^x>oaFjzsI60AZZhf=cL)JHw2 zVnc|z8gG#jouLKqUkIl_Cyk=6F;>8&h|_c5b+N1!+UP_)DyGjCuEE2ZL|_+BwY?(I z-K4$}V~ub@WaE3l|Aa%>4-w4laBuIHrKQG3pi!X;R7y- zq8=$`Y_mfe&3;=BPh|@(H>G!No5x8DUR&d^H7Oel=?hwJ(LPOAjYW^lfl!m`#wd<> ztgRlfv}Z~EYIKpA#Ms|bSxe0pRYw0Mm(KZ>XtDtzICy)H6+esn-tcV}O}~xrcAc<* zdxAocN4CW&`gCJwd>oHe6f}4ks^; zy`zmvP+25uQS-!mT#TcpeVrQ{8QCdQw@vW~$e}kOg-MWG63F>Hcp5+FlC&>OPv1%? ztB_@n)rN5n)mkON4L%samqRE3J_7`r;0F4~FkEA}6ANXsTU2{;Ur{v0l;&Wjl7WdI zX?d-FwCYExN%RQcsfT0rywDxa91(t`6+5{`Ku|0d->xx!ShdfT`%HM2{}Wg&9a}I5NnLkje8gxoTUK( zM!P}XA8n;OWx*`@VgmBg&vo(k{{Tjrx{`?<-uL&vNOF4$1ueNj?hj+Oa6;C5Wh~*c z&*BMb8O&7I1P8A@EI{6N0Q@f>vxQc@_$vzn1Aa24BH2)8w24r?uv5QLeR1;?xrxZ| zer>eS?`?oX4X%{JLs`&~%el(=PXn>`Y{HWMDluM{_HpUqQls!i@@9%(xEL#?m}?G;0U8lBq; z!TgCbELTb?EMCCIsI+JK%wW{BZ&g0VveMq5Ke8Sh!TfbUncJg3?N88jZ-G;@=09nE zSV1ZjXb2In0eK}aq{EU+WAn<}B)l<@U3z-^{0?+``TVH}C|QcXAHx~2Plc@Q^b46A zXkCJn1&ZdIs6*D7Snwk};+@+i=Y>8uO}N-EbOqUe`{!t#bwQtnFMPe5uh@zf8;67Q z2$UH7dia?tk>*w~dxuT+CCj{M_j>LnAY6>M^n}X1g#VwQG5`h^0ubU(#o0c5rAS$_n7JCUliS@k*3pMB;@~0W&eGZ!~lv81Q~eS^dDl7r~qsa z@*>OrpG?L7Z4dyXfdN5cu;Q}+Kt#mz`isr;`lV%1{BI3Li^2Zw6fiR3ntzY=_DYIX zZ1#4Pu2B33!kvIv?`oen@I6b*L38#n@&Dz}{r82zVEjek&Hp_8+S2;lIep?MW95g6 zf4eav8lbaCXf~fEn_D3Yf|{E>S-z`KoulF^IsGfaQ9}l>?2W4-D~P`Y0cA$G1n3Cs z;{LUdVOajQdft5YX(9ugAWCW?9A#|$*3%%9;a};6SV3G8Vzh51b3M0v*WIV*0C3X&51H~Gze^T}rpAyVgDa{0 zAA3@3+|G3QxQvxF-?*A0* z76&?YW3#&XUvCj;B~b<#97;to1*QKXILrHY z?w0D>Mq~n`tWmL<-}Ylg?O%Sx6xd1ewHflVmTL4lLzEBw+qm8Yk)t7mNp8W(rZWSO zO5=ppgPY{ThU7P0wF=WOo%Z&P|MFzmme2#C7Jn>8^rQ9Ja*ZR+v8Z5pwrvKoESWkx zs{|NNZw7s+*xYPeT=~n@{lA07G>{Ap4yvlsf$U2g|FWK*UhA!)Q7#ohyp&{tnxy1T zL6(4*m7Dq-SYLUOHP9z;#Va%A-&)=iXyUPkfg9#N>V@lh63drK@qtojEzj zS?y0SJu@d`ghbtaecZ%6xz3y7_K1DC=3@!s2$4MhKdUsq5$@p?d_N2aci;0HY>U>} zI33IQjd#}{-^5=Fir9!5?)>G|C!^uBEVE@1$snRl@O;+kMs>6AKMZqhsDsQAm6~3jXv9} z7S%n$FhRW7GFI86Xnz=)EL_)yJ^83tEoeVDyOzVHUxsMnWM%cuZwos$v=*XPgfpM@ z;*cGN(iRpwgf5-xnwlPfS{ZOKR8IIj>4>*owK)H7 zw;W6E?Cb>2n|->3XYD+O{xWB77gXyEzN3f%V#p;#mR2%B!4BEpBz*2yYCK*z1M!1G zlGy{K+_wv+e>93B;c$m$CHl*%d- zuVl)iKaQ_p@n_*(KJ|ZAP>3!iWpCSI$WIUHC8guL?JX~~5c^>6N=f!39=vd4gU^3u z@TJp`mSphFsKwk#-c6ki4@eHl4n44g#LL8f&EJa5gL@Y6J)aCOhj(1%yAz%QRS1F6D!p3)|I3p3*_NuM}Rx^+N$+owrla%q| z_2~6PEX%v|-@bqu;>!DGir3`xZ_h4ub%Ntm!V7@v^Yrwq2f*QVIQ*&wz$oWy*mS$+ z`W>H`*Ia&9eweo`{gv}+#m-{*<2m>Sj^S&Vnsk4X8kNLtn;08jj7$)9Oq7fx{hpO( z1(qzXlUe@bKc+q^aWS6@WrI)Wg06|qu=*TU+YhyWGKH$|PlZ8FBelm_J~1$Qkvurg3l|ko9$U+p=GrtQG_#zYRB^ zeCOhU5i(BLD#phbNk|`Mq>M=KBR6n3J8@B8YMA)b&xlB?QXYu#>41IZSLuC#zXG%` zsYOizKDZA{_@94}WNb53==Bp5W+;6JO3w^h&Y4IPT-eI8yZge!!;g=Tb8>Q$lau$K z1n<8D1c($#iU{gdSIuv^X4F3N_B>7jX4&v~vw`s60ncZH+~?x=n|&d$A$_KHcJz{8 zD$$~$6?lJt#8fE*s=?-bd3$UXqwc9XJFIaRVv3200i36sH;+D5@gILq{MptnetZdT zccJ`s5BOz5U<>a66)KV7r_<3S$_P}_`2F*fgZP^2YAIDhLYSO zyO^IM8%xPN8qivo=YE3htPMuQ2nyUFu4b!m-T%kiVAcxi&>l$e+&&q^WYS9ecr7PJ zyIC|e2;jDapku*Q>o;DV-D{&A&!1eeRe*<4@i?ys0G0^(HANT$K=W8tcfZhcbq%-=es33UyN>}h~8#?m{b2TH4LV+xZ@t0$HZ7uievviTxp?gQUgYI(T zf=Z^a`BEa*iTy~dbKZQsFn!=txgtW0^&JHShghnrX5108;VIHM;p4SiQH%3Nv-b(h z64{kshc(oIpih-7ssHVwfr~IuLy83OSzU!v*{8fU4J9vPe6+;cZS?$U>s;YgJUBMu-|aCRctTes#6+!wL|Vi*LrXdgL?}O>&BDZ?u01?f z2%bK+*nj=6P0-``A4FW{v9Yl%E%r}4X~fHVLg#{U2(Lf;#_b=dau-kaWL2`*P%v6Q zzxVAi{yJOg`g}CrO#A-jZW2fzc{oQ6f*rF{=*U!;#1(Yg%WAnV?eMr{mQdoq5@VTeQ+QQwP;@jpJVLco3 zC3kWF&PsA8>&QcEQ&TFZiqR-6cc>yts`I)=;0H$STIytHlBBJJB@G$$k!OTidvuD$ zwKB|TguA2T6Vc(3;RgcB;dqr%!j3Le9r3NadJEH#q*$ylglkP3j34_`*?<2ikiDY? z>P#WA-DJPbZ&=;Z<$UV5xKxWdyx-{>228Fan*!K=D@^2e7(11{NhM@pmG0fwdIFKC%sUX61rspy#** z7T;rV0uy(P$5*9HZ#Y{&O19~628F5RM8~mBQwR1yDi~Ht-Cs2Z5iz9AQiaGu@Lo=? zW?D&zpOf60B{6z$OSk8%DdtO>!N%0h2_T%IO}3$`%PR}^Up89&dp;8oZ~XF{fra2n z-cC3@J|@Hy94FAp#J3CRGxd)~hma713ylF|{yF}EFs;rIQC7dx6{RZ|m1i#^n@^5~ zlhS|s)b67T&Xnk4x#8j_(uVcPMJ@fhK`>tp&KX{9e`{##mZZlEB@e2D-g_U8%+L&; z2a<}2rNC16BZ1kg_d5EAOOP?1kf%s&V|&I7fixYY{*uS@uw&+ZNA|O^eS%MN6Z_ol=;|$~)@o z%s>Yf8i-~K3&B+g;+847STjmRBOOs2yJ{=S+=n%Bh)64v_RicvrHU_&W%W{?QjYy? zIZ31NhoJ2O2U>cMdwO_*gh({E&h(d^<=QbL28xt1dlV{?DJZvn>O52BR?RI>wLcqe<-Z49{TXgC_;B z7kN#Eg%sBi!pFhaSx%*i-Wfu8qo7Or1pGYXThpi`@%cPb*8pJMUH=WtJdKzLo_ztw zZy(2*H<5Pibh(i}@)yX|J4QUbJ>btz9rjqfSyN*5&91}3n1;s2sJ~Q?a%Zt&k5MQI z(;i@n8~GU~L<3lAZMxQ0S3}9px8t~xcY#{W_uhJ(_ngUi=LZ6sPx%+vaN*hu1g^1kQ zLkit)>-%XfNW+n*#M_$dZN7b?coTcqRW5}a?!`d-gj0`m7+aCTF;8j2nWMzNZqyvG z9L+5+AJtf@dVMYCe(*Xp(Lb)MuWhWa&2R1%KV^Da9UM_K;$lr0JUz{``*iO_f=etx zMg=hmwr&iH0EZ#J2q!%vWu1*(a9Qqf7$0YpljEJv?kp&%FD$I=EKKfcaNo$Rn5kNH zz8w$6m*Ld>D)DB$C6UH`V%bm{rW5|t>hM~E3N6rlIgo3Vgag;ak39*EcKSA5=+-22^WdjUMl;J6+geG_^dqe319;M93NX5Jf75sN1Jnu)Gc$5)&Uwf&U>)}3 zyg#^g*OVt+XPvO+oYg|4=eC4ZuV>jey_WO04_7a0*A8v^hIVYS2JZ#YC zb#I%#U}%*kKryErBAS6GhMo^Q!z@SNX?wobX6)?+Q4NVdkRp~HVTHWHlz|U2MTHu( zKf~>L2m?m)(hA@NJD5tpgs08vZJ>*MMA%_P>6N(bG*^qi@H-IHLW9umQ7VuMEa4Wt zicgEEl2J)e9dRrJd59l~CY=`q#@>qG9G-!H z2LLtD!F5>%XAp)SfjXRpCFVR#>7{XEC$g@_bY`V=OeiAwodu(wzl?0qG6M!TJ;4{R zi#9o-tf2;1sZrTzpfhYpeez>QFPDR<%@!6Gy2iZcjlaBlVg!Q~=NhodL?@6`K~IN9 z?c#h0x6Dq!C#}54N(YTYOsrjfxg@>fZf^br6a?Jw?c<w{Ssh=%8U& z)wyni3%}c=1LFcC1Wq#$Ud=+d^5aFV;r(L6{rlS~|W&VC8patlY(vMDov0i;E;9Zbj9dYYtCBadeGyu z=mV8Dd5@I=cvttx+g<ocq;ZK=Q%AYipOpcl=3yC= zggu*{_CIs9Z{of7K`gTd6YB$Jib{b_ts0*n>>A&3%HI4&84u3_=+-#l*jm`R|CAHc zjE^%sE=!vtlLj5m&HIOj&K`uv%|?~6DhjcPe*|p*Q>Tl8)_$MguSnm6s$fmfoCW-> zq#BB&`M*Fk?02Bs^<0Lr&brz75zALgPBzg!jn7Khg83nd5<+bPER?eWR8gcoi{Atb zkSCjz%YXODn<1zTg)p_`$vYKAh||XeP|lrKGAU9>#?i>mYDBN5X}*1`k>IJpZ!G5= z+Hac}qO=vRR2<6*z1u@E%TExrs9ed+JD^=)`>}_Ud+PG#d6iGK7Ycb)EZVhYDPIjk z0&Mh`VQHEoFkcCI0J80js|r5-z`prQt3@}5o%i* zLq?MBNWJLMu@%i6E$*evG8qW{=`nca^AjTzvQc6)K#{D#Tzcqf)dzSu&fn9!IK>11yW?0)Zr(e$L|USF>|lK);GD9qc7Z8jwPD$T%I z9Cf^p{zg>ipDcj0g>P?MD*7TUWW)O6t<-w_X|S;)NQhHx7t3T2q^XJMGkDzQo?F-s z^Z=X}HDzTBTaLQF)oaOB`g+$w^p>!)!;e1YPREuZDE%;7Jlb#2VV2#^v6jN zDJHN2L8-70!0x#C6Ozqw7%F%g?zp5idQJszI8_F88!ZMY8B>y~x?X1YmJ-$?V+9Be z4MHM61WvaKLtGI%hqZDR$ZqtwF=E;m79A0Cp-hcs`L2Ducth(ZdVSO@t=@nz$W9bw zpq=2AsO(C6NPXm8@ji>gb|eT&*2JA572wMLo~$ViRz()wS@M`5|0^znpiG5G)N>w+ z7Sk|CC4t=!0TC$GJ&UPh_^EFjGycg~Rou%z<>>WGfV)9JUn_f&3=-*aMCAhV5hX&O zVgwpol-PG~=&{_=K@@>zC|#cYSMG?Oz0jtb61n*m<{PZ7dE^n}J3CM668MI`UO zR8wV@rSQCducJOpJG}M|a>&xebTNajtU58ILAgx`_y2VG&ZzN5iFpS0|<8Anex zFlyxtU1Ev1s1H#?vpb`r4z}4tKRZLz?jPUB{vFlu6YT61? zeakCJcndH|lCFP|)oyFM-WRxQ5Yn7FL{=}kcuy5{m@EjIe7NAkXk_wk zr8vUhadwyAR96!T&z2Zq!oY{{3P>D5l`J((ru6&D_7YlKGsZ{V2Tpb6(8r$f3>#D* z0IKMZ8(w2bhw6R;7O=})DTCX{IQ=-FP?jjfS=NyS^8t|p8CN4Jemag<&zJHgczLhK zz4WAy6==QwyL^yWfnroY!n{Ka^*#9>6T!JC!v~@TB)|v)@00O3@jfShu0v!9E5v`p zi$YDI3MZdtq4dO)FIF}gl!U$ttZT2v$C#ux<}<8Lo1NzC$UoN>SR&Ykb-^AZrz&sQ z-50lF){ZP3x||f_a|)!chOv)sAqCJp>*6g~L06d>Qz?=RuHn6ucLStvXz5v;NCc>3 zRDzfuSDWz4)Y!HBY9#`fqy!-AwiHKUv`eZCm;jVkS8qOf#FiZIVYZBp)fOktuuIkK z!%>ccK```6)346qR7rSt{X%RwwtKP5hY`HnqkD~TD{?AGBu2+sY}}DZ-`FyGqexnX zhiFZhLPH12+kBsn${+>%&mj7$SH8cK3B$Lu!YWMMeHiBa%ZO_!@J7okg!Sqd{jcxN}kZJYm5ZocSySrN;xI=Jvg1fs0cL*9J5Q4iCoS?yiOK_6y z%$s@pW@dlyPblcF>ONInb&JMAS#g%;C~V+JL_k0D&Wu`LJjsL@0tGKNEtA^H=daq_aIFoh0}UQ z^*vw&zMGIl5bO1M3yr`_Tu6yiOlBx{Xk1Ha@~5t&^gc>ts!;f(3~UFI3jsdFc#QG6 ztkQg!ci2woW!X}Ha`kJsVfQW+5F?L{im@maWZKiW%J>c&!W6B?X>gW6Gb9}*{6K(Rv7h0d- zpERcGy+#_?93|t}CRkbQy!%wMB<-{*5kZQdJ9L8zB0pASO9;UVaf18%JEgPESgnaT z?CuXKOTvs8UbH?p(>mD}aTyS)yFio2W*Koy677N)r%U@JA_yh=lPqKiK4AGyOBLh3 zWW^hYC)c=dn&OqCrygokEq=b*>MRVyxqlttc=onmnT14+)_@C3==zt>ycu;0Mp#gB z@*>l^24uZ9VPe*z3ha=EL@z!U#iC;PY|dh5(uf@@Gw3p%^6Dm?!C3_!wue3E-N1w$ zj>o$eTy~80j$4nQm5L}AG%sj&-$aqKKds3sX@4|O-Eab6XE04tjNlGlH6C{v60 z{3!N-rUTPL19?f_ht7&+1~ga(kw?+RTEadWKqOyb`6n8j~X(w<|f1_Pc0;c z3Yq^nI+02R(bm+~nq8P|da2s^y*O-bPwRrcU0e0w$GyLOp+TCT*SH#b(4kG-Dh#($ z+q*Db@qV@;JkU|`vwNuCJFl}mSitU&yht(0=+ca35A?%$Un=;`Jt{i%_ZM{0G?zX@ zysb~+=v^J&q4_?mJLQtfcfvQ`?n)WD~)l;l2 z=^7cd{pP(ZzC@K=XBdXjA;{S!PdXZw1P(%P+@LS z&(tw`2Rbkcl4vT#aeGm$h-lfY#x%3{aG=FTaeF%R`ijO6#6nkgDKCtf>Lv{zj+BKe@IJkv)H1#h=D6iS1?~@4%3k&0$F0Fq3WWTpxCrB_= zAbgZ%KqSB8lWzZAXN2d>ne#1YO9wM#r@>|XHYaD5y~H*02l)0U%2dlI$VR~B<&DUQ z-B649Eg4_C<8G^$POZo_RqO?qFRp-&oMeo=J#KumJ@k9N@#4w%U~Q|573&WZ`!N84 z%3en1(PPPg$8yfwzImfiazQ@~7M}oc%3KiMLLFya=l9BxXexWBiO(eC7#xkiM z%EVk~E&o2>G*F4d;xl>kl5SFJm4Q+ToC5Kw{M&V^{Q=~m%pV_xQM=cy;-HwHOsvT5 z%sI5dXQn4`pSXr?g7W*|w^hK-mZT*|79_TcWAItd4Gy!r4v8u6%7pafDhRRDZ$jsU)oxq|A zmqv7+_rM5kj{;kfPeZE_@A|coF$YY?#WZ=OP2yOt*PjJ)^u&;J2svaav#G41?2-!^ zE8q-jDni?ZNy(85_5GxwV=X~^AY>>_n zEEL{n|Ge>h)3zdiNT5vxoKJE#2ltJiWHFwg9;CP@8@v4ixqDtJu!rV9z|=x1F&W=9@ob=V-QZLZRZRP#L>s zw0)CBAiO2b`F>;K1!+7xc%+z{E}w0>Wl-u#;;EoXwy#(l;Bty&Du-Zfus?Gv3`)MqK-2+C6`v$E1aCOO zr}iDPVlaM#Ol$}9h15z5*W#L3ijm;z2^+fSzK$%$=Up}BwPQG9-KVrB6JqFW61#uG zij9E;9DhTv0pMsz1hd^k?faZ}ifjuRFA!s%?bpb&p~dGf&W6c=VE}<;ZAMzKWr2?u4^BYum~t$|g$9D4zSI^L%*p$moX_l85aQ$U76CR3DH=htoaJo=fu=63 zYNd*N3dmuL|Ey${vEM285aMo@Su>JFd#pqcuWr`jNO3tco1#!2iF`!#m2$A6f%OQI zCD8}pBFk;Rv%T!>C@7Ll^Q+5@WQAn z%sP3b4a$WFm2IFp6CYZQ$mIZTyHRkIX<0@X8_Gkl+N+Oe6XaM_9Vp()Af=~S3QV1- zON#!C38-kGP=kH|EMbBIP7@W{w2-O@sVn1BP4%R+nLnIy)NukS_bU^+uYRAgpR;y% zmUHy98kE&&$U$LZfW+Yld85yC4O<=?)2x=Zb~8pZN(nzdyUT3I%BXxa6R@-Xez$}>U2P0V z81L<+>(DEV{_y)MNxDOhxusJ0vEJfXcbRO9 zL2^~!nt_>f5ON-IlH^+?|5xt9@vgd|gT5&zEizvTc7{ik!ips0=BZ%tO{Yk!3PUD7 zaQFsEVBF*MKg?jwo>^dsXW!pMwo{n%zB59sgVhM{Le#?^lBZU(oY-N=N zZKI2v?p`;^LaKn~-~_*c4Uw}p8DFhqU=3c9mdaJ?wGfgHxGR-b8E!OTCUMI@TAYHm z@i(1md9Zi~itZ~|4l4ZZ?E$QPuyxOHEL-NiJ$qb?n|6<(7&)T6E-vRJ4C>B6h=A1N zyOH->$L2bu**PkLk{@+#Bd&0!p>O;62vG{@czKr=4Zb{`b^26xndyc@9w(`57`&-e$x#|nxtue%lc;+P zF;WLokTL8L{VQPq@UO@H9KucD>1ok#FaY;F9HFX+*D_N#`uj{EUp$8vL9H&gFym^Q z`fW5~x9#8g^ts$nfZ<4 zZ(qsZJBF?%^$8iH|3Y^sg^Lgh$I~hOM`BGGT>V!qp@ ztm@M;g0+FmQAa`h&AQ#kcIWLbJ=k43Le@ODAqF&7kv}cKzK^QIt_iu}>*fhGbz1be z6m42C{xaLTAaG}xMd-S#Sw+Ni9}ko-yvW1Bkie`WYFRYD1?of5tyZUsH1wU-qa!)b zwch6JF)juQ$SeZqRGEhUi_5d!+!qkjV46%L_kB81&nf)O4+l)EGzSLGXI0jDpKat+ zGR`Nz$0MEk+kjpGc% zYb+Cdc?)@^_|VJ_W2mt&IICxUD*L&CS*rJ4l7+%o5~y!pd9V>f3oItY0mPn15o0ej zvd$(jQ2TMzS^PHKyhHl)2w1HP(C*ACl`RP!hvUT-!8Pz%#B!6t7rqZFrXayml_4lw3hryS703uFry6X z2n_OuxtM|_n&;3{z~iMo(-c8c(^J!g5VdvReyV3DO^-1TP~6{Arz{M5bvX;WM5&5> zdo5HL*6}k-p?;hcg5-djBA}0S4l|Cdu^3%cdZEqJUue`mI1T#) zWox*ZNEUy;@sZ`D_w&Rv%tK=k+XctB6(#PTqG81hX44XkES@rxcIK?Ym`L0v*+rct z(1lnr>b|OB{}%@y=9E)`pyaB8xQ@Q5tYsUSU_=ss!|f-9cdi_vwQ!C{8pz176jcKwdRglt5!0UfP7mgL@ZgUyrPIIenKkg4ZKG7k&nGF~(u;Lw6M zG&lq%PQlmkk?WUG@Km67;+^sHIMQG9Jh7>C3gai!Q4D=gZNuKBx%6?(eZ7%WfM&D1 zI_&2x1xSKYO<%w5iwUMVL}CM7bnYO}$xJ=8|inH?iv(=7G-cU z`OIYB{$@T}eVnMQwUAc(9J-+&=HfO>B?|+8VS;5sPkX5pLJ4+b+eCixI~MCF2C7-V z^O8!w0lsH|xRerwB)$=bqC_0jwp*}EA8bsW659O<8gU(_=$&sEAtZ^BERNHkdiL*bOlkawZAk`9f|27hte zse453*<;?g^{LpRs97VHXN1o#^X*!@%qGn%2G7%gDa2Fym82p2Rva6gLy|wRI0Y|v zq2xwTtRPAWTcx2PGB{d;5hldr2Bp`shv=HWLdYl(CwE+EWNR!Tu@3VG+R|B0AS@59 z6?2(i2tt*#(^s6|x}igKFqDlM_}q0{)CclBFA#fFbbrTB?vO4vc7-qrHt}5PCh~VM zsTLcBr}IIj!lFrTl#@bU<*Gd!HxTn|oS-kAogg&{mI?b7xk`@pLZ7hoG!=k4nP;0#YH!NWmFI2T97=xQ_D8yy?6o-79fI3Lwx*_Q+8+psf zozdd-*ZV+dT)c1FxX#_&Tv**CJNv!4x#{s__cW0E>x$iXq$Je_8zIn?WZvXLnXN-1 zhKz7JGP^ur4a*+JDZl~h%R3>VdyNJXI#L5TYx1l8q=DCxGVKP==4NyjqbI$e5b}HM zGZB!tMna0&Jh6=lLJempQ0djk_fuS!7RVo=Ulk-2&vu}x@lk%CmgF( zVKcp9@Cb(da=b}g?R~K%phUE`qJiAw<3+h<`;FJ@?{X)C~9RqnV*fH^|tgF+z-)@Dkk()eIpc2f8Ol6FxfQ7)5ADQgVWJV*s?Sx8kA{jlvC}6 zJ}u;7_itd49G=H6Am-J{$OrU54ke2{*dgn2PVjN7buM$CXx3qN@`pabZh+{&L}oX7;eO>Y z#=HD^TTR2v?41G!rv3jLtguUk9x)AXa(PJ#yVCVxQ|gQ(zTsj)omW2hv3wB>She`Q znm!oha7%UW+=gb1awiSSA~R^YgIOD)9zaU4k1OO`;Qo}`dLpN&oDgenLl&?yBXB}c zqaqHfP$ngNOCP7Z6dh(aTfoB`RmRyZn}wYc_WgAxHTV}<44;EkFZ*G4XSb~ds;Jg0 zqY-G3Bx1FRupi;k!uCloG%BCxeuilaUElhk=dj=T}OQx+0@4at^Kf0tQh2u%bY=SyAQ?8=y-BZbK~rqV3VQT;p^d8dWz z=HjM?MJ}Wzdzh*6F?L?f@RHx-#=kkVusECml~1lW+*8zZAvhi&4uA5X1x*I9=^j(i zQYnnxT36VU_M4@q9)X(ZIruFm&h);4WYcj< z+u@fU68G@%a6C!Id~(xuO?j@IzR5rk+iLC8l+y=Cn#>vEDK3Kba5@kdXhfIyJAD{u z#2NFujusZd)jY&!`~j+-l^U!Ul0z@%*%pN1k}-Qp93Ku22U;>`o^VJIspEG0Zd_=T znQK;lR&WrF)CW1U+BOGM%DU3>Ny}hFCc=y?A+r&za|>iXv`G`Mas058BvS&Kk6J0j z?kW>-1c^)76=apJwxjZj6$NuH^2;YT>67N6WqSn4GNZ_;Xjkj5P#0U@T0Uf5mnbie zDM(X4*+^SP^y2a2?2OU&ePY!b| z_E=02bGAMuudeL~;@T4(dTX|DiFDJ$7|Bwo`Gf1yT=8b0&Xkr@0`0Cdt?`%y^*eOc z8}GeFSYZ;2;Z&>9FN3lWsD00ItRk5_DIH0L4TjpD_L>z0ZX}bTHMX+4%E8`7>FRYx z|6{T2%3?VV3b1wfdnimK?_mm6tx$T#cwpIblwH04nIk|Dv70YFD{>-#BgfwS$ObJzHnFWUk>}h}%+)KJ)b)UTXcTE})Q}RpMDy%Zb6Lt`>>d*#(iO$v+ol

    eQ^oLn&Its7lFO|kpV@|GmIg* zx!3Xlu6(jgIwdQKh64)XLnc-OoiKc=c3-X4;42wh{xsfa!FTNux?R3J{Z0VPzK4dP zxKBQ`Z9$2V&vmegFDwASH5W8 zZ;g<+dRtY*McWC_Z#civYT*4AVt*(BT6YrUOVn{hKx>1BwoYj;TO}9;Uo_w5t|W-K z`|W+v(Swj)0q9Vq;Wj94V38VrxiizCDIdLmXi=h-E+4A??7jkFLfi_ON`gw=k{rRe z#`t@QKl~2#4Fkpd>PY@tp=^dBS0X z;J~qBp(+22174~gQjLb*tFd>U7E1C8G4Y=Qs8(nDQP{FFRRRKbd#_MM;3~=xKC(i{ z*PU3T5uTK_tH~t_?sfku;?womMTkwZnEr5$fg!_2m1?cTB`} zbqUFxw7cxuTX-n8vg>Br2B#%^Sn7ohMf9~&uhSy`Yc2U548t07)AFNId=ejj;`w8u zr6Ag-9ZO$dqa#-UertuSs+BJhn!Z$JWjzg=uu)BYy2Py~3S54ty;5cteVIfdCsspI zTy2(F&9`<-69LIK_m(Ial0*+Pe@QM*PJNflZ6ON>qn}Y_%08F5H4V~bpDF(Cd9dK% z!LY5uHQRp#_5+%6q{85(%!^lk^7BwoBrNoArNJXfsP_hIE}gPeRho2CXu<(g=Q9rM zK5i=)VGC1py5rxe6ohqnitKJtT$t`)lE4e4-mbNC*4_A1 zIEr_c*>~$UR~)#12A(k%M9kduNin^O4HAXZ6u<=!s+ADY{^NS&Ac2^mN)uUZE#H5? z5pY+r0!iS4)7sf!f`7jea94LcSoPin%F3I6>%re2f<+>M3l`LHK$$`Rcsc*)FAnM9 zibGyWs~!D!=L0Z;mqn}_!DoS>oZL=P)xCdUlmE{M#R&gCl<=)v%!S!mb#?WZ5_-n) z{~T!~HWvQ3k{KenhMu24SF^IR7Q@_C|KlMBcSisnG_GlMY{@h&Y6mef*5u}JNwg5V zywUkTeFbh^cnn|SYmnACge;u6q`#bNND7sW!3LXCitRyP7cUJ>Dz zq+5MSqW_Nv@F)cLbLZ}X@#H5Qm)*IymtDtLcUdXc9JrA2a@sQ3COI z3XIcJ@W+_ie>cqoO}Fmo^zZ+>_y0c+tnvGYlK+o!xnsaZVsO9O`BT2XFFu)U8Z4X?{7d%R2&9Q_wCz_8E@pPjGP?2CD;O^-JoE8KeBi+wVPcw^ntFN&cr1cURO}Spq2+^v=n4vPs><>Lx;lDB z>iERCscBr~{-JMq%0pEXQrh5QH<8aW5br=JX~2?!$j6g)?Fk9h-?Zav z&fM32yO}zgZ!!g~EF}C5D$-_`9`6&8bI9#9udc5#Nrke~)6;WviLIFCUB>f2vCO#S zW7{})bc%r#eA!UarzOKLXJ-vB;ePg}FxdSore!H+ayQ);V-KrU(h{vm5u_Cft$KcP z6VJ_QT)7UnQ19=Q6qg)Ik)FwZUJfJzmX8YjwcHp>GA;7pg#~U++P$)}a_>6xUz;9e z_VTK#b@lbrtjqj?>xbUuXP)$*DhKoz4P}Xk?%?h~g>i_9;al@F;_?0g-1+8%qFL2v%mW> zlR&@LQDgI`m!rH*IVX#l&`)c4!x!5^OUFo@*Ie#)3n+!RwArZx=_K<5(kcE>PT6=O zt9*g96UNmy8m{D2!Vc?M*j*M6JH~45ogY7n>_?$zfqsn<HB`QHFuUMFL(1W`8& zKXGGcW}aHUv$NaX*@+vKDjXhBZBr))CnP5R`2_&{09fo-UB#Nm%|9=jQW{~%afy7< z>~qA9?XWyie>uN@Sr?pJL)gbB5n=W;1H9pnj}f0AANR|xS`ihZV(o~&85mESt6}2* zINHI5FGIdYGvi&ZW&WSicbpL4__t ziu3|oq1u}IZ8!1wNQ&beoO81A5F#?N$%EejZ;u@_YRSdT4Yy+^Ror``avLZJ@^Vs5 zt`Ky?<}*bLe_yORHW=^J%I!|`V(8K!C52O0ci+n{9gn2xrgLCjmW~wJVG-;KHz)v% zCI1pc{5GFRO;^V*0Lm1@Tlyw4&i#w_aBM0W=M_x{9Y;r&j*5y3V2ZROO_Cq;U>X%@ zYcE*h^_p@xe$pX&re>$!&qU2q-bw7FT>Yvw^lH*p<& z(!#H;F-J!kmp@qp+zS61=^o^FARc=8B?_#E>m?5!`D+-d9+3#k@`}N@SSymggg)V! zz7ZKb$R$9;xdY5~QD3)&E=H5W^gK)0?H|! zK+fA)VMhKQpzn4*{;UC}2{&LBi<3hNH{!O0gM-1=la}S`ShUD|mt)E?koz{ZvEfK| z*)DOk&GlpTzNE$LszQG-Jk4vQYCbt>DJ3p*Z8MyloEOLb$E(MP37bv*I{f_4H2>Uf z6*6IMp$`h$t@s^lXrx>ZC9Jl1Al0AUS`DCK2ov)<7Xk*pr@ktcb~`MxpJa9SS|i?iNqmQ>kAKD(qMhR3zwf|4tKM}vx@2o7!Tb(OGuU8a z=A@VP%Pj_#yvu8=LF;yJ1Yb;G)9|H&1+jrfy_>zbc17=+g06{R8Hi3*7 zmY#(z>N!RaJXD%{E4hKDJn{>?<+Mv8#2RRfKC2A?ni3@@#`FBN-AKmL3=J-M+{ z0yVD>W%8Zb@eN!3bR!1{i;B>x+abcop5ES7qge3}E@SZPG4am`|XgG3o($-`0DEo zUm#g|6Q6sXUFS8|$3LtDK|8SD9OouK-WT`JpB|C>-qD#=>l1yx(abCS8lA!D64tQd z=g+rY`H+6W{L9L?hY+N>Ni?;Ug-nPEPfJR2o9wq3-9OzPT7(_%3AM|P^O0vXEG)}8 zAF^%-nM;?ZAn=G+m&CVMk!+`%z@ZEf7kyLrR{FXHpcw!*Z3LesntvrH`h$wYRsMP@ z1N8GYz~_0W4)A%N%H>U@(U{p(Nbs~f-}qAh^|ye|Bw#c7e1EwZb&-Dp*wE@XTPgea z_&7SAu@94&iwLI_;+i)7wUVw-&gBVsbSH@D<-uqgf*n!nN=w6|>j0D(@R@XiMgf=S z=H_O3m5HmpaZ^EBa~T|3J8%}i8?7k@2-sZx2JDnI{aH>I0R~y5tTEC9N~zrDpc(UH3Vd)I-`cQP`FgVc8@ zxU@;(ujpadfG!0)01OMC;MalD2~?Eb@1u=7TxsA6x$KwO@KQiGb8~Z*;q$7ZuY{v~ zPA_SloR&3Pu#~#LtWvZ7bGDG1fjmz)Is^L`fhu9S(gYLZ#`!$1SZSZyq@ROqFYr~p zoi5-EP28_t>Wk&3;g7y6_uB(DA3vnseDe1bh>ykx{t!l-%}{o;{))bwrz5*wSFAsO zZR+!jqdWK@0ovHvUivvlk6-pu&zn3SR#QvcP!k{^7gS zKqsbE{SY@L2zdMeW}-AV{S4N1Dl%4pDfgzbflictIR5xOFm1!LK z@Vads?x^?`9fiReZoxgM?Js9>UPUk-rY6e>5Q|Xh;B`JK%M<}@iI-(^pZ0&e zC~<%BXDg;W)Qh8kEzXftH4iBTaT@$bE2$LHbpCl@5gR~WR*ZjeW!f+vPqA{dQnm!+ zoxdPI@>5q;Rq1z{q7}@}Youg;>1?g^CQD=G=H~Z0kt6zP1&OclKT?!7f*VWCKb ze5C2O5mSrLRxh-)=uO~>`!4o}aKEk+jh7W}V5+mSY}~zkZfS8nP6!zqDsL=_te}2E zJeN}2fYI#hXrXK_RhG6~Se_NZ3b_lvc#_NIb=cr>k@!%6_4baF3DIf6K{`7}1Z9YtSDWyB1r{3TH73!uAQ{B1G;d7xRsc&ew0LYeT7I7_n zDC$v=>ZHi#Y*_DGAPj{L@WbYMsPE1<6NR*}pp_U2V@&m3P3`U8H;1!JMt-kP(Dq2u z@b1W0vLTmBwNC)Obd*`4E9d^E2h?qSGu_n8$2biVPo#BKmSO4+Ji3ZBuP6I0pG7N}Z6pqIAZbq!|R5 z*f)Lqre*V5=pQy4zt;e{jjV>HYPSAE-enXM@TRhOVx~@y^*Gxnd8K~^$S^?lY%!hZ zv6=f3-Qm+wqAsUo@lKDnuBnOWtZgS?D2itrruJhs!}=#{zir11SANfZ3o64iZHOMr zh8AQDS=BXvs_-va-@tw?B`^U%+V{;S<|7S^i3Ajm`eCSyu!bZQrHDZAqGan zq<$xFzCLP)iQYCQia)4@Ans5Es(k{^8cwst(p)8gAdE1;iiXypSci`EM%CUs3*D!z z1XVe!=t$BSr(t0(3`n>}&Sw9?LW~u~B}MM^yP1`xQ_cxvqv?zID}zxX`ogfXGNOAU ziLf24CMD-&W=1Ix&DAa2^J*X!p+g@G+e4j~oBNlMoxY$v3Mcv#Ak%o+P+USH=J7e@ zki7=cQ2V6%a9k9-0kEJq8sOyp4UM&HW=t9i3nKL%W2jgMq;6_eL($p9+|9OuXoSOW zxe`e21G7by6@6skNti}spFmlE(`Vg#)slv{k+w`yjuZXAhazT>_vIxVBfr{b-xn(p zAjTa9Na-p0nb9^3eml3bR?S_xW=Mwqp3Gyn-DsRI6z(d2Q@3&P`%&3WbXmyi8=FXd zNrT&_Z&#WNOcUq8ejP$rQ-ufBRJ7v}d#e_8aPhSHM}$4j<9Ud;I~c6ON&9VkL;|I)&>GnF zjGQ$WyfBYts|_!!pW)bAL#RrThoXcJr4W~(4)Y%kJGZ21&9f2nw(Zqq6{U^v@$uWg zL*#7b0drRLe+<$6EW8Fe-`75iTHbsX7NIWzv&B?vYkrwGet#tlqhMhWe7ILX3p zQ1GSdDuDWNz2Ar{TG6Ym`clyj($4=~-26rqqop?+D(QyU92$OZ>B|*N8S$}?fo@84 zFU*IofbS0AvvhCeX+nl$s9eTd;c5lm3O(br>EmcUP(P23v+$X2uM3_9dN2I^)wQH$7j!LvKnzr`}hO>Q&llzvF`HE6zpfssFe}Ou0nF zFM~u$bofg0z-@pW`XwcfNj0ev11o+QN8=1yOAJNA#HapI#DZ-xNQDkSl`%H!xCBIqZbFbol{fsD=+p4gO1m zCOvc_zh`j?C!gjGF%O(JQS2fpMgH{Jv36hWa~FJe?_JlU*gh5T^R2K#$2!gbMOh;d z7Q~KT(Z^C0vhUbuJ@|%;PI3*j(3qU}bbA`Av#mi@$BpGIBoutXWJXTc-WSFDab)!a zeefEyUK8k!sAbrlGKh3a2G3AtKx}$sl_Bz*UY0)St-y>@f)eucAZuVSSd-*CvNp?T zEa$P$wi6e+O+VvRNG8sH99@!r23jgLieL@1{IPXA7cRzM8qThYMV%G~Vm<)a`4jyH zOWT{frn*`!L_blY!^|~SX?sJ7J8a98k?AH2icBT5Y|%~~q?sIqOTNAj%aLXMBQ?w) zfrN*-qTO5c7xHWBytJ&INU@v30+8KjzrA9Kno3nT_ATj7d`qGzA z`a#4Ju0M?V*g0+Y#$t>9+(dP>4><2bj}4do$aqF-{w%JYCq_pLI~S%wmC}##_4K>s`n){JwuzC>H;T^7j@7p5qIj6axY4n=vVao>vU~lnX z@Oq<3vg-w;g^QtZmd~~np)Llyrna?UkvV#>SGpWzd;a@UtAa@;3cqt1k5a`nXVkanEj@V*0U`pud-T2*}x) z1J+_RbHz_el&+fjVOpN!iIUvMgdv$1H}IC>KA!?p&ey{Q7!d z|9+mL<7X-%YZmXCNRxR`>WP54Z-*5?mNF9OsFy^(K=VQ;U>@}R*{>*r8$^e-fIMSG z(ci>mz(K-g@<_r3zkzUIszr@N#d(a!XEd9 zjSKImG`9WEe1>73xHi3niLq+$|5{`{x%qh z5V&#@b0Q+5UECeS``VzWA{~kU3F#UWw461TE!)Qifbt^u~Zjs`Ygl-D>R)7V$U!2PBY8~Bv3%bG5> z7OibTJPzK@(X~XQ-bTMmNj{b7StTN|BnW!yy@XVa1o|p{G=C&>pSlF?BkdeX;J^;l zRyD~!y&C?P7C;!^-9=wxH`>~9qc<3L!^HCIpOENcl1VS&wX;MY}1lMkBF!Xl@LU0)W7xtDX$hBS|r zbW4i6$}2_H)R>DeK`K@briJ^0^F0ah2JMg)TXH-uQU6ucI8*_7^Iy^KNRL)1=?GHU zZBk~)x=|0kUtC<^_!-^?g*%(vd*2~D`GLQqyw%YzaJ%(6vt8QNW_sXqmtab?i<&f< znAhpO-@{KxLG$bqnTgf`kX+PnJO7~*Ao+56Z51w$ko~szxqr*0Lbs2RTNyll`PDrj zllxOy*&gec$5v@|N*$%isVM_CYueaJZ^hX%o2A8t@4io=75Pmz+m~+l8YHijok~$a zqJg76B9)l$X(BTpZ}aYl-Rjv!j#q0$H_B*#S?vMYw+>GW{Z8x)?x;X761)U95K4|U z=giFDy^Jo&2PV(Yoqqyl;-3)Bof~Lg#?qNk8ie7sk`|$GGOlJ>c*BcmYUGFLPpD1C zeejq<&VUkR5`u{g4Q9I9{=abAY(refJXd9lhxTay!Y`8(J_Fb=$Z}(y}ZMS4jA^^-dN= zKQnHUSZ|+hy>6x#?B$g%|5Z8qOa$wPi9wtbuwNCc8D+5<~#%vSa68)<(cj01QN z`>|0{5)Na^@20+U8k#C1E+5(=O)f`A8GWg27zDSG*>#N2ztV%#K9TNdT{2w6~iCBw&0eU@*lxen{-^Qw9_{ZQ~l!R zy(V(Y7=pYFCGdhv&7Z+jVzjste0sxsfL0CV8qSU;yGSaI~L2qrJ+}VsBTcgpm z{#CuH2Av^O3FVcqA6%;U52#5!$0&=PIdz!AYkzMciJG9R`bRK(j>VLxoh`gE{_=9W zwnF%UMshTP8nVTuNvR2!Xo)5TG6_A=Q|(%@vhK7%lUiooo({ZiPWwg@l!pDDs#~NG zhNPYSi9Q5#8E0B08eD5xk8H!!3_1X0_>!aaaP60Ub9VcEa>9ErjZ#DU@lachwG9n7 zu_nLun!qyG*Ihd-CGTv4sdUBI|M%rWVLj933QqK4B6e*O3t6q?rc8eOQ~c>=I+2^X zlQ{S?)XCwMmq_D_xsz|9vS{+V%PX{Bh3`Hi)|3mqCg`&wVbFU(9dMT=yWS=4U~Uli z;_Y7ic`;i!P)RjzbXE4Tlnw}WV#v?n2>Il=nj&i2<-~sn>lb#&gRaZ0b>S!$wBM7E zakb;9*wz=vJ2aT#W$!-KjR7|MLH_z9V>?*_ck5SkaYdf+180Gif)^ms|98P0d@D0h zs9!q~4Q7WRzJ7lH=lsuhI6!g%D3iE|gNVl$!iS51GS|g-eus=Oe5vg>VV6q9G0Ie3 za2wul-s$36B*L>q!t}GQ41R#gv>%+>UP#K#jfKepjIUMUOJl!t*!P3!hJOFbH1y2w z5CD=csxswJSU^rW@r6PutFfLX9Swr#{Ms|xM1lIy<>C>R#scQYey&r5O8*Oa4coh} zOxCUoIMO#56`*&%n*n$1Z)WgQ)l0JC8&Vr#F!Xmih_N>JX^7<-_=pFKzF5bEy!oJH zXLk&#P5F>xpL+Gh3;$ohkQplE3*VhBOOlkDd&dVkyt{z&V;Hp85%|X^XQs;ZS8|V3 z`X`zXQo?<(kMJxntN{g?ujxr&BCSiD(fsm6FQQ6WQ!h9LQmQ5YaOm9D&Z|BiDE)0(;g_eU*tq zoh0k$Hl=Fs(#1rKfEn(OmK6#H^vzHg?*IWdj>?p*ygU+;2tYt?H;F=W!ogAqDUvw1 zj}vXmsou#0YHx)UPWI=1_#Uz%2&;l{+6Cz2uJB?KMQ~!#_Cw^FOt$D%By_68Bf^|1 zo;^->OcV@Opk@j}Ox??mWdpkt18sf*_Z9*+G$al{b}(&nxx#vGI8x&6WK%<1rE; z_#Qm3A>%qc>(z%sM~;GFLzRME&1R9s^FhNbC=DIO>GKiojhpQlHlqHsHzaOvBW_CQ zS}A3Dk#OUlr{Bg3yR}kWLu}oDFJMzO6?Oo8sWrtz2SoZ+gf@x z8S>NT^p`>iqr@BXp2O?nFGvxX<3mTAE-!IwR*0NW1sujT9uEGZX5p5eq*?60vKDFd zJC5&h{8qN#g#su#$v4i{y~(w`9`YeH`q3^`jHTDS<(9YT~KD^3NK881fA=T-T)+e)>hWmvh9q% zJeq-a=iK0+mXW8=0)tOT)*Q+lP-(K1?>rvA#9@69o=nPqDW^j1L@eZVkme6G4%P2+VHsO{sJxp6jV7aT6*eLS9Nih)0nm#vORwmRKX5A#SZfee9 zjb47owuOpk6dK7S$;k=%xQuJKWcJgzREO-2&Zd-*_%u(BB7wS48*v>*`lb1kO4X40 zcJmp{;<$2nG_t>@DgHPmfVb-yptJQ)@y2M*s4mLF;HqcTjCzq0u$SKp?;C9ic|DrH zLQA#}rCO0of-T09Knzb|o$3i2@ZL{|tP(f-*BTopb}k#lE0Pcs@rZb=0)k9@56bd1 zT>SQJKD^meGHv3`}ND3hc^n$WMfh?iBv+zr8CBUF7pU00#wQT_7vG6thf-pylxmpqI zsRmx*@{x89_1|UD9knhx6ZJ7z@o}h_5+c}(pR#DMeENSM<1}Sb5>{qP9src!rWau3 zXP>2g;+G)|cDV5;BF22hd|hXFP?m{Z`BPFJ=!cr+s20VHHZml50BZBZ)Krd!R<_~X zkIofK@se$58F}*$jN$(r?a0BP`?I_cANq@%??n^L1w^yRq41KRN{ zrg(Z*GDJe0VQTN==9<`F8g&G#l>Q5T5nKLuz-gj{Li_h-fwX0UGuCA)})cR84P;lsm^`tOpFI8&~ z&p@lI>DC#5G796ttp#hzH2)11A{FWqXmz ziob|Kl6e`t5~Ee>1y!14z*6R`GqHA-{EG-XWPqMAx~Roii3t`3muqQb0JoQgK|n@u zM|Gxw0F)gju=n#G*~8#*qTQ1Qd&Ctfe)k{1WfFjT($jBzI$gaDg$FoQQHBb5kvtV- zzb6Oh2wIvyQqKN0bN>uow!S=0C-vp`ZtZ_?K^`K2Pn;T7#r-Fd^)l3T;RBam)}Q|d z0|C12{)KK6D4F{=dQ^q0ARw|Zv$ z<cYt!r!;^H@rsO8-MuFe1~7|9GQ*m)0t!jzPhs2e-W z{_7WhlbsD`Y^@$Md1+~7a}AfGq0q;_D*#|0f3z*{nHYc-eUBUFW&3&*r3}>7J$0&b z65mK_!2Yv4gcL~g7=089wtigM+SNN#nkcE~1J!A6!lQe$C4h9d)Kv!8+ zZHLXr&HtLkA0>w%gshz9^NfE(#N6drTADc^U5{}K3JGp2W=&L<_+<5t&Gtu1?31=_ z^6xm{FIQ$mF;Juy6=j08#R7tk~fep@bjq>wQ7!>pGgT-z;~JkVwxK&9N?IO;t^=LuwNGnbe!F%gtIL z0Ii&qwEqwSfN1TU!9V={UWMHT%ABx?msQkMmsRz#VOu+>I;-Ap%f0e|+Bz=BrnZVT zw&5!+b^I1j8k$a%HY3DqC<)CAM@RClPVKKw$MT=@T=aBtC7+<{sOiq3d0%xe&5eK6 z7tfNIfVw9;KBg69su)l^zb5*#H>PJkrl}y{6hyXwKsXQJUF|iLT;%1?*WNsG-xpuF zm~9-Q)aP{jNH&n!4DfqdsUQ;aX{x9s0$K>^={NvKDICb)aVM`(n%D3?gBo!=2wV%z!B3}salu0h=<8Ya491No@M0hZm{uSgDGeB-U(H=R zA*VE3X3J-+2JDG*L(w&F>~jSJN9t+uOz9svfDdm~HUhBtdqu59|%6Tb#`Vo0ODI+S5v$Tvw)H*hwsxP z5Mit%^TWObV(&--ciT@te+XWsPv?CkgY9_D04#+$>2m55IJJfchbSqBhKC2mA0OQs ze~!I*ad>7Sm!?9v*a3n9^SI!8_#|tY?7SOMV5uO6Q&~+X~ z9lE%x5$gG2ot)GnX4h{~wNk+MfR;M=vZW|8kHZ6wdWlDu1NJKgD@hqkj zX|VvnF%7IxEG>^S?YdtBDkNdru6y?{iyiVfde@T`hjE4$WSsN7Tai{xv5xxg`jYPM z`h5Qjx-XQ)PKp|OL}&!rX(RX%+%hN^3VB$VT9khbst?p)w+U++j_3NdT~2jXSt%YX zGbdM5eBlair?l}mF*9}7liyvxF-jms3+k_jDugJ1qK59jGSImGdmQ%!xb z&VeVOljHnEK0U3){^`V3e)RRi!o+wmK)k9{o2xBy)PC$j{SuC^MK%gYEx_pNeU>H; zXG#^yU&v&PnT;xqQdg~_dP7A44BX|MKZLZ*nd;`&t-cS6i4$1bj!li=^EIv@y0zB-?oO=bXO#`{Q0ntIfM;C67d z3RLfy*I&|JgfD>Pu6uOUr>~(>8mp_i9C`*%4p4TE5k)@?|02$$K^#oBiI5bJoj0+-9LhanIFBrN&E(lXF!KXuXNTo`%eP}qqkGk<%|>Rkm*1Dp zlhZ-9$U;7!XHc)|%V1gEJjlBr$7J$zJO*qp^!vp4k%PX+7*gxAj?H8Z#^P!jqos|E z{BF0BHN8AK7#TyxswwIRocp{^C9Z?k%>aDpll=T>1%Q?W`@0~%|RfL z5f$zHiC8sG|A$p=*DK$Zc9*a6T$>PPezP7w9-0*Km&mn$@G{=9Xv>cipJx|{rZcW?^4)1R3*~uCh5XasgO;3GPCUc zaHua}S4=>FGvf)s!7f-BvhY<33W^{shPMz%#^h$MuACPy+0X!-qUvIAcshqaQ&uAm z#Ad@$vaWoiforKaRZj-QtcpZoMH$k{s+gWp+Fqgr${*x090`+?pCKE^r~i&L$4T!T zhUMbX*qbLk@N4eswinv)0LS3K5LZ8RCU!lzg`CZ0G@PY${_d~YQ(h&CUtrh6(B;_J z@x7C&&_vGtf%5k?OS`4a4`o!ie5(Kkp+NDTzBjDI2J)JI@Gzr&;%|wC@ZvFvSgmT`N=@A?>wmj!R&{la zCjlqn&@M_Ge$)5qWTW~VoP>=PSx(vyBz{Gq1k)Y|Wwh7CuTwlDeuG0M1m5sfu;Uvk zsqG1tL0@G1z93QcP=)r-{f=_7vM-n|c#()DT-gQ)4tttNq#JDb^^o=oj927#LKFJ- z{42jZGyOY4zBxzt;5LZ26`6s{g5AO^M#^Y*hpzKJwQqy}v~?Hs4BtHQ^1>2>mmLD1 z{aO9|7ed}HAR(c?x0ee+JTL&SIIxup0r1MLPf;@wzvmrPX3KItq=>GlNPyjpg^>}s z9XVfIC-fct$d7<@Zn97$tww>Gi#R)wQZF>o|_e~PM3t(Ufw=tSJqq8pQih9DFAk(`s#;56N> z>vg-P>(hdt+&#&U#CsN?(u}I2qcZ{V5l}8*4vQJ}q34xePQMM?3APvEISc|AOd&qj z4dAdjAZv0AXbH`(0$gY$hd7tbT95~sL=-lwaoE?hLYtotH~YMXeL(;~x?90&lZEoB z#cpdVo7drcd^?cS=S0B>hNZ9}UnFTIp|E?Ss4lbU=8Y-Q1 z8V6iastBW^y&V^i6Uf@X8IDG!%;-oS1p?a^X-YntB%Ey5b%P0s!)3bxB-v&t>|fLV zIn27&;l^Ykj6|Mql;pNjXIMnWgh(oEvDxkdZSV*vfyF!c8Nje+m6vRGxPkLpXwiqV z285?WQBsDy843U+$Pwz>hnrHRgDsay__o?eML`iP+nBKrB3_&bA-$k{RA=Ql@G-*B zSkiD3M7D30vDXZUyx-j1pbs{504Eock~QMu7#p}eTwEpr(xMggzR+Q((@^E<*d-Hepi}xSY~nth{=Lk|m%yBM z8gF1WrNDO~RtvAQd%JEInmNlu2EHov3{akt6N6a#a@e>)J|ZXl9lOWxAD?YF(J?UU zTTR~WhVo|ZMmz%a83=(0UNrGL!TV7KMHc|;tI@0{HE0{JeSc~dhd|CSKTPC{FO9Dc zC}~6-eFTuP&Jxdl4IKwaR?R_=$qbIP2T(Jo0xX~c$cAnmZKX&WM^83BZU z;(I}+s);C#8gDvrdW{J^_EG4Va1ESObsWPrKwJT`@MoBJ+t=-%Q*PaX@cnM74BD=r zJPtfQ16kwkG}P7Kw)LcFb9yd~f{;~o15+JT@J80gYR5L0oH zSpa-4=iUdZXG>aL$HxZL=Ys~!*Cj7O~ zlrwyO7r#UbWE%0h3pe}baX8xb1F?>V=vy|Rt^@G@x8I(x50PBUI&E6l4=JRK@;SOqj3-kaegRl)FX7b{t#h78qmRT)a`>J)o+RE z3$_I-KdXmFm?Q^KzYXvkkEpEc2)hmJ^jf!c)YE2de2SZ;EkK_4>!U}t#S_Af#T3Cy4voY1Gi6DEQVK=zVG?* z;?y0^xkuz=zr0IA)yN{$LGgsViZu>#L-LO&`+ApJ`y*LH&(}#_R+7o_k_I?M-fI!K z2lo*`UP5i0ib094_pQa70%7c`7Gea2g0v`9N_{rh{=PO5nu7P z?q0nGvzz8v)(K>bW_R}=ai8Y@jw}F>k{7}|$LWH;0$?Etuf-Q3Mgr1aGH5y-WV`K- z_jL{pNeqy4uK=dXaqm<|z4#m3Z5sk8TFg6za;sQBy@ynDh~$bo7#(0CI9W#T%eC9gs+K&AB)F zSjM$v0%6`^6uCodfuXzM%N=G^*0jS`zj;mGZx+TB^C<|aSQtF@?IEK84auX0A!-Mr z2%y!IeE0bU4Bl>JgKA|y0fI0k_hHIo1MULjh@`zSYWP{ZbP*RX+&_Zx3H9p5_W<;q zrq2MgImk>~Y#)SgLp~0~poJp5kkvc@6+^#WfO9vomcel`WO(m9gku&LqJPw6F(21! zV3-Ksj;$tla8^e)Hc|5^1B2%QHNLisfY2(!(!d(cf*R)sK|V7KOi?W5>UD0ljCWQs zn0Ar)_P5>U89QCQIoTI-9gbn1ySDQsiF{F8$Z5D*Y(7~{q^(j-q_s}pSszL@cNj`) zzN>=K%JKQ45c_D0(>QNNa0OaXs2M{YQU2A&MjMOqt@0&1!cH_fo-V$7&yys91Ghwp zF@f5w%0-!AitKyKd$x}cH+{Z5&l@z~GCFO>KaiXfIWfEM1R@FEty84xT#tsM4Hptd zekGy5$sSOqfU)Vx)+xDSo@Nvfh-n#3q+Fzz~hJcoEFx?O(fd_5;v$ zd?UdrF6J0y1m!S7i7Zwifo+fN(NBj>b8%^i#K?+HCX*aVmLnk(iDe34{#dY~!ci}o z+MAKXSIZzb6un8)2|W+1tl1hP~)YNV!ijt#7I%daCVtH z^~83Y!6B{c6^X<9rN9UXc9sk5#ZXq$8$->Km1-A)VSlbIcMmc8?iVZV#EUxbn3$k5 z!XTT{51?{|XWF)h8yilWr0Pm2;j`ZYx*NrTBU4|H_k&?PIcO#6s<8y1wR*LO@6?K=K9xA`a@rd_kS-S<`V~Z@2)Rdr| zI=Y(HfGG-`Y*{qLC-ShTi12Llky@Xd%4-z)ew-1eAWB7=GkIl+d%?#YB(j9NZ|}c( z8T951Mq(KWqQW_>~!_wg&6c7T?1!XGvSbnl~k=Ad&+@>iA1`Rn0$`b`n{_BLzufH zUnY(0DR1-20cpYh9;t`|{E9a`B*)PTjB=_$V!cpx-{J}PR+t&`!z zOX(0JhcI`ABO}xjvmE8IJLV_$hDn)D&{n`>M6)AgPi#ohZi(f(*Ni!hxl5QV>oZ?%cf&4#w`Wi&E+}*a_tUguJ9K z@%kqtyD8w_r{XRSf@onM_Y=hXOVG~_#q?)#wL~+@Pi%)slEjh7Tlmj1Vp_1mNC|TVKRNj&{mjay;4p!`8Y!}ob5UWCHSzjS~{IR zDEBiomEOzn^EmkgVfe#-&*1RcZK~emD2)o`mi-!aIVTrsDEczV z6ha87{WQiH#06gQELMs(z6n)`iY-UY9MMwBB`-`BUO6&` zp^$gM0I-8S8o5A@BZ=NOB2A9KVT#%GF-)?dL4jI{>`C-hz#O%=t@}MSZ3EQ7NWcsQ z2+Nl~2hL7Ia)UT><#ZMJqJdILlK32j{fk*iBKda-!7S#gyJ%>n7N%JdV1-;NFerI) zZ-d+m@s070R2X)|2J^9klT1$b>9T|07OspleZ)U${Xy!GCY*Mk&SHUlG`?&O)`bK* zz@UEw{WuX%cOvW+YF)BIO_d^EtmLjm_Ib)jnUxpn!-uAZb5ve#r4v0PNd@c{x4sFc z6!t2%?na7CJ+POzpeq@utv11CTi8MSRO`$=9MKSg+lUtu_fFuvRxL%bnG|#_wAO3W zwS_>*qz@^d^?E9$Q^MztnwPi2S`vAy%1TiOz^6)?9*4n8PGY_67b`gA=ytXehEsk% zPPLh3zdl%4U6mt>J<;t5g?NPqTBEKLq8zzPmrEU?IU4Ap#cdY0LyQx9;+~Eu_-KzQ zwlkU&>bM~}FuaIh5tpPyPFfE4jHmES(iBDlYJ@u)bF#x`84iX5 zvLG%ukQjLvrv|~pQy>xYjukeZJxXV=S*qqdnZ-0gN);J4I!OsyeO52!^FAHnhd82H z(j?}1b`l&+EuLqw!vZyGLx4E&9tzH(zZVgCD8K-M_qlU%@L#;&p}KydE1*V1brJ)) z5eK`LBPYvoRBlbGizv$9;l{(U#!M(#(1V z#dIZWIKnPyinLn{e-$^3E)sf4>o2r|`4M4=WWKFaJ&aMxWmckzRNn%fGm&CVmU|2y zn&$nYkROHYf;h=WrpMwa>*vRU;h0sYzPm1H^7QFS6yRXmmcim&R1)yUA@+Zg_8a#q zHcZE>ao#eX`BD&?yPf}e#W}fxSl>2|Jdd%VbZ%RcX~P%G5OEJM#gs}?$q z!}jmB?n#OSabXci85;7NaHi#89&{#z7AWKPjxCDH&rxe+{+{5zrl(F8&*F)hNL@;A z$CZ|p+_Ov_gcDLF8hG0;KMY&kl+;@Zk3bc#ULI} zr=cFg{4wkB5IZ`HDFXW;48mkCm0fLpUD~^Q1v*ut`mDD`@dCW(4Hz0#)LzUGgU{og zHLb-pF6i07MpAe!{V?{g0o(}80d5;5z!QcICZ7zP)7sMWqBvFBX&dw#tegJ+h|3fzT2h+SG_$jcEz zBgl&{JN~`JLy!RwI+@VSesRXheK1FTHv#eKAMXa4H0gpN4X`OAmLmtEM)wm7G>d`^ zeb5I{T=M6`%&y7&&o{-rxddpS#(~qyo+0}0Xs>3@-v=W`sRxbqfnW-d2&@Vf-i=Y2 zD%wRT_%q_79mO}R4zy052P<2TArkei47IAD7HZ4}fl|%I5#yWhS0oLWXX(NWAiSz2 zAdT+LnkX|?Qh^mCnJ=B`{~C!CALTw{2=7*luF~w2B z>P_FvB0;_%R13IVV6MdnA)%nMzR&D<%yK@%Md4_TW@!x_5)}pXe!9j?n_CSXf^*j^ zINx&Jo_7!Q@7Vg45e3OJPy2*%VZHAcm)&i9J8$qf8*(8!f}<;l!db}kf*6-HX=^iy z(puYG*vi_*RS1&bO^fI9q~d*k`nG{YfNctKGIs?6?j7NkN^d*fasPvL9fSmV8JF<$ z9<3ksyMb-5PgjIX9Kx&@X@qno_OJ7Vbu2HQ9HI@*jp6C#s`oE@L6pA0Hmt#(QGOY( z*MsvRL2ZnIGaDV(L$6XvXxInWd3UMr#yOFkM9ZA!VrvdI?!Q}FAYpl6CDjWkqs~hd z(UV4Jv(Z+DajFk}Q-yD}m#WceODGS=7K(N-Ff`Y-G=#wsK#_%)&So`{Y2rt%rUrgT0OeQ^4M*J9~gWBJ>Vd_FdCfOZN=} zW|1M2q~aEkJ3>l$#3?i19}bf~%qP6I$czfc#|y$>ZjkaswV{2dd4gmAEa#(_p}iJK zC#-=K(p9o{lg&fM9=hB$Y{s?`uu(0va-H(*< z{&CiW341 z!$>(E7iZVQ8r~HqV@y5C1Xfu!4yVO#Q2P8Dx1#GdPQ8``0}DKuS%j(^Bs_VX0_ zTw`*JQ(>RQ6v+%dfP5*MJi&!PYcgTfC93`r5ERLDY^c|spf*wwV7P&1^T-RY>y)JZ` zWfDQ$yq#ivF^!!9qwPun&nw*{;B_LbfGc4UB6bpi8p-K;k=S?iu)?w%0LOUu^!;P* z^vG%}!hn330)FSnTh7H1g{KKK5`=Dfp44Ey@Ys_RkkRAXPRX$QbZ?U#B0Y%nN`s?3 z_qLu`1d55+ppYxme7WE+l`F~TVeKd4{EIU0%M%48eN=WI4wrgsuiEDzc`UoG>hGHpZJ_ZYR7B8?DyJ z^0@?co_mvORV2`Jd}=yDD{-+qUrXfWzpmB3n=x~}O^iqi_*q28UI;)YMh?(ygkw1C zqhSI|?|Z9@ur+#p^wKe09JQpuD~>^Ap)d8- zuR%E@@-XC9$grS}(OdT8B?+ZFm@_xmS;8?5Q3FYo8b`2)*c-5jU>S}l$Exe{CyFZc z%VXS=nDIaw5@-Wewd6y8e$5{)$b%MiCU)6zE_S8&N2Pp!=9JryF!IPFvB+u_Gu!r} zp!*HcV=^4QNG>Wo;VN0eJ3Tvn6C?bJ0^(qWGtlMT#hRhqexKB}9<^~oe54vbj^r4< zz)~8h9rbZ1LXfn?_f5xf2I9{ktZ)a!91h<};1pheMxWcJ9Mp}XEy3r^I5?PHSsB4) zYqfWXPVh^QXiCC!vSgxEQ%84A8FYKXxRO|0t~DtgSekSptJs=d9e|lOe7`A`f^gW$ zJ}m<*1mlFhp=?LU@9Ygn^PjC8gNIe9cRlaFzfXN5PI7^YdEqVv}E%J_spk(OEW|#u~z#Wixkc4AH4Atq2-w~sslMM8-fOsgwVspN!l2H`k+R` zFfEyfm5R3|2arv)hD1mP-?pBge|5IW9g>U!>m3;V_-LGzazi1pcGDK1q)l%f2rE)Z zDwx(N2?L#<_=X3DM2vM!!XBvL)LDsgSYmAYo=3%Zv^0jJ2^94ia2atRrU#MH%t?2k_H|s_lU!R9 zc%Xe{!c)5-5Hz53I*y|hk05E0BiKs#<8tF-#2+4!j z=f|>Zy6l&x@x?P|CU3*a9xb-gY3;hhBSF5@kP7vsK*Od;4@|Qhfy_qs;wqP9tyd;d z%Jzz`R?jx*jqbc3sRwT81Ui8p=(=z!8JYONT4nZJ1j~pnsbhlVuc55Rpqq!95yVUIxgNJ%|xv6-FDUHzHUB zW`-jdMk^G->=iv*#7#TC7W_os4M1l5bf<%FZpn+{(o{lM4BosgBZ!a@D!?C$#l}1p z^PJTagX7iAqhpTJvXnx(MkI-#(dqR|tSfXo;SbuE2#qZ!9etUM=&&x4qe785q; z7|kGw5ZCm!dgx51=WvG4(t4nEJk%kEBv1)yN_kPx?{|W+?A>FC1alnnTf=bP%ivb^ z!NW$m8XCSG3{@FuWZ|#wiE4=N8clcuc0(q4&eziizHy8#Hb8_GUXJc-U}g28Y~nD) z^LyMOL|*C3o9G$`wC0}v-Q3ObE+I;;Nn1mK(%8}C~;%y@81wbE)FCRk)Rb)m$< zphG9UsvPaXD&7Us?@*=~F;|pWJMH0fjMb!JPpIXY21_NB1b`?7BN)8RlI)@qy@(YM zmf-#w$AAC!GG&{w|6quEjP)X2VT;U=79kNkDbl(hW};5BB-~M<4=eN|I;1tRx7W3?HE5=G2#5?!s~uiXl)2tZ*@T){Sc^#L{EkcWqQ zqlVPRjVI5Uq7NS!eBb|^>>4^5O~JcBUJnfZ-coQ4d)q>Y6LdR}OXcbKt!udZ@_4K7 zU`d+SqvjwV6J||1qb93>lo4S9Bx>Noo(K!BkMC#c4JQitNJzt1Xp7xl<=d3V0VRV@ z2~s&$M0OA;eKMo&p0FL&F;oiGAr-{RjxvP5B) zAs{Nz5w}SGN1TV)A5_{elYTma<(~A}dWJ08!ge&pl1M$#1I|m;=r+uSUnfktF4X_6 zjwFP;rtg6d8r7S_z6~8ywr?!XSi13PC<^y{sc2sk;y)@6(Bw?*sVg#*8y64KFyDaKDN^ zb2oMX=+lpkODS!nc^9W&NH3BjR;W!9Yjt-W(fL0rB@4pvL`)xS8e;dmJ82EO&sLZ1 zTH_THn6mi);Z#xvLTKh?L~WLnZ$pNzPefKXF1V~W!lq8l)johCRL{8=8&IGbc}9XD zl|O#1o~ukAXLFE@9n8+EP@jw0^X%*)(9mKKmqe|kw=wAT9WkFHc!LEmsnxw-!_Jx8 zU3HuD6BnbN9G*4?B}dMDfFN`xh?`gz4YP}ZBeow!FzDI^fJA6=20 z$m#xF1O^hE86yK9kRo1%ICwsfdOBn)|_0jR@_qye+_^u`)_{BKH&FQh9xL(DOG` zAoYt(lnsW~Zn9y)J>LjlNJT>o9G>9*9vNJwA#*?TZu;J^rwBbpqJpXBA6ufo_Q&`W zKfP_t8Jxq$>VJe@Quuo7_vV(wxb>(+eliLhuj`;JB&5K4v=aR!_v5LLp_-jkug@%V z^ax51M#>L$er`X#sCrH6$N}cH#ESxW2penbELIcP7=fkP{NiK;!*rEfBV!mGQ^AH~ z>sE-$lB$>?&0mcMwq%Jw$u~lx1D&077&fj`0+R}jLpsM6=BtX z@?)o?D8fGQVrxQZE>XPlRnOQQE%)BF$M|9dFZ(jddxaam#@z~f;Zqrx+>f#ijFf+W>t!H>HwOc@05{)v zS$UU0$nutcx8+D-{HxqQUO=xwiw$zgh`2EjH;t&B?6uO}w$rMLe?9hp-YJv{O%2%* zRGt>zz@Tnvxl&P_HrcZH_gk_Ra#6~m7}?tW2~=tZt#QXpHCK0 zMjka_ZSkX{980Rm;ijf0YfD=jD=Q$08{M-r{daWj@%Wbwk>8z{3;4(_&atrC><5zd z4^1{EUp=14PN?^U7dtp5{#GamP$=Y~aVGU&wfv)uJbuvB0PE>~J6;81ZMTu;P}iKy z*MF4`{7@Yp3^k-1*`u(s%ZIqK$Z*7jsV1Di8AzOD8}my1$YP!5p9K>G^`b<7_4lHH zI->^zmZ9di;idT9fM0DP2V5ek?{DpT`PaVy8Sww&Qk&BMwD*tC#e`_m6tVxM&#W5 z2bKRA^yL;I(mZrIYWP2&X)jsF?@FNgS_vH$1U|9M*fH;nvm82R6rP3-^wdqF-!#SVwH UF}$WH00I7_#ovh4h!_O?A91pEHUIzs literal 0 HcmV?d00001 diff --git a/doc/assets/multi_gpu.png b/doc/assets/multi_gpu.png new file mode 100644 index 0000000000000000000000000000000000000000..88c025b864b04710897623a4ac9d38a25fe8187b GIT binary patch literal 104155 zcmeFZcOabI7Cwp~NCXijx`^JRM;p;1qPK_|(aWeqFv>(iLX_y8=v|axjFt$Z6TOTQ zee^PVx$or5cg}aty}$eK{o|fLFlM~_-Fxk|p7pF}?bmxvb!B2A8X_z#EMk?%3ffp$ z_`k5Qu4xnC1MftIj6;DR*zVfOkFd)6=+=N=6s(@9SgWaFaRILhu&_hyuy8Sd0({W` zU%;iA*jU$qpV*l9GI9R?6#rM|wZC6q1MY%_6&uQ2f`tXbQc;l8eTBV&yk2)-vGHj4 z;fkYaeCA{PPKltvz|7awq*^TmdBw)2*9;VKwX{gx=X~(A1X=ExgT7Pl; z_6=9bk%rgN!I9wq{SUEmA7Rt{dPFqC6o`dG@PGcMnTkd1+GbeE`u}>{|Bd$F7W|*x z{@Y>x=WqWRX#ew&|9Qy&u=g+A{QoExU@G&H|IZ0Z@EU27{cg3UQ*e`W!=qND?F5^w!Rj~=)FkhKI?as6mWGmYduf$J+gE3&bq-N6Ix)D7p)bBG|wFiE^1SEIZ zbHW{}EdBhjo?8g4udOcr*McjeuLk5v@Z`c@=dKOqr;HewNfUtc)YB$M5(1z28#tD{ z`8T$JSa8egQC#jw$k^MCtQ%Lt-`~=O`wGd(fSZon_>&ngR+X2_11=mkj@#SYd)jH8 z9cz%DM*H(50ppIqOe#(c8U4=}7*opq(Z$QC;_~#*<3+_~_A`eywMi_IQa?)?_PWTI z$Lzjs`yWZVw_soQNL`}Pf5vS4to!-y7Ghg8Kh-T%>t$UTVxFy-}Pu`{!?|vd*SyA&HN^ zl>7Cr)?MvJ2Gn)VuYb5J)Z>4)X)|*+bCitqnw3sXM+ZOgkH5A54^BjV3)^CCUcCQd zSAK^7B;Bhy*z&g<{Eof@^=tb%G3pPtUv)@M@}0r^{rc$Kdjy>lbgpgp2VA;WEksDx znA8;ADEPR+>+G%CF)vR>ZYl=7V$ zU{9JxFtQYCma>}s$PX-mpcp*e&VZqmnOL9vvX8Ddk;&3+}WHDzooX;`-fA`V%y&R zzOR=qFOQlXZWI0|waW~~35_|LcJDV1cX=o!LesdPwk;gO;A6Wyc)5Lb>0G8H?xbL% z-?U-H?{H@MZ28u8DWOf3FQ!!o;m%N&S6#dsEUAaNDfV7~#}H}=A10HGDgz7?pA|F~ z8x(ER46wsM<;;{PP3aN3%?FUDC^jXeDbMq24uiuHtbDN2Fx6E>2SZAI1lP&7&lo~` zC0ETO?mUdYY22~tWZ5oEfcU@W`wizhy+;&$KA83!oqbPDD~arM!zH?0j22w?!o__h z@iz zN+chPd^7P`%a!|(V5}kLRWzt>^0+>yj5=T^h_NnmzBpL1?pG-HxBe-o+BEobvO~{e zXH2V|&;CTe|0r~kh4H})Ay7D>r4oyq(Mm-PLOUN18qghqI%`v6g&K7j{*|MZ^o!F-iwM?Wq z)g&c7gxI@?E#}LhIkPYLo6q;=QAD5nr1pDugq;6@0GM4X;*WCV@c}?MW0D}BrGiSK z_v&Kys_ho446zmJHyYx*1@AESUW%LdXe$PA;`EJ@IB(`Wx#xscf{S6};PDFN=^*>B z^!?P80}q1w3em~A!tgZcUT0yrOY+hAimWWP>7#9vpE2(Dfo;p&!tqcuJQ8|*Ne0Rk z#7WcN3pnXEYj2OzVU_j1ZaU)HCw(;ijB>=}ZTb^cRr_~R^KZys`!6BSyZM`^8;MO3 zrHktkvI)AKD`3g|uNXx67crCc1@;9%WJ-|fa$C3ca(we|tv6j10KC7>xX?By+~`clOup^}~s=@c{H+B}V!mWY6;ulV{E- zneK)fd{3uVjW8wH|viA~A$dTh3PfRE|j`TX`SU`zXZpE23b zQR4PxRoqs(JD9tj*9dMmfk1!TR5~K_bG{Zb+D2t2^&+E2d#2JW`tADE+$Z!-D~ZN4 zKBsRrQ()RMo1l>R%{+mc@%QZMuij|+Q31(j+(7D7IE|a< zb}d<75;-Z5pHI7!o7rMu;P0Gr(OQ z7}UZ{yJuXfjXGwW;^FEe)=C!;2DD$#6VU;uHQ!&;PJuK=(=%&#`-UEb(0e{}Y}yVY zws^UI0l>`xZE18>()CT44z#@HQvn2k(6)n(Az5IW?N#IF z)g*_7i1CLtuKYELzqr~enxHvqwF|3wf^LrT{m{qlL&^-^mg{c;!~=_P}8N%=+wzk@+|%~8tiIa|%lrx1o@2Wd)-^i#xP-YqLsboH2kNgx%c zzF5u6V@HEAr7w-loE>gO{_nLHC0{hyQShyVJ=C(9EowqK$ z{z!Z&R_DR?%#cqF*+8o2q}?(=e}x}jf}^1(Kk371Xn`07cUmUgLgD19tC%Dfq};yH zIl5n8^?>^Vh@?;mab>UQ)OSja^ZojDs`(>#h>6M0#nvNP*JBE)sD>{bb%!C1h&l3F_Ulraw z0>G$A>*!Jz2%ry6Pj9sMdv>3%X#_j~n0CWi;z<#hi@)*Ur^{XXGoSTRD&I*B4S|cf z>#_t1S_Wx{%Ogx?24;Q`^CN%?i;G&Tl*@VBclPXCzfZP% zt^+ZjQ~g@rvSGhZ{Mo9XTKg(WyFy)MOvt132P;Iaa6fCiTh(+g+T2l0x+0IyK!h#5 zNyu=FIoiZ%GNl}uCu9~=(Nsni^_%YB(5-kICB+;Fss{~v?M{T+Y}{fo@&-?|e6qS0 z5>HI`T-3#OjZY9cVtu22C!;h_{YR$L0MUwrp)_)YkKT8qq5|T`Ds9Z_Jr-MI_pd8} z%X1MsSzZNpcO~1O)rGI7EYd><7Y2Rl5x|f&&ZjP4Zt&GH#mh>I6y(o1wijYf)<$SNZBmQh4MCJ~dWt&zyjjA_hbD3qVNJV~XM5PRks*j>~W?HrksviHe3T`dP zZtC~`d5uJh)X2H{;;_4lru*Lsj42RAj2jw}iyj_*!d-eNjCq6Fw(z#+)0)vP?s7V# z(HV1UBhTW}noyuHI$eLcAHyg*?b7OU0`P${^udruNZ4v&scrd#n!y}&Ovd^BR7Ss- z2sOtEzPqhXg_x5-;f6><7o6HYVh-PGM26!`Pg8 zQfo@9?P^lmMkx%8$&6P%o4l#HB|aDUtSMGml40VF9z^Yaa)5Mc0=pgKAlk7u#amz1 zWI0-ukFd;j9RKrZx7p=h{I3AXQ8QE}I4fOYcqpm{Ry(2=DRBx`+An8k&2IduumV3i zjaXBut=@+1t;Y`)J0&_yD5)oyY!^2f45By2-Da$kV1TddnzY2*)ZpF+w%iFf^d>{- zExI?)bawN2U%e)YX5SHV=H?Gw=%I{)MO!HJ3^`?^r@I>0)89h<;?ANr1vB`EzTGd%?<8Zn%u z%v*ZpK6>3Ug7L<}=OrhD(D90I-@H4;>X}D*idMmK6sw?_9HPY zO`oThq4RwyzFt%VIX4>_|&Wd=@euwLGsr6fQPH-5qf7 zY_hR-;XsWV{Aes)I7h+5xPsUeXkr(?}B zqpTrDE6XX)p3XsIn3D2v!tb+On%1R>_E?&RG&=SyaeXABkjjY9IU%%jsebM7+ohkN z=)jG7&Jpt&)FTCCVXDhFLTiAXh@zIij}gC#&@=H$_dyj_xKm0dzLv2ib@x>^tNQ{O zGZFjYF8;Nt;^nP&e9?sU4%y4y)q533DD7c<>vsGH6>E{Qm#;R282{!be^pKF=CpFt zQ*(N*I{K3qVZ0VqPA`==_L)QIEb=rm7}O;fJ_t3mD%+Y0vm$cO(d7a3efm_dF$CmW zzn$~2RW3b=Oe#e<{M-BYVv-x5lRW{3*a7^KZ*IMIbc!dIvK z$%Zy{^sP6)jHx4aAJmLS(RpjT!52OE9OcKf)n!f=&7a}>@e4b?l~fL$9a+jx?UUSY zAWS*iOK7HhI^;qHWKZOqlTEg9b^>-Di;g^!GAd3s*_BlHp0)f8@NlVZWpSU`1UWcuh!^iml~`rlzh3_*7g!ByQJ37bbabYhZ(dX@>@;Dc_04+eVrPgnzf13B z^3wJSSl?4H8@1|1NmUIR+jR>a#W?l#|SYuTwN2?s+tISt<*o z$}tuLQk-yOH-vZ1?#D>z{bx@XwhCY9zD!(ad)DX`-V`ZwS}@q#Mw4$!>`rr}5#cEw zVs3(Tly(>`kwS#=zRXbi;5XV^MzcNO*+?TC)d&<=qSqOrpS7}6+aFDP&HBCUURPT4 z-d}FZTU&lMm1ggl>~~x6Sf=2U7O|gwonDa1Ua)QL&m5JQX(LmNy`K|lhK}r2f3(B( zp>lx*O+{}`Asf~F)b!uPz=HJMz6Q}oP2iDq=1=H;>Lp?TNVqPcF4qH>0i73ooGzai z+LKr!#J?427k_EribNzd4Uv2 z`j9)qMDFxmyPsh&m{C$Gp<==L?a5Car7e{r7mKbG$*nq;)S_T&G~mvAMEQ_?r7^l; z+niTFS=&&dO9vW_Qw0_3#8J;{%+@Zp1Ddt?B~f3LRunZ}tl3oo0Ur64kAfDp#wWo5 z8P<9s8r{JNA*Oob9b?P~Pm9&Fh>2AmV+p507(>h64o4Lp9~YZl*pnVjR$zy3J{LZw zLcXIhLk}B@tx=S}dOP?9D`7Zr*`cTUB_saqgqj#qxXwBgaPdYgh^=( z610e>LEFbZEiP~J*}1=Rbrx{NpBe`}T@K%)=AF0vb&=$zg^LcKQ=K3H(X#kQm1MzGN1Ep1|p)BLhK=*2PG*<+>siby}6QAHcK zG_Y)BLoh$Z>dbFZ;=Z^P`Xt|Grrd~=-Hg}JlAV| zM?{AoCyQYUF;UMRG53v%j^N@P>5)9A%ai_qTfr^jo7F?#+S^@N&y;2%X|!oK{H|xK zoG~*4B^87+)1m8OhV)(p2pgxUp!jw^B5Zgmus%TwAd#;_Q?@L$m&`N@6t~}XO_ttZ zn0~7qe15wz}goCcnhP;5iBEZ9Spu3WJdRWQY-HMC(pnw38aa-bG?YW zX^}V@=ckTBy_VZTxR_{}Yd*bkb(kueAgiLz_-Bl~?8TY}vLiDjYS9z&(t*rwvf573 zkxf9e{{fX5v_j%wK>p=|x3bqxWWYHK^}30xAz1~~W2)&0DAN+X_fux+`bLMsP=;-} zmX2wnp2PO<82yzSP6?7OEjr1PSpOQbW5yUK)r+H9Mh~yTEd!Co?|`c3%OWPdS?IbBad&bXC$>0{vUzO$WQKe}y3U}#r zou58mSfA;2iv|3aNi0+>+0)DWHb;zzll_fs>^9ZD&tSM%E+KzQrSzrM~Cl#}O3 zb?=h^SM-c6)~%z`f3gzCkZ%6kb8_}gNug*eDsHj;Ii!V3ZXODtxmNZghaI6VjUJvO zlojb(BP^=s6DkKnDpRt%dW)!PFI!0!efaBA(O0+g`F316)XK4jrpoxaU2dF($%7fs zDJOF98%Vd3S$WebYE^m2?n7A|WTCKKy*p*p#0%Ntt|`w`0?33FB}J|R+-rF&n>AB? z8@03k%K%qIc<+XpIY!T#i0~=Jq$dLu^|wnim-X31Hn0o?ftqoyTNNhEF&m!H#J9kn zTvqb$0~Nom@Ti;zotU(DzTlehJNNwePyF*Ocq0GZmB7L=#OW;C`_0m%->%xF1LeZ! z;&!S{4cAXKOVK`Tt}jazedCiv9P~dk)~5REX^7DTf6J}r!$CI4HE@aQWT&Rz)6Gtx zsXefK?vW}ft+o^);mj}+7MXs_;|gGs{Y%gFh@TC5^h+Np$hNoJ42cJVjp5^uK9bmz z4(|{R<_M0Wq7M`%IrCF*Um^hQ+06B2*C8*6=@u#J-*{7VojhE&{K+)Ics_SMNKzCjg=9VoGds_6@`?qY zfB|&$%5H!74+ZaLMqv8k;4=e0&PzQi7)C2g86I+|4!P^}Sfh;kMOkS5g8KM^PQcH_E1&T6mWgvNxN9ni)dmS zAN#VHSjUZepk8t-En{EJQW;~(Qt2D?=wwY$kX3(G&VHJuvEWoasAP?iP(K^%u^ant zep!~@_SQJoK!z@>bh;9tJqlC-cHng^J>c~Ssa2yoX*&GeB)3S6@iwNtVFMf$K|v3H zNaC0f9Z0%p*7y)I1Pi)EO?=kdgY@vN^~vG+qxfZ#g}2gAKlD$v;m4g9FktxFzXQY; zW?AbC6fC~h4OuBXBfKwXs;9t+B-^JYGmd>SXB*GN677)M^_%PnBaAva&9l8u;=dEb zw57d~8!*U>)4*-jGZ012o&j|`)>!Db$ql;okVDi+qYDx}ypxTumEeDbm`wuM=#bw* zlf7f5?UM+&7oc$}+oU-A@2AXc7M=BD`T#Xhw~O%}B7Z>eB({KS2dy@E?^_H-GkTN( zeYx@uHs>!R`tvs7yyYibXFEXKNh)>4H=A8GTSc0cLInK61JQu#jdb0~WG;K^bodse zhf(+V z^@p7MoH>9nHljeA1LS>ga=)z3+KiGlhw?_x5?}RjG`*iT4;<_!^_jPJ`T_j$R6mV- z539(1`?{dtT0Xfcq$d=ftwJm_SiRd$n*vBd-t##U?#qd_4NJUAV$p0d^-~QSmE9IG zvw$esCS!0B)V^u-GF;@FV^Og>Qegeezj$}|CF5D*`*d09yG6${tbPfmhm{HN$Yr3V z_*+E&);}m#CMb03JNN}A72Y|xxx)bXLqQgx+UiMhFA~CJZp4y_uaediGX#scP}e+Z zYGT1OO@IQ<&jR*|X2_#s1u=U8=KZOa6+KHvb%W(!syLHCERKEZ&$f$(tKp?g)qd|j z_op6??jn}-Ku6CS3Z3S?-B2XiP6eoj%K+}aSL@%O?{)UCIwE8e5wOZFt(}=X9}BoJ za3q5qmFNE)9w1ZmaoZRv_lIm0vh&h9{Rrcm?L4!+7GF=JOCkVJ17}Ba`#CruQ1k+7 z98Jn4rUP&&(I3L^zZ+)V70WrFWncn{)B1pE@=O)1+!0A%W7C8f6>d=hxf{OW(xHb4nI_-zZ zL^>tKZYRWMO$GV{W3*u{^Y52r5myvt3tCY&xWNh_iV0h+Ov#vN6Lw3jf-E?o{^Eym zym}{d*9&Mt;%P7{UiZ|W%Or~CO76^A>-3+F>6Y64PW~0CTeQIzgTsK7SW+!2@LQ3b zCA29nKg6h>%Od68YCIg8bD5ztK$@Z8I|Fn&e(H0HRP$M>^PScT$-P?=i=;7hR6O+! zGuxPaW6*r6CvNDE(q^_VYmr;HR`}!j>koy$MNtaUFoJ>hrd)QA5wnI=DSLUeV6g?QlUEvK z)HUUapPqatrlvfoHWgk|5s^mc1dI0ST+2IB8$3$nHg&hnNX6N|nOyI0@Il<%{LWI(3v66DZ#G7k&! z^Q$ra#Rs;IswoMgY8@_vid0FMUc8fMZhqm5ry8zBFG_|EzEL#^9ADlKi60P!^gasr zO3N|voM>CM>&j|3+pwJF2?IRj4yXCmC2m0AFePwQ5=KKX-jzzzhqwe({T+_YaU=4Y zmw>S7gk?9u9{c=vC$Wi`&LYEUjugzj9^M$kUz5J)40He*9e5nR#5>)0ECtj&^3g9a z`EF0D!OFP0c&ET5fmuLd^i3Fj<23ce-_dZ2RP064BCfBOez*Ry#ri}Gvp^-WB<~bx zw31?d97>qO(YqK9;3|?y%hvzgFLr)zxz-;6`0srW`h_~x?g-p9zr$z{H<7IwdLN@*w{!eK=SIV)My=7QHwL^r~d%x1^ESCqS1WuNZMO zEhGG@qI3}vnN(gA?TqQ5KcBHn@HUt38Af`Y`dcCilcpQedxq%40RM}f7y3uYps5CQ z>>G0yPeUazv^CR6#+Q8kjoNy<7kbZi#(U*1SEwY|4!jtn4C%`uVZB$`#Lmw_6wlB{rCaC2S8%6aVko6Y%9 zeo7FDjQDvfS_#RY?%IBL84v@CfIjl&gBMzQE)=>2Q$H}2sA7(-erF+4mO=L%$<&G( z)$wavQ9(W0jnz>n#2?omi?UOJ1#z9Ca0aOwa!M0K)F{m8YfnFGu<5-K0SER4mE)us z=i6tDplvJ5kHLG#fCxI1>b;y)SX{*DD^YYdU~5nUexbLAZX7&fr=doiG+;V~|Hj8n z+&d1_uNK0s=fAy`d--M)$y!BLfIJj7tH`JMHLXcv-!nvQ(z#o^=h*F}Ay#8g3!IF) z(ax_bda#7l7whJvG;l1HDnL9jER@s;-*4>VD)y-AG+rOiKY#s+XVK2LjPNjbVYCUM zR_~v+kSJuM{+WnP7m_AckB~l&qi%>#KZPKZ7@h z4gUK|QVyfJWQQ_AAoHdS4iG#!#7T#rET{AW=OMH_^~V|inDKxkGq_Gg8XPo&;mGBI zuS3eAOWgBNJd*mb$p37MVW3PEZ`g1IP*$z2`_t(^NR+uEjagFTfwLGhAQn0^%D?V! z&jXS-4%c8XRL-==!6ng?WLdoaA5#NmqHt)apa6KQ_vW?R%Wl;zXs-y8{rMXdGjO?d zA30nhlM=Z6i@2KQ?f<&`5l(%f2ybty#1IrXS0P60+~7Ltn1?A40p|dz;o80{?tFhQ zg9nsM!czIahyG{o09JN01fZ*}uIUm_{<+QnsBHesIU3LQ9rxW- zT@!`HKBFT57I%v1oy?ZAVM1iODTvSi+DFWI~He7;~- zQ>VTYr=K4`a14@;<>)-0uBc;DE-!=is&Rw)9sfRih~mn`V!yjVdN7Q3jCKZM!`uaj zZ`orv&zIgfV*GD%Fl^Y;mdj@i$ykRzU2L4eD9>l-6!E@z5BZJ;%0XdhJ-|rRT@Ev-%+M*skRrouCiSB3`7X|RC^STT<62WSK@xHLyFXQmd zz4zQ7>|t3d_yfh6L^y+iA;EC2RI)t`KE<-nfe)nSb9CTqRZ^aZe?#nD9N`|TIe}sq z0C0%dOPBEpp+(-NKPdjPM=KYob^3FME=03W%=lWpWcwNW?Mv6*7<8&Oe&tJ`nSrsmG3 z2A#@GcnS@ueCzFy;28AZ_L9IUW5{Cpd6u<)Z?f!{HdQuCh!6$_Abnk_)MCBvQ9GWj zqIEx>#Go7xvijapYt7#m`s0+{%1S-TX2{y)n)@WDe*5=%xaT;hw3?Po~62&yC_?V2?op0RrE6MlYCcB{ExOXdezO;GeUFjcn0pyQ8CXjD% zu`UWv9BwN>fl1`&sHxwX_~VR#_wkLdeUVhf7&?D5!tk$ou)s_gG{=tH z$(`vdss#R82SF^)9GH5uhE|m-rk?uC;zkvM1~b3i7=<~VU2=uPzZYSH{d=$>;(GHA zJlV9e{=MI`bcLDz(xjRF6%YOM_!rEFkb@5*7U`Mn{(HnBv|$q*qiG*Z-#*qo_#Vak z$1bbIfiz3?e%gI_l6=Q&4oVZ^82VD1S+b3y&EkL3{^=)a0!(L_2PfmfmkFvr78+Ioq3;_Q zpoC}ZK~IAKeu~kKTkp+v={I^t*JTdBr|ka2CO|Q`^}tQ*bz+*Zk@Po+U6F8#8vR-r z2~`}CB;LOsd$Yek(5vG%oMP|sx|0h40JecO#rwcv-vBNhl|O+N*z);?$Mb|}Dq^?R zv(it@;J}rpp&KehKEoIHy&xnzITa3n%vkUSl#k}HO2&sjNqzVLaKUSos|VJ=Hs=}q zTR8w32jau)u4nuNn1HI++PW@IM$=xU>z^U0?_tv{2CrJW7XTX&gXLKJlON)1m@kvD#yAKQ|z;OP1GcgSGj$g>XE1<`4A5RZ%n1u2@Z;yr{5d?$C2 zzk&b{DgHIS59w;oOfGVOY3!P3xPC5kSx(GIXB#t4MzlH+;%{EnPUxY%^7HEu(^y?O z)^z{`$zbZv_)Cv8aU*O0wh4#?Cr$XrAj7~l4>im<{p>|UEg@lYelpCC82JtpZo!W- zWAG(jr++$r;u1x0GYyL>AQ30LO`z@pYl8p%$OH=F&X%=z@8WiM>Gy3di z!enq|{2pREQvS^%UMVp-H$Dv;qV5<2I3J2l`y8Ry=PXyL`%jX{!G(8dp#GV~R<6U`Rxe`(9uR7GB+FArRCv{Ez(!JeZB#lw~7uoZ>{lmy+A+Jy-4NDYAr*Ch#2W8ex1DpiWz5imGtOS}~+}KGb0E?c*+dKU6EvCs&mSW@Yy(L|J7FciG zVfn=b7F@@mK`$H6fNe4s!uwZq^)JT`R;9$!3~=`d>MG;h#3CuX zZ>E5%9rZF7w1Gjm#x_&$mIFH&h`VQ;qjk5K_EEN#i!56Ax^`2nabrQo*~=*cVMC_p z%0IUg-+;6OABb^&v%&QtFVKds`_FEobPiq(Oi3_)pA`NM`INr$NG%B#c?LsiPF_4l zh#Tw6nHS(5OhKqZ(aPa(K*?B3;AEm9(pB}$Crf2#PPN4vgHv)|E6HDgR-qRlUAtHr zC%P`>>@Cb3`Fl8GW8?nBoyN1ES^DV_Z1a4^2etFgC5PZZBa>(etcL=QCNMPd0;`{w z=pP~&+YS@r8(>~t+gYp-$1x$OyYXGP3j!O<)Z_4?^>`o=+jT?t({$-0zw$ly(3|%H zL2EdAbxf^q=YDEPN-8k1&Hd7*S+d{>zW^mojq7y7Mjo3-ErBUe{vr?c;`+IWUFlP^ zmoMxPt*?6s{|1{98)`I=b#?7=Arf#I%*jezFB_>FtI6ogT zhkWtd+sGPfo~SM3*f%EVlEShE?mmwbs%m0VxD#IW6-HD?yG;=p4cOozP7sHe-mL;Bb5ePS6KMyFiDHvCwM{QCL*wDeezA-BaB8B%-Zv#F}g^zgHcp`T*{_ zr)L1%f^NlH69D$t?x}Q?a^@{K#)0%ZgdDGqVi9;jl@hq8Oq!l>)q$aqm>+CpfPe-< zJgYXNGVy7oE`f5ww}P{;Z!9b3?=rw%+}1>X%at`o95m)B<|9th8shz4)WtMZIM#r) ztjZ3v>fMC3mX$ z1uWTwv4^hu`s>f!<@+eO_>c*fXU2Y6zZ(J5=ck}%Io37~xwUSip#-n3@cjQ(re)xA`K+-VVHlJ4g9 z`Er9En{c>rnim27A%+W}wt+e6k{ZH~tv?tsr@P#iZ^>}{0W^enr+gKk&A_b7w?Vjl zDUN2^t$ul6&71e^_0^mQK;(nS3*_NbW#^iPa$<2oXSYrcpUS5y=b4adA5hgJFQX^! zud0OZ+=czZJLnh_8TP1z&@4P0q?xXaMFdsZ^aJO5QJPz**D9mEG>T(ne)><9v0%$Y z)rdOn13XWI*%62I9Jh$}S}ARwAW#KP+2zOD-QZdKic3XsY0R_O{bxB+Q;#x*VBjuf zB~8oZ2-SO4Ny#xQ9yNL6P&dR49z|12_Ei-L?5B@1Uju~9HQ>#Z)yFP3$xmN<^QY-k zrG1q$$zfaxh)q-|u5?-Gp8@Nu(U3R(R*ZZbmqp7s zcbNY7{{omlzLqME4OPg*$Jy+9zsh{a&|0Ck`)muB09x<iR|L ziz|X*jm(WO&dtLWT)RN`+YR?LrC|-tFZtSUT~04a8vtG)Yp{rgoe}8Y@gx*)Dz~x= z>1m~Ecw)Qq75Z|RRLnC~&79|g49=*Vto)Hm_aMk`B_vW~O3g9(5-VV3zvJ$O?Wl=E z__!3Hino~K9w3IDwN)rwdJAh_hSfWrt7KaL)cmkE5U>tt~kz6ORbLS--n zP77AapH)4Tv6r_O_Dt9({#7Ihje8(Te^>Bp6pVZwBJ%2vE;-wLM6 zWP*88*@iX67&ca>9`2i2{c^|%RfgXcj(ML%p!*1fzIkCiCGumsB6xRY@#lQEY9@glYkuFCpXyjokMS9ET;1UZDdh0{(3 zeVVnZl0i=aF_N&_;SG1?Wr4WvnFCcMC32@DNeK(K2MoZ;$8HsFws}8tamm&(rB9-t;|%8reIlQ9r5IWrr7r{#i@4;1({dv z2Fiow<(T9%P2gF2VLV0;DjytFad?GXa$~hb5nvYW_4RESul!!_q5#gN6W zR3rQ5X9zznStsWow+V*T0%tGg`ko=e_PE?3EKQ~WA0N1h-Ija0xjvH6Rpe}CLc+~W zA`|o?y)3wH@CDvLBn$VNg$kKUTt-O_W8iBZ0QEQpNnkz%izm`6a9iWZ>{2ivr&qBr z_{Uuw#T&nE@8xK4#4{PZ3CO+>yh)P+JA-r&H%q|tz=NUFX6)WC4FTE-1l8E7-&r)m zb|a<#U<+4wxo8aEOdF}U66_Et!<@&WseyeoRrUy;qF0{1!MJhcupyp;pYnF^s7yB0 z@S5)W4WsW&g`0=jlIFq$qCg3h%sPkPGHPCE*GUX;2}!m5TDw?8<4J`m4uqoM3B1@1 zY0#0gQNCUq24&GK0bru|bK1hF zb|d~w?NIX?q=#3exmBdb!p)bv!Prx~@+u{`j9Hz^4&5=B5K07{+zi>BgW3eH5eUn- z;0ZOgi;E$7$3A2;Y-nk2-5SkkWD&5%Hb}4E6G>tOK1oP5`@Z5Sr*YSt~5VVon>p< z2UM^;0)WHlC8f_RS|bqprGk`-c+0Pn_2^#rW-+G=Xb?*w32zKyNp*d?w$`z)D_L2T=*IZ4~}z ze+Ffpn$qC9Pc$ri>ne3w>6$tNI_>_+d#sD%q_*`S#1k)sTy@Hp&i7rdEy_Slwx4p6cS&ZbAwe;hV!Fd`<=$}>IF~5bTRBLZ4Hwcn+^Ko9=kBf?~`(N~8+Zi|RQs)wMhb5?3+{9SnDA zsxz3Z=THFpwDWgOsCydtH>hm|}4St*aAOR}Jqdjn-& zg;{$op7BIC73GZ){CcBQl`;IGhcW2u;>ACQI{lvkH5H0I3Vp`Dx^@ZG-tu(rNG1?A z!EjoA`YPnxle`SZ2ol%}GEHd>G=*80ef3f4Ldr~;aVHm*-;^bw*a)jK71L>Y$3fJJ z4R1FxswFM?Zo{o^08c6rU?XEQ^tNgcd3p-zQ_|GBN(`am5p0gxyTq=it%)P`B{%LH zY7{L>cb@*V-eg}_PVi#n0n{@z7x%`%aA6D1FdPn8Y8x2j4)`DVMN9~=f_ z;V2Se+|^HKDVpCZ&ZY^HI0J7PcNK*+J@*^^K|_i_CVY;`?(unJcILefJA+h8U^oLl zQ3+y{=Bt)n%V_m?P@x^yB@o&Ew97zJ)rN7!BDRA0yZoEt2!PXGeD;`3l*>@p6}%0- ztQ+=5J{@j?5}z59ZYjk=NNvjkyD~nK#XPs>U>lJ#2(T-A} z>?8M}7u&&(*Z|)xc$D-hQ;^lY$B+RasI5*d6swEqp8a!Ft6LlN}B& zD`1N%`m+xEJpouS#o?lYQkhnE#rV)$1-PdyDZr=9d}|^=^cb+R9+B3|JE2)at*WJy zfm$qY8Aa)_L0&317SM`}%3;AgA>?m<;&HeSnIwi{;cVhnG7xof8BID%%-*i2OoH^5 ze2`k~wi8OQca%@vf*!K!i9)Hq`mG=5)lNk^OXsl!wz%A|#FI8$>T)UOl|JRk;4#<$ zr06=l%C3+Sw_Fa95$UQu(11>?@RXeS4$i6cSr>)L`46re^__l6XDGIS58xT`m=KaX+A7p~ks11xVA2Po9^NYHT?^%AkP3TH>WwwCg#mmH^j)~ns| zBYRVv5W}-IVKahULY4HfKNd|Aj9(f8 zK-M}cpJ)tJcca^Nj1USlWC7#ZuMOb$ecVd0?p_>!ug;GE1sU=8CU?r3v=7HKAU|Zi6@WJhj=N0a;8t;nt{{t{>YT*t5@^n5bKwx1$Yx9B zYcexcXEA(3DwWXbNPg#2L50XT#t@CrHG%`aOxdSEr{o|~XRz7%Vji)P+^Q{9IQwNg z(gH<9oFTsIRy*H7$ABZxqWRX%AVBrf#mOfs_cX1`zw7GL$rHCs;@j`qZ8}@I>Te}Y z`yaR`o!#RL@7ia8T@oE|^Fh!N)$31G*I8#bwqYaCbuM>5su2f&TT4b@8}DlZf(Ex} z-Z}r}q8RFZT}fEA)rHo^ioY)Bi z_7lJ7pcmfhq24*?^Lj;*mbDCjTtEhBkMn1=yvv>fb$`yO=@gr03tPn z3xxFT1K~}hw%wZukbdKU!SaDcydFQQply=oe6Il)cS%pY~a#ikFAS?Hm+@b#8& zUH<6A2FQgfXsL`p&YM@DcqUQr@MN#gqZ#xP=<_yFD~f%y!ZTFdR+W{K283affh)+1 zTQ~#?c)DA6cF*pwzAwsa=C7Jo>&Mms6a*Ch6t$@zlPKeY$Xe!9k;w5m$70S#`DFE;XS$!zmBiNP7~af0UE z&ntZgoo4#*6e;o|7lrN@Tuds=GAx>@fjEKw;T;nyeBC@=z;T~5GF}^FQZhX`M{oT~^NCl$&ShbY?5jgxrjdEo>+)BrGO57%#Fr1>e;y73fgm4}8Nje^A` zJg0Tz48X}TmH|~`;JE5RWv@kqy^9fNRZloWZkO4nWN8GoNd(Z=8Zd+TcjN-wR)vDL zo{MD`IFG$k`x0=kkLrM+kWo%!^fI(TGd7Hg9Ppl~gcvXp;LC97ymR{6b9_E2p*e0*U=W}w=aDORR z<6M9$=+DxxwqA;5?wWbr`aFnp@lLs_syV762Lr%4p z>!nE(9uk_8Q8$2^7qn-|;!U=!+d8aP+HACs*23?y@NFs_>_=L-1t{Vh8d9ZwF%xN$ z_F1Iug_qBE7hp#y)h$lU3E{L6(S2EuKlG}#+CjL7&o)Uf!k^6>jft5A>^vsN`@h(G z%ebnxsBcsd1Vp3+q)|#*x=W;`g-s*f-CasZO2ejF6lI9n^@`+`{r21tUn7kDs_;k-KQbt4sNr$A42V-y9i@p0IzXEEHs(?eYb% zn6IJ5y_3?UzNIfGNApTvQN>Gh zj0*$luGVac>5q?yX(fseBDoDaF`0vK-v)%eo0ZJNT^K+%Tpx0q_FTI}eOAgldt&>( zYB4lpwy1~W5apFHAAbl=vnxdo{dBqH$WgA<3**}Ku0wox)aCp#wWD*R&z!BEz_s|1 z^Km4C;7WO*E6JgFU-&a1S#20o+Ht9xJ=u@r^X^(FWm3_Jfa(S(=R4tKRZq93G^OJ& zxmI7Ni0RDDb@7XV@04d8b*12sD5Z5&l}oKg?ZZ7gvWhyI)O)4mM+>%}TRwJjaZh#8 zh*6h%gRQ|bOHVN-LmB{FsSKk?r&PE_8!ookswe>3+;G!4ZUtFItoYRcQE#{fr%7E9 z`%&Et=9a?lUEE9PDOY#s{h@KzX=S(^3*d145XQ>xY(V}R85C3xsQ0lwFA$EX2qHfg zWO#hr^lwT#zMT28IR9)-Sd}9_Y+YDQAy!;9Zm0J>%%c~t{VHrwTGuD_6#{cFUMzmP zSelSBUqZLMu6o#S&tH+5+-fhJCP^G*v0c|i?a4O|tyCwj;V5HAunFN!(nbYLOe|Ho z!0f)lVFnc)%PiZudl1f?!N;7-a{nuk+FaeXx#aCp_24cY)pDicmr^(R)yiaY8h1#* zf?+2E(B$WWDtR`gfS|uXv1A$*Lx$Drat|N8qCf0DeF11czRI8TaYaT7XgDqyM|4am z$gQ3SR?y^WT8z`Rj_E8`C> zs1dq1ZTsf@bw4DULMakk?P_(sT-`r;# z>%2quY|++0m7ABG-#!ROQUl8Ny#Qa(0)%|grW?X=w*Sf!iP)=*H*Kdc(%9w#`QIc^fO=?0_{? zBMWXBXXVlIvIBIfLOiQ_Fdrgo+iNm1i;6R!?L5;qzV}$?nii})qBm>^cIs-pL|Sdz zB%8ZL(z`a>vb0M4d^`JpI~2=56=PB+$anI!~0y~4&1C6kD{zPbG*!n zEaQGC{_|4pNwvpr7Fg4M{tPwK%cG!rj12Ap47dE*$`?2ovd3pQ3!k@etQORxCHA*x zL3-3G;XnZH!N&sy@FkV|foW-?_<8WS{`)2wwCJChz*-1j-GZz}>;jedg|!WO`G&E% z4CGNlxqoHeO~eFq)462PE~3g3#q%Xd0N2wp;NEr^V=qz{a*$CaEc>p0^z&(hjYZ(W>rmm3((l@N&E-4VdYv?oeWQceFSJ8%d@w7DbT zm3WZ|vYMfWn%yl?5^nGRb3F?GB!y2wibjbw8!1J;T08%47(RFx7Y5)e;r_Kh#*0t9W3AhT;0Z59##{6GS$t!h88rQ%Az4)Z5(+s4?4$KL7`ll&QluNDU0 zX~P!Mwrzoc67Y2Ne8u4YKHF0hW3M?)i8U7~CvvyKxF2P;lgpm)LMJBp{+5`i?-`U= zVtTx2a7sS3606V1aZDTQpP_*Rf_2$`&4n5b>WfikYU(eFyZu=7uc?cHspD*U*yEfK zbktP$u30v1KQ8^bD*u^RIothJnJNboUpqV>r*iI7k&7-0B5FL=UH;c9pAoHLw-oo# zg9PW0N7a+!0MJQybKZ*YPX;SBuM?fYEI^X9&+8Z&Tlg; zhH+u_IRKlCIIbAn-$O&e6v4XG^7nx?%fMh@ezsYBzq@=k&PR8}=J$gblmKU$B1H!W z(uC3wz}q+YB!#1Nm(lxcm{$_@APriYhs@(L%eV_JATlrjHaVl zv0q|~AB7{F0MW#+ftjRI$&vR-@fKruUq4amma!KXj*uvhN zt|cVaO(WcYHDr7(Q;!vWvC)lQ{<=oaXdGpAGfv+@N*gxVWq^cHd-u940*QrQu6$zV z-K6E{W1Y{`e;5)FS6FVQCgxQO zWG~uSW;x~jEP4L<+?_7z*4?gqdKvqE`Q3$N@~+t&C87A@VVT9(*xL-=xb{tRt*)?U zy&p*hhF34DAI6sr?&uBR7{d{+syX^AIUFLHVcy;O|0Fn&2;pM80@;nhC+hp)nLCTI zZ%`_A)i7zsq)kgQB*XjF4uhOCK^!FkpOBkNxOC#EveU23OKlx*T4+~r5h2xdDP&>Y0fq0h4r0Y_# z;Z7ITE<@|60GyhysMp0LM5{|~3yLFW-ga4L*NggIT2~v`b&!#P7x@ikK8A@t1wE6d#lFXXVdNhy5t=0 zH}cW3%^^HG=_ZAFu{HDQdP7f>xfvXblo-tOl@88Q^dQ?4R{IZc1M>=yaY9Tl5Ko?_ z(>Y#HFrHpnccqq&Fo%y!zFs4+GHKS2Rmjm*woYVOw4c_)93wwcK3tU;>sbn7VU#O( zRDbQdzZeqZh;?h%!0-bGoUwf3uFYoN16%hxFGCMpI6TTUXg^66!&7zjdEUj;x`Rx` zCuvpvh^lyrH^et-aHc8P@Wj1q3enw=EUz*+wrKf8&m?lVtL+H8th$#}uRUBUMsP&u zrZgJ2*g$3?-OG@eF{(I!GKS*hIrHnv7C~vFAUdW1!kcm>he?T(sVRbY;+J$zi7RSu zUkt~s862;JOcE1~TNG3VyXr0&iuXq9ntDHVo^ouCyO`@&dnpYEPPmj~27yVXxXfB^ zE%kG&WzVI?O^h#HmDrybf7PvKQM+nN(YrcP@xDr14)~p7OZ5FHy`(dr-2@ueb`^o- z9bYmXbRhC5*2o)HoS=ds!NvH)p3kPdIJ-8u(MH~d+hflq3P&O&%9*t7#=Jyx@-Ujp z^Y$%^&%U_BS+0L`zu}HTKGj^gG5_94i$5y4Kf2(1v#i&wp7Cm=#4=*w+vN`ZVZph; zp4)}`Wn5uoBK{U#O!95|JNi!iqo}>bZ|>u42OZ*#Kuef!X5y+YqXMeguJ4rX1sHV? z^SV>zM@x{x#Y5=*Mop=CZ%!w>p>aW!5J_>F+1!fvn?lh@Jz3kqHXRC+;C5V-q9bK* zQ&qKtBa`B@LgC!WWfmb<>apdu11cEH(h*Q2l!}wu)xj zn3houc4_2wEQ>>|{WvUt<2xHD`I$!7u^*Z8m5C=;SUy1#$}G=Aohy~}Hp)0*F?QMO zF0y2ZP$h^5(|somb(xG}zw}#Fewfq_2aZ05iw7lEt}W zl)k;^QvNBez>d?XWS~yen0u7#`fyMVck$U#w?n&|-VlKX14V`0!=#%a^P0-o%S#Ph zZQS}W3a3P-lj(>&dy!f7Bh^>4cDXf%liPTcQ*Ej3dRy!{wAcE$7^~_BDvXVfK8KGg z@j~L&+`~+DhU;W--}aPiZv*l7ALI5s!ecjBeW9N};mg%!Pzy+0)~qA2g>$+ktwX|u z^F*l0uzCxd$xqNa8!Uk#w?2D?o6mxHV-8hG zt9JrnQF8CLwRFl8wHDS&NXsm<)IL|OyV6-^74B~yackhP;0c}LH*FS2A)5sz6rHo^ z<=0Z!^>f<`Vj;{3yB06V(%`AtnRJ(KLo~F6^z%?A)6H&%0s7=>uUxI@+dcWQuIE!5 zJI3r)^+|Xus7!8)ciXzBQvVTaQkgCD5 zS<(^s4DxTv0a(6p86D>(3F;qG9=o%Kr<4_%UgXm~#+fPE`R0DxdE;EqbIwd)CLFno z?Oexjr8)*~4%ouCm4dzfc0O`4ZNt4kTGVM17BOo;tZGp{=XG8vv1Zmv0XD-2rz1Nt zn6Gqdpc^w8kGpi}6uQRI#toscXcwH&4w@_~m&g?8CUpfoay0|hIM=f zsj^`bIfS_(3-!HJw_tc;*tyQTDsS9;Rh#sk)@uka|=H`gosRX*+re2?ck(nnQl3crbg_`6iJM9`ZuTtl=xtP0YTHIIy)_&K zV`*=On~l+H{S~(Y#GR-?U&j;C{7ceMcdu8D6GyL)~GnS;+d7lsmFcH&(X5B-$Z+LNSUBB%xE+>wlRPFUldezXBwPvNZ zurg;;2953T4XLd|PjcVgm;My01H{ay+s2ulbr?}V+IG@|I=ZBHwHa6#(MFtyw6|)x zAS~PV0`f+o#L6Ew6SU4T8Iw)r2}V8F+r8fMEeaY`e%QzSYF1SarGwjo(Ys`{!xoR^ zSf(kbx)y{9pE_wH_?mpL?Ruwfo|4SjKDW;1jJ|V(1sURU9XV~V9hk8}i{41@J0DfU zRu%6ggt*Ke99!U$*B^={7S1}sw3gMH*LdFp zmKsC4Hijj>2Z9vo`jU40HN4Bf`weypsrv3hC~uqvFQ9?6k7|9 z^lGVPZLa=qzbc(r`+ajSA)z4G40od%Rh{zPu!{}yd#0>2 zW?5S8kgm42N|e`@8E~C`Isux1Eq${bbIl5`V5%VaP@SmiFleh;9)`oJ&YDd&7_V`~ z_B>_kSSO`$#a5yr4Rz8WvxWUWS$hcx5K<2Frnk~40e{?BLifYgiVOCthI^WlNfISE z)P(iZe1etT%9_=USV0x@dvCD6jUcvJFH1|%>~;BG%7uM_z(7KPwYz^ulfZaU8y+3ckD<@+u}yAY^Czfqxx-@ofBK)FZs=fDkmJ7^EI~=^C z;Vy6Vq;;r~CKHcuLpXd*jd>iZIPfkm4?oErlU; zqkp$=IUhiC#!`*}=mJSCY##c)B)%ck2oSZ-8DBO>7q{r;^{DV+jgF*NO=Z9C;%d;? zk9;-RQ=Gh}vj=90>W$8`{aFI}03TemC2#~(dq8V^P87G561^{N zuz{NGHk@*e*=c?c!-27<#>@b#s$Y#{Nw^njm}&ZCpX+uQpI}pBly@eHu~1!T@)8JHjNdC0EgOC}JHXA;fphKMq$uLZFnArx_7sU^dmyYS zdCsiLS(or@x&in$8?TXJh*oVxC`FW8-0%B*>7qlynzMp|sW zsqfVo2PVL;oHNo7L7Bkq^X^B~Gp0S!QW8r=kN#U>KBD2{ zlvvd#vO0zwMk+n@q(pq%ifuezpHce3&B-nrDKCnr?vb3z;i^w1o|@aIJ=Y0vKZe3* z2?bEpeJqt30)+3bJs9kHNgf6K;90fn9^FVQJbe9JW_R_-A_W8WZ-BYu0^I%B$&i+O zrA8_pvyZXAgSFQvP#V``pdURDn9zQh|#3C2PAI) zGzyedb2+$|8v~KtC=5P)%`nf&zG{)D(vG+LNieIQw{oGxFhek43+X#zLqcHQ^Ew2k`28+$e`Hj1I(o}nDcUam`xqR+NJ(*txOZMe*dFKc+EynFmh(7%jxE~9oUds{=0Smsf=6m}y_w+n`y;Xn&ws#8>b8|k zpT@Cq)2EeMC?(TZ4HTnWcb-?QL7W@#vplUPf~!9zX`APEmTX7I(zqSH7+m6H&~EMd z&3eE6$Yf+_x$>F*z%IX#kQF;d-2l^WkyOqO!Q0^0^#1+Dmc-H`6`B){vWYkr$TSxV z&H2|-3r#jrR=?n$jvWe8)rvXRfSo$>sfCE{GJobbK8#yn3KcVbAzSoyMhcw4D#LrH z4!9R|VfDQ*Gs)Tc5)obuoK~du*j5?^^iMFPW?kW=zqD=Iv#ijtMO@twKuHjgt*;U} z8&92zbxy+P^vGJ1)`xs-^T8D<4$mlhIy+#N#3m5+9zzMpJ8RZCgp^Q*ufjlXeM2U4?A8GMgB z@9+*Zd)qBGVaFul2baQldFD2+=8jm-!(f{o3t1H>Z))RcG34AurOlW}Gw?%XclcG{ z_5d2`+1{zfXubDxXdHQRMxEYPWr*Q+k+0I#TH_Bk3+Be!; z1hDVKN7CyxAU}7_F zdG>^$cV8Y6{8g?0u4&vc(~iUO(>wcd5b5 z8(gWX3ZP*wu24#lMU}tJ6pKZv_#tg=pASS_TjTiZTMK`4nsyE~E^qBL%jX#0-LTm> z1FU$X5Aii4Rs1H|V33-u`l_03tgWHDi5dPM7}S&FF;F1bTCC8|CETiJJf2S)mNo5C z?qa-CWSg=(?Q)t8IayPz*^Vym)D#)WuDVKMT~_gyNt zg*BqzhOk=V^)EAFCK&4=wxkt5)FhX)c*jsE^`J^OrIXrmLzMA$f;TS1-O|S}4X-?^ z+d0u{mDA3pvt=bt#SR zuj(=7IEg#uf}IR#3R7yunbaeZ4B8%ZvsmTyTW8cdT5@vvb=&bF2Ie{K*WzM{(TD`c zLK~lY)yd4r$cb+I%^mt(&rAUZ#*z$K<^TjXk~806W^{ zR?pFj0o5amP8V5Ymgjacr(Db3r{Caqx@}#I-ntCj5Z~)KXDFmV*9KAaXzdSM{$%g? zlhL7O*#aPlNC_rj-A+^ktc;}~dk(;X)>*Wl!ntBB%fR5q$e4#ZA^|Uaw-XIg;eVp9 zUEZ%m6@xn2`57lDitE~DOm`6o3yhBn)NpPu@r7_!pGh7sR)Wv*3b- zY*Xm@kuQqx^`5Pk$42gQ%Ar^Fg~{Gvrq1P<)YGx)%*DkI{{ljQ>xyiIu|#pq+e^#SA@nipmiE9^{w{U=1BL$qR(?V(27W+= zHWK?6yg#e^g-+b#Qup9<>0K|2k#i!}-+Q?K)kCzsAkVdg;Gt_vgQTp8#5`B^N%vXM}qq{zcPmU{+)DCYS$pZBQ^(C_u+OL?(s*{ebzB z6M$K_KgRs0^CUdc-tBlbx&eC!$Nr_1`RU9SUjq&r^ha(@!~p6r13pzgbv$WdY+Iop z?2Ml|4xbj|V_0(O`!HI&gm74s)LblFX%5Mt!s`0Mh`bJc@In*m15vJ0yWQ!ITaw1~ z`{|9H?2WvW3!YV*s_LkW#0#$SySE+;%l)-cpo*05W&Xqe`xas!_*dw?dC{Zcumath zM>AW|Zpi`E&i$8__vAb(hNL7m{`EpoJ_}D}vDe#qI{P?J0&iG$qPV*7}Gk^<_QgMtB zW`qJS0}SxigymO>Gano@$pQx-q7g>H*R3fZZ1`-NA)t)58@7fgdbVPMy>O;RAqrl( zep|8n!k?MnJnrJ29G+u{6|SH#A7|IwpzGj%J{Vcbx#LO_+930cYJ6jV@x@8~k|zNB zt_cB~WOahd-8>;IjGu0bw1o1IBZbC)Eb@KPZshBvQWL(}=R1OUb|F%q_e7D$aGntR2ky3y9~ z7DkJdPZcL;XD^SwDGeeciwgY}0hmlEVGZevUx3nSJMx)2??PwDt$O3rCKG#ur_fpG zp}k2Y&C{Q+4Carq?Nmb6=MgHdzc--}%^yRlyh&E`#8u1pGcG3pva|py7)W?jlrWP= zy&m%wtZn#(w*m z&WWWB52x4VL^TEo3{fgPT+AriQ9_gT$2*3Rf+CB-nDJIc$`%`FL?NdM;t$yu~$vWx-)!!jV7qImk&ZJS#_?t*mPn)@J$p7-^6pqMQ@ z%qyZKPyyov^c0i4J<*)4%XKH6tAu01`p}ORMcBJ7gwAVChgrztw@y8kRs#NlVgSfs ziNtG6VInPnoPM0i-QDQXWe6<`@NbELgM@MtV-J8a95ABIirNS*+BTJYjTJ4rd2lftSvBN|#3 zlP_41dRVgKy#2&ect+mpD%DMY$D1nCHb(|-8@xy1VTeDpe3NNPGi!y?8u)O*v88NX zWf4;GeVB79G5*sljr^rVA(ADIw>r!)E@+{>FG1R8AF(Z&!$eH&cqOe3xDMGD$ci0d z80b)%5Kv-;VSh3+ph8LnUttOm!2r}p2^E-tu{0FvGkF9>2$09MsY^tjWC3lc|A17S z0PU(R)=!qe$)xm4ym+5k-+LG|QlcdG;!qbx(t$&hM+Nb7p@}@YQUf&gmTc44op;&J z3;@x?`=v@4)BxF7-{`%leL0(PpjZg|3o0DKT#OVcU&@|EI_20vSY5s^Thhmj&9xK$A9i>#TD z9YP<(G9oT+gVDJD@db3`0Rhk9?sXHU07M|o?V?=bCGE9q7s1bqAe&c2?QH+UBai{K zoDgZR-Q6;8VV=hg@QNG#%dXuW_8$}U`%^s{5vsn-9`ygiSO5O`Eufmu^+hHAHpRak zy?bXx5imd0kJbPB>)-#KO9IT?Ep^sCD)&DY^?PPJ0k7m2GHh{=L?BQ00mM zaDTg1##Lxo`|MBuXwMT^k)~4f799AB$1cFjo@C9S0tdzJ-o=DN<_`3Ze-FcnL=7|` z_zy1_$aswX)DHK5euMJCpaUgz{H+}fiT{Xc<;jF`b)MJpy(97aVFqeGleqaF{n6il zC$a+wt2Ii>^r{`&id2OaRD6y~O*$S@_7BJZJQf=|K9h;s7tDXo$PGBRL-8;tlrgG4 z35KAY`7_49Sb9&uV{8anaO2OytFVYrNAJdvChl90^__+hWk#Aq`Y(0)R$%=k7;niR z+#gNTTfnh-$Kj=yDxMskiUVp)+cfv6{g+S!`~QYX#1d2mH^y^6R`N4{wj02U<4c}q zMCL>YhBjc4|9w$=KW?gs#~71fLVN!29r0-ahd^4kY2YvgJ0Y5H$*el%9Ded&=52pQ z#3G?r*KhRuWJN*%Pxk1D)ESM@Q=ym3g#6X#aoPdA|DL;N3?d4+rN-L;C?)tgSE@R& zqmAkq{6N{{ND^F7qu{uJ$v+upcjqZm_n^@5;xdN&cl6_V_Bhlpz)P-Wh-}ATC0ITb|MRC

    eX2O$>`RFkV zEh);H?{laUd-0bn*1-5l)Y`<{_YMOrC*Moj=O81&WTE7nE`f}>3_aMHKLh^O33J5{ z3bT5X-~V~4gh<*%6C2b2Ykg5<&M@E!G#SO#WHhNHbA0bKnzJ{n6lnJjd*b;?pH_M` zp|IXyi!~oYi5-+4`7b2{Lk%eK*A{%k=-@{%xE7^NX;2U4ej|2J0KE}WMt8HjiF{MR zvqkem(cssDLEL1%#r$OpU=Np-uHFQa`nsFtoCp~HgJd0ndaj!|XX6|0m|aJ}{@ucy zeQE!#<-Jf|*J16uzJi~&*dSp+Y8~IrYt!$!nnI1f$OFe;bZLJd*E%$BaOnlg{m+O2 zPdS9RG!StqyDG3Q=th1$a}W=gzQ`?d@#rypyfFPPS1C76 zP@^ARV?9^DZc3k5f~!p<0{rC=W`1y~vb!ptpM=UbeUtzZn`iRkKn0%p1jaj~>dEO| zv*_OS2JVF_`6mvp=e-Iy$>%TFKCFdilOtfE*@%F@6r-HG_M%e<=RALM|B#MsaWj*-1Iq?OQD zZ4pzDl;v@>IpK7ZV^(J%HtT;3*zC>T5mW;Z;D)IcI?zg);PkqEm4_b#ysvGX8Z*J< zv5q&?=nxN=7;vPu9)2Qjwz{>0hU-U^5ke~S4Gw0f(;f~+~`%1e|eU}(jG zc@y6;COrKzFFtZ$?gZe?Mzw&Bbo$~qTYc@(YY~BJ{{$?YELras(o&^#^-2{PZSho{ zm!Uo6_6a}sVo!=;ENW=7aM*%b^LBOkXr^VY=P-$(Jlh zBheG6YYG7KsPK=Fqhh7hK^2Jib!)k@~2`1*h)^CSAo@fI4g^_ai{-3x&rsb5je!$L07hEh0-aFe z-<`mYu-2f?DPdmk9gWH^CVOj1QCBncqHVd4VlJ=et8(LjF1|!WP;Q~{@GK2RwL8KX zn#E&G(C`L#ld+$B@*)E)T+e$K%mL1$=l45l3wvtM31}9lVrCd+K%7nGKQC_q#$gKw zwa$(cJZT)`=?Csw0c?wB6Ce|ETGhcm`igTMb!VzRr-^EJ!F&TM5iL?{Y~GzMNC(va zon*HIVYY-2Z*yKG5Sfe+mjDUyXryy2Fn*KB&;2JVBUnfBLTY@lMc5Il|Coxq=Mexv zNa91rc-%V^qsBQ>fUKdifTu{Yt!RgM?^TI_Jtk3z%y`vPO>f5u|7#~b4FIuBIhNWO z5p5zZT{ZZ3()zzF%5Ss9mkYQO%~j3@c)u_shX2FbtS!X|~)$;~eHmDR~X8y2&Uf%fI;L zpw9pCw^8m?-`b@ICW%P0MPHW>Njy_sk*TrC$(#I<&f{b3?JsuncH0&vD*5G77*9EL zOcEjA%xxAQwJyi}NfiIZ*eGZY$H=;mcOjV+pW1`qE~x6dD# zPI5?*ycjx-IHd5TmV+ttWa>~wejHYH``C{sg(Pk~D(sz1;Z137!XREmv-qHgvH z^SXC5&SAEIfYGW1uZ3!`|6lU9j~Yq>+V5-3eT2kYCyA7=oS%K|95K*d!4_e+(5;|7 z#b8Km|e&f1FJn1tMNcm$8t3R@UqyH|f88)I0PC#EQ2Z%5z>+rEjkY*5~|Z zR@28zld6f$sPOW9Jx6k%-mFeiM@CR?_)0b79mr%;`Lf}q&4=zb{K&Yr&rJv4SmuWQ zcgcK#FpmXwn{*ObD53|;kz@MSCy+MBCuN7ru`_F`^E6}m$ovGP%!%af)v{O=4gfRy zJ7l4MHut&Wxt&-PI3kHd;uz1ph_WX(=O8>IGhXtbjqQ}7NyzbV(#W8jQr_(n$^PYa zb2Y%FYOSK#-*=zjQwPB!L{3-RESLn~IPNPk9~>O+Y;tx{;Bm=Gu;~}llk?gV8v7?| zuVxLz6SvN6vh3Up-vYU1moMvfZHAo}3@PhoZRE%FtDf391ybs=b@fY_=gjMmX7(e| zDS<7GwRc4`auT&ylsb=R?R_`YETYQ#ZNyJ5*+5xXI&9*jCAaM zM0H*~u2C~OOIx)yxhUD6mY~jUnXsGf<_h*( z=o-(~pUL{E5XfCgMBDKvhckmk;Rv1C6v{(iyE^>3e+Bx$15rk&UR|qI!E)zeY5?u1 ze?nnu__)aVH#$#ow*l3Jv$&i`AYLOFniZGJ!;gowcT!(2#h@-OD@Bkt?wj+4I7H!L z=Z;L+kWt!_SZ(%eS87yU4Ia8i&n@u;azv5`?N^*_(TesLDfD@w^t$N@1dKr{ z?L`U`1|?GDEQo_ArelQkWxLwCqfd?P59C^->F}&FH=K91Ql1(^UB7`Ita<81w(dNL zL|v^CRI?Z8%JK_tyO$ZyCir&_x4!ZrJU@ydOGgG&R#sWYjm`DK}sg zUB95~S#$J{*>5-phJQG{y1On4IqK`;=x#Xe4VPt+Q0T;s1n0IVW^B9N~>d1+#v zSxI-mb&m&hb#!|BdbF;YbsOy9!^=zYw4=|XUV56TE!iYg83L_~L6dA zTvw*@phq5P9~M zz^e)Es)oRaw$QsNPEDBHT`3SR@rALQYIEcEZbJPB@rLa?Vq2USN8p@baM<^Y6IcLy8n>P4PB2`a_uR>;l(Md}U*zdZ2H}UM zij_Qkr(kw2js8f}Kv53{w(ii_P>6#&faBigXC4Di>>Ida6kgAf!a{!VnYNYmnf3`%$*) z^63$!HL83}KbuZEGouHV73>s2LR}m4249mWwxfBN1yF zV~eu>hmWE+PvS(`3=<(8=4<=Q*&GaRAvIo-UZ0efI%>GfZyTOZ4A#7n(i-}jK~NmW zuWIxUc>WR-Fr0pz9^D}bA`PPXC1H%vY40x9$cIBAl|`zyX-Ar3yek$GmMvA#m_dQ{SRDqSM6LIcM6m9uSRf#B`7w*&S285 zdM%6N$MPc}c$R0ChLsRcyPM?LmfWkQi6*1QgTJ6#tyj`+s}k}Cds!59$aKi?1qwb~35aE}A5}g!wR~VBKNlYunn3u(>cz{Is^c z2whi1&#`Ct;Lc$^*`+8{D=lMz{`QKF;O&d0zQVUUF@;%yYXk1P+d^-?L)1>fm2_jI zv403k5&%fbZXQ@G!_Rtt_}KH z-Krh4W&fvanlP~|$Aq&(*2!%J*W^(AWw=Z;6WlGBn zVH2pSx@JMgJ-qUwMZ!W_*iKfu-T6zWxP)1PdJ%erGK{m^K~wsiW~8HoIZl#1YR8Ed zpBZ*5G0Ryv%39YqxLaXk|M=k_$$?mb%eZXl9iIjUd82q+%tab;uuL<%u(E({TecN4 zTcuXM!+OS%m3?OaC~M%bLHs<+iR?|}{sq+qS-(U8fh4_|Zi6FMX5ylxi9ys8}l}%L`)smv1ScSEe4_sBEsRlw+_=$*FYGtZ>el)55GRWovBLj za-6j)7EHOoe3BQ@v6GNTjXA+^?wXO)9_>;@hDw1OjcN?UBpVer((=B(Mv~u<@+|)} zw?`342BHNRJLL&5we}Y7;udd&94t4ByC|BqFo7{?(o$Cqq!=q}n&Wkt?DniF4D`zQ zXTT3)@p(e8N4Ay(IAF}8Y@)B8=DR2cLH4t=CwyEZmdVs?9KB8g|5cZNCkO3lnHvu8 z%z}tQrc&E4KFV;*c4`V|FK7xYidGKH3$GgrKW~)pn29VmEwqyjvv78E4LIfzrCWlL zVT454As4uhR6u{PQ6k4Q1;tAYv{x<6Tz(`syfwHlb09(~1Ge(Lc|0+NZ@o=B!m*b{ zlq+)9vGe(1ZSF2IdB~4e0vT6*2o3HZH#Ig|5WY1k1jvVy0??Q^q6KxSf-^r(X^#ct zG>F~%E__VTG&f>|5{qL-b}&m@4T;*4uk9Q};CO{z_CPK#gu1+Q2ujf`8K~+)AYV%b zUgocU3?JUJmcj+KkLQGrrj1*_pUv8(C+qAkGe)y?GI&=k@{Mlmu)>DvI!c~M}c9F5~KCL!SXJ6Dx?L6hAZ2}!PD+|>^ z1=H-PzS=%H5{~PoZE-tuua&re!_h;d^zqqW5%0w2D)W878%pBo5{4tLGD!8|iDjYv zsng1a=BtdI!SBT@80ITa5EQx?E?SIN#3Pfq1SJ%Y_+8~H}=8>np|$lE{$jt4AHx9@dEIt~xN}Ss@0&yL^E-;Gy4-5vMs+^p?JFGD7T$Hh8u%XFFxK_6IEzSG^^O-nh&I3AgNKsK-w~caJ&} zExB_)znCzcX5CaKI1~>*FLT4af3pu1OxVkM3`^1VtcY?sG{a#lrRB%4+SC_7NGO!# zcty`)rBZyzqhHbC+oVlJz)}#qg+}up8i+LU8)%04rs>uXupN$xQDaZPe52>bU!;deCeYEyJtU- zm)LpU1Xar;5LYsp+ChdksCm6YawkQhzg#xdp@4xYQ`nGQ6$3d|H#TzCtyqsKI&)FB z5^gGUhTJeCB5_e_2n6Yt#t>!Opn4R*sJ|Yrk)gb8@7ga^Gj8={vx;8tk9hCjIX4cc zQhLhdKUx6LDfc>3lS_gjq^F6+SlFlTYy104a*`UN3PbuFn8-Xs5XBD#%h~m) zleOCi7ms-m9+2))5Ljh?`IxhjI@K2^pJ@8r5r{lWx4}ciKeR+=y=gOAFUzAn(@vka z3&)bR^Hx5Vj-tXt*%|gYVcEyI2VH+U0)Ap2=Du9MS4wI#z+|s1JpCB{e%4T*3Ojfp z@guI={`kZs{#xV`3nHN5N66nO1XAk6i1`;gNwEvGmaPtx%nR9~<=;XkyCtxDWm?k) z{N^)2N}y)Z%K*4;CdrA2NzoWlHh*?f%~g5jpu5Piwyc%w-UlJd2T*As$pzbFwCzHF zcxUn!6jXCj6p2^F_`Wkm3_4-D?mP?9V0=Y3w!&cbX)2NP280Bd75Sskm&a+}Ry&pa zoOjMXS%z<4oFBNYwMQDwAtPr2*a(%VlV~AwrzT-~>cp&h*Xd!>O|@H|Polb-wNUC1 z9Voy3ddqJ1gxAPm8hwOpd85?l+pcII;H^1Tl1Dz-*L>t*RjOh8y{tQ>z+C#_>GW5j zBkN&1IlDslC#!>Aj-1W+&%vhq4V2T&1!rw703dK z+Pz++f}Q5|K?9>vuen>g&`rh6=hm)e5}xp5(w)haMU9N-IZCQV8m;tXOspgG#es^& zm*u>SXklo=ae|UYk>MK6LJJ!m!T*P}uMCT_YugnB1raF)QR$Kvq}u`k0fSBzq&pQD z!T^yTN*d`#x7B#x?i7`l|E1uJvI-zB-qT+Hi3M zYwh=IYDXrc&bcNtxo|VtA;!9*UHC}RNCYaF`D>5je-h*P;+-Rd0mqnP%6BRblh*Rq zy&RgH^}hKUj=1^r!Hf@-!ey~~{>=4TPoWzPG7&m(c3DEh`Vb4@J_z%hx2=;>d7kW9s9d4waiGCLJDv^3)O{xKorT9 zveJhzpG=J_gw=)5l(fE9<|i!-f&|;!9#@|Ds<57!_5eBxA_TEF^&ft4k&RNYH8%SS zZ~JIiSGyF7kurX}9(V5t(s$)B?Ix9%S7q7G3g@V@@KRe$?aZGnIV zvEsTQo5I1RA58_UuehU^&#s7+s{kg|Wcns47XPbx+o5;%iQwSQ7?HC+fOS5{TRdy` zYr%9typyXG(jN620V3vzie?m=>8n&@h-*63t&dn|EV*57l>E1O2eRhe1hGoft>3vi zz?Q^9!3pkU_JrMY{ieXoS=7JuGx&RozyQt-br33=>>|no39f(H1lLI#d@HAZ*C#tw zCjR-s>0f&&WDdLJ!rxsuTi;&>Mio71=JrGS^sxW+-=`ms0M=?Ygm(L{iTL~DO8{un z`xQMpwV(b|56^%xe;7HZZ~DXV`HsRpy;}RL2Ic?Nuo4CEqLUQ!@4q_iq6IFyBWOJN|1u%QpqcUYXO-$GfHpnL|2IIp_Av(hK!2Oah}r<}=2=ai zpOXO`@*w9VtH>h2$Rx4rV}n}*KM*L)F*=a+U4Ykyqn&Q**nf}=pw#b400>~rdk~;&bR@u05QMZwC8GWi zh8TFxtU zJiQOAmL=%j5{=Pe#u(IBi#n&j6k&w}qz;IE_JDwUVy5)qx4@sc$PXSe0i*6q_X@zV z$n700PG%Tan|wv1 z;FVJVY=}gc{#j@M#Zavdsy0`Qhs*FT?L-&7d5buu4mhEHKh6(D409;)%x;tM#mqcq zMaO!J^%g?!AuT-iH31y;CEu#Wi$~Hw&KfPMPpt48qFew78)_aqzD+0Kpt?Sy zU04A{^@wc*BC(i5NBP5NSjAm)G%_clb_BH;$JlSl)I6k-R*Cc&C8#A9XLnj846o0U z(M}D4a6JgBQn4_ftd}5XnZrXUP9*C;br2Y%qmiKPA2s^onLcvLTRXU1 zngQF=$0&2sIRKDl?_9jNdE$aEV&1>{ltT%F-(dR*8L9v<(}U{AbXXYlAPj%qMCTe& zxVU(`8-czg2*cCS6K~+(wr|l^e2Bh_aQwhTW)rs92ZLG|^>XXA^9C2sEWOKY<^*?9 z3DF(*k0v(P$=M#Pwh1ofw^D=#1EaslU=wzdCUT+z+XvagTwh|AwebbAT5UY|e%u3c zoDBllY^&`_?(RAu0eCApV|3nl-AyTWo5w~es<>zIM*J%*tZ$bsjIJifeAow)l?4y?qG8li8s|nozV!7$ z^~m02xatk^zN}SuQ`ou6gcOpTv$ZCXS0Ql|pJ0)(>wcJi_84TBBjZ zJyh;Cj*bnZ`1E0zXZb>HqAji_%~++_Rg08iz38t!C&%8g^$~391SFpvafSmaHm6xu z&77Sjaintr2H_!Vl>+IrT`{}ikeB&`jd(M?7Bg0MYC0x>cAzLGQg{`0oq*{881C%^ zTZ|(x58G;7M+cu}YIyZLv8(~6X~JO~AWayIU#Z8GsomI>X z2}(f}7ALYoR}P#bd$3zk0ZPh_FW|T_ORa0U4R9Sp9^w$(0q>H*j#484y02m&wYvE6 z`C#d%S%)KpnLeK1DM}F5``#o9HZPSPaDNJwNP~Z5#O)6CyNtH)2xXBI*UImb+V__{ z$Qul@$mpr#1XoUk%PqUh(g5mD0+fDWIjL>G*Pft zw(WLKW^Ri|>i6x3gm9j5E`tg9tXOr~`f>H6VZ3|Bh$DXwZ4>;G9uF})0DyTB1&|rw zOmu?XDL&bzNMH{3hs|@zcCO7*_G|D!!>yvzahEcA!VM;5#q%iVW85Kgm$vd!7mTkU6z(=$o z0XxtEud{*+pqLV~mDGkAUU&CJC|`1oL9G(8tTPfo$A@4W6dZK?=qRLVPgjEU%^NIA z@~o1I<2RwCeu1Mj%BWP#)OSkgY#2sy}4tU)}sj3!2gD z^ZWxs|4)7MV+A;p!3DFOf9P`mmkbA;2GDXs3#xm$AUWoL8-OuT3Yrz2a|{3HZlQWf z1)6~nN$X!d{_CR-z-g-&9E6wuTjKuRf#38fKr8rfeWO$ZIzdkF=n@bYf4%3=k7AEO zv-|J=$xZjyO8)*h2sEycjALCOE&tPOoql&8G$SG83jgnFUPCa@7k;Dm{!8ERp<*2I z%il>cIH|r{&p)U0J(%E<3HG0t-OMcC89HEk`xm9oMeGGyAzwG+PGA!}n1Rhis`YH- zV$xugs;^&DVW^Vq^%Yk1ZdA)QI{~D&@jD3dUxYJ|SQjE)+(j)sx#Bd7ZK5!mH*K#)GG#w_8t z`W_BUc=64@Ceo2`j=o??S0g%1j{9^@#8SZz6H)~SL;0|*t#+uCYU;jSG&loj{M_As z#^BCHfx&)f9(A<_TTNQYIfs{nU~cMiwQQe9<4PB(s`2N#xqijACe*ggk!`UJ2lYaJ zuW>d|mq|+5zTGpxpPO2Gkm)A0c`XfIw&V)MgK~$ z`9hC+>R+4paj2MJwyt1vqL&BaY+ImT1FyQx9;Bxd6Pb%we1omonQ=+*Y`IYH!uJ9q z1lS^HWbHf@Aqxc|jgfIxY`bmzCX|P_dE#H)1HZeDo>3c-!E&DA>|tVhVC@A^buqK-JZgBT3G(~bOFUJ(?w}%o;wS% z{9Wju*8%EbkvLHzDv91=F;b7Sc> z7YOOk7UtI&F@Uyw1D~b$3>Gf~b_X}wYaVnbEIy6>8Ln1elV=yt`tiG$+vu_!$=!Je z+kc#vj!4kOl_QWxIT&+EscKNGP#q2D?6JRRwBwVDFmZn6O01OPKP#$F{nVHrN3jc9|7+h4zLZ4#PAByB<%V1!WaiVA< z$Ji%hyaEg3+j_2ki*KcVJj<0wM}x9KJ9D6Z`fazVC<9=8ujmyv&&KS|(sY?LjkA!4HPi>+!F1ii zszq8|vpL-J35=?=*`XCizvn3NGy{@RShr$H{K=CYJskA9Qp}2xPd1cZ{2kOR_i*!nBa5AgZIIdV{}z^wCs^Ot&>IJ z>Bq78gICAChPOJc>SjrzNT?kpmVKT&{XnBasdHsbv%yexun`)U1y!SDz#oH4c00hk zSWN<+w>M)<08p}fK0V%=LVeRkH+`LAu3_a-S8)`Apg0HSml>R>Ng7JhZQbbzm5a$n zZBb?{^jYB1_+C}p%zdQSv?^a`%wCwX5(FD%mb1kxON<$Yz)XsY&ph&MCl#dn-4;rL ztWDmlFK;(h8!uY?&21EQRji;P2tSyWk->V4jUR45AFrp$4AXesy|u;bK&S=o*}WZ^ z=W62AY`Vhz#-9ZxD>STj%D(1@%~1Mj0)`uXJK^42x~VCgO$bOqr|A{AqLMqg5)^%s zULqxW8+6jgW&d4(9=1~%q2(>Rnt7sv_rM0Zs=q_#F~Ak>OtNHS4fr6PgYNm$x~SKF zw|~D(8IwNh`dND4XieJX8zU0B^({&_B>H5A$WyXvP@-_bAO4TY=ne5>e zvUjc{h*VLTRcRkonPB0Avpy%Pdg|!C6$k!O3r!l^9Z6x&CE?buj|_EfU$J z`EPA0l$a0j@1b7JTUzH(vg~y3fG!HQUL1w~Uvy8Gc68cF#|sGpe@=qF8rT)XI_ZCn z$?qx}6vR3ihfcRAc(|AV=9hUm>kd;ZI*6M;E3OdHCSe{fNNvglrvMROAV^Te^`{dU z4^6TjVYi8EI39{46#SYPJp+f9yP|n}C#CwHyVtUQLaaEisXio0<)0tL(4bZ%JNtPc zIz%rb?TdR0XV{(Z6Zg2wc06u}cW)E~46Ov)`C;6lOiq z`;97pGm(P!X*B(BYXb$40N6>LZIXmdqz;W*Jcx+SRD`klan<4eP#poHn~WgxwFTQ>Yz z?kw%3As-Y;-^5ps$4`bc9sBvmk$$Nf8KO4fxn5u;ciye=g6lOUq#UO;;yD?rIs!u% zH`?rq`1CRt_>Uhd2$?5dA-4DWlG>Y^x-ya=uKDgBuJC4H>dMf3k!#m8SjtF%Ebk0J zbts<4gU=U5Fa-g8>Tyq4;q1z=mjKSic6p*U72_xEatK%$yUot6yY2R)&ZA_h8!J3? zliPA{SNH3;zL;vAXjCfJJ(x|N7`O6%#fo#dl;l;gI+!7@X)lYLp^}jL;k=^ojOX`Z z6Vv|rV%=!linN2({S2nIhz$Fg$ka<0Z^c^@(K98$_Og}+GQ~&V%HWFZd_Xak?W%qxr^f@VP!ZN+YSZc^?Vim1<_x;)iCwXe z;hv@wUvEzcI>2c;!fI6kI{5`Ar1aCoJ@?cIAJ;S*n$)s=6B;%$$nrIqNT_uq!*-6g zMNzHRwB3lcdZxr{9Wn_?|HDcUP;GcC3?SL^uSVzVcf7DSe0>%fw01}rm*>I-eAKgh z>{qv{IVu*Z*@8^0H`P8M`kC2+mIp$}9IZo{%nCj@MfQCs9<7^Lg4-(hGl)$E5#zUMqLBpy^Nb0<%<9ktKZ=9GP3;X$IF zzEPqDV9ItN$O&ucgryxHIwu9ny7Yz|^*~s)i}keHJ#Zoxb9X=Gsf|u%tp?sTN^QO- zo%;R6RU0o-e#v=FmiyVoFA0MZJ#0z+>fW*4rpke!}bHbFFZmbA)@00goII~;W)xrlkOI~H%sI#89C}QY{Z6hd{^=(6u zAy8sF)S%QWVY-Q(|3xdy%+|uoN(-APeuQJ&qR#Ng+)=%<41oTBQe(HZn0kfp%J;Sl zf$XD1?Unj07H=}=U{)l{J40o8{ZStS(hhsp{49FE7Dq)a4*;OF;addJ4DF;!=Iw~B zZ_+`7W}Mln!;EaNnbE>jXbE^M)*q#ww5hd9`m|fdSvsUFFmEp1i{@|FFJ#4LKvF5K zOO6=?Ln}5choYSa9gT#1>CDo&O;-5Fbx>D&@cX*3rr+5Z#hNguq$q%5$aTrj%ZN*| zE0A*HYr$@1^IqoamQ}ip;+8%qXPBfc%=VMPWLyO(Ek!<=THL9mRS>*A`RI}FOK8AX zRah4%lZQ&nTh$TYitNq^I7sAekz!o6I(kFZTl&rLHiNsO`vsU=T~DJVBVn9upAwZ+ zx`_`tO*gYWE!iP<#%o4~-7A0{pbJQ~zXi4gI7+}c=fc$a`~z@G!p#)kHPcUCVkAB8 zynXN`9{}q}9|&Dng3YsM5D{x6+nyYqTx8J~T1MKT^iYyB zF^Jk`i@jWpVX@w=JurKZVd10vv(>n(#njSn@oP7HvQc-zdK_ZsU_r~b0_k?+7eM-% zS&2?1fDIa^_`DL;s9>!N=@@0^pv^TrMaPRGO|Y3jU+%URBC z`f4hxNmnH+LM3}6$gjk6?GA)}V_EEQblIk~l%y5@$ z6F6z-#!|0IL8HW<%V#=uGmwzv`>RaZU6FjmcOS-}`zvluA z%7hm5*H6(0AR7~PoBt<|L@{6Tunrc&J6ob}LoD!ktirnVFq(w>0KlxH=0%k5lVJ8Z zOS!@gHSIm% zlH?`g?X{riI_XZ}5d6Hq_;4Oqy8=_#8(p)ps6HW-@SW&@E`jhT7 zk!=}z!jFboES-fEdMGx_cc%l=hL>s#AN!5VdsR(%1Q}MV;CK&OvR^3nFYGR#&~c@L zP@H!-Hz}X=B)}H0L3sAo7kC8xNv&j8|9k~jj_SK+K}G7eqGtsUG@hI$>Uaak!vPka zIkCBPJwJPEPcr8FFWS{3SyT{09SC8YaScFZrfCX$Rgz{kDyY=VJm#9Y&*8Ab#TJCX znkaNroP0ibJhmV~rajQq*Lu*0`(|;W*42GAO|rVN@^{cwId@kzdJdyj?tSPBK`f`eb($$0Hqnqg3Ode>p>Tv}u1alVv^ zUW`2{$*1zzjb)38GqFU)k|@Bx1G+O*sUuz8SoEa@?oTT+{K2(krym~Z;BMy}Y=UI$ z9q7+U>x(WVU(Os9OKa{;uf6$)K79l8aXoift;H8%;*?repLK%t*V5+__}RP<&2YJ~ zd9sd!h;)7h7g3f54k{`k*0e+|!1#upQ|N;p<8Z~t8N#h31?M*^2q%XXt1jK4!#Z0t zei$No)htuQO6N88KL$_^co|*HNe&@wK)ag@re(unXXtG7xJttF7rp|#fe#(KDfr|0 zYQ=iY)DQVJ#4|}NcSX9#6`U5hj8gmk+*5ff^)iThaSkE%sRC;$S8xS*>2#NEP2+dv z9ioR0n+-CK@*qa^%};}W#*BX(%U!p{0Fa>k#^YN=HG8rvcXfjo#$5I}ZQ6H^10y$= z3*CAovmgB4Lw>|SvKY)aQtoJ5%QGanrHA1h_R#C5lR9Wk?IPO4dd#D9Qn{t^ER+SO zruT03+WN35-5yf1b2J%Q8nQ@?QXV)FGYTeH8fd@o^&O!Y=jiyV5>L&v>ngVK|M>C5jJ?k2XpqD$5L5D%JOCEf~cY?H=y?HB~ z^d%gGO&;~utsi>y_YtfKiaHx%6{X0Z8JT-j;Nr!ryV}ojaB#tB)!s)ih5pWW-8m8z zSMHtU(h_UjP2R8U-m?o;d#yg+W3fqvx)tlqyRQ%x@E!mZHduLZ<*y?U*;v=nryX8 z7h8MQm)CtZTReKlUu`BpiPSb<)Jqg(>wa9T&KkXOU!N)b>_mV(iUj@KuS%G&JI%i{ zx@c8iZdhH7b2Bp*%G?AfU+t>0Zg=u7N^!>JSh;g+z4ydk71r_<&E9Qi*Ln&BpF)7h z5ku;1@?`onTytQN>DF8G2M!T^i+~rUAYfY|!r#Gj{8^SqrfiHlXmLJUk#z9;2pIX& z@RwKwxA*$E%rwFzwag|!;RR5Ihz34#>xJe14g3ji0ntp zs-;429D;lh>&!kL0u_LXm`V>>>vGdkw`fzRlN8962i6ky3>8bYNAAH$f5XSeC6dxE(RO10I zXWNTu)^M-Gs%uyHl&5W~UK1Nker8?2-5qCIILIR!eUZFEPNrl^&PJ9~Plz5e#Yr=+ zOw1dZ#Ig{tpQ)U~qcU>ja|6if?WjM^!FU-LTJBW<>+V!nQIK^bB-Tyjq_nrg^OWQtz08J*vs=i<4G57T)ck>5R;Eh*d}{;FV$WRo%?y=y z!od|0bxL87j};}YuNtJwb>6{jwcqgze&@3)U?Q`ymibPN_7otd`?a6>_?DWb<6D$w zL~gA=+K(&*)c#V1e4!|oR$y6f=^6Bm=S%WMj=`8FTJTiYW*zJ*E)}+SMghJ^bjCVkBYc|E@;h3eTjfl?s1nFR&>B zv?Q#nGgU~H?R~F6V6U}E_a8w&5XzBUxnAjPpDS?mb+y?H*NN++z^T#=+_Xj1Y%j;S z!Fx=6E;$D0u21Poy)i>*t2DF7jd+e z>O(iDneqX5?gb5dWsFz$`u5sq=xxg~_RuTW4_3=Hw%PcMSoRaM3DRmWUoSDdo9}Z(aJZ5!yuvu z#nDj}1hx==2-vK#_Ob#pL03eZl=~{40RAyIO5@H`WDqCo3wg7=tF&e)*9_!h<+{VA zp<%kL=_fL|4R@0N6deNBD6#uf)VUV-31AONM{$MMlgY0orrS@GFLv){6*EM5J$+qU zE1+n_BCs-IwaA((H#+mpdJi5(qzb$f_SPY@J!6y23hf-5UY&aJR|@%LR@!;YfuF}3 z?x-NNhA%ADIIr%nJ9s@Imxf^aYN#GPDU9GH+V{lL$j6!5Ly z43w_h7CK`;gp0{Kze+B}XyD;}ACeGP4|`N-=A?sUIvjqid2x`Zu%z&4)cHnZg}n(E zgHm>*pU**ah59}E8r9p8dj?4jK}Pih+xE&$sj`9m^zfmS@1}#XPp-C+p&o-40HcdtNmNP_(p>(Uf)y99A z@3QFO)__(VIhA$^7mze9k6`7^?Y|>gk=O8i)vX`f~%C{HH1`xJnXD-`p zqJey#h{K~=QBUO#^zb-gJR@tQPV>o`+ruJUlYM{}iX?nZ! zJ_4Va!^GVpLk~gFwl!Jkh85n9LUR7nWR!qzlF0=U$eu+3D2vgd`XGga3weV!%q#3+ zgc_kKaO{qqYT&b=gRDqkhqq^rzwfbdX;NyS^%bvS%5pe_1zg5{r2SVLKRt~D%w zeg{yHIyinGrEO0-?f>{5$_644^TLl0|CzL(Gjt2{aW#L=9fGsn0`|>)fYm3V<_Z7z z2EoUY0KyiP_36*HWtd+^azjC`veWege-+^@!HZvbdwCVHcj(Bng}gXiF4PK0_{xet zTxg~gB=txpN=RjUXXpI{)tYx&*2dT7e~$p<*~`P+jnl3^#^=0WFXPkwi)h*r z3Bsm!0L5LB_B9_2#nlVDC7f5T^(#l*By4NGyLU0GGKwR=&sK>rqF>V?Ltr}aMNc5( z3$Kp}%Fc(wqox_?!S_M*sBrahZ6evr9F)P{@g5b59mvBJB1WNk2vKA)?D^YaC`bTN zkyD)jOuQK4VXOYt)Mbl&5=#J4eSMJXFwR0R4Ip3vKE-q^Rtv<$4@6=x zvGQmD27xSNiB+1KjUIBm%`P}v<1#M#y6Ya-d9;gYPG06>k{z^#lWaB4w8=yUhBFwZ z*2UCpR`?vl;~`p3gGFYc>b<(oLc4G)ZdT+tp0o1ZQFX=46@k664@SY$-p>mWw&8Vm zEG8c2>$cvonT@nd4L7S$(XgLS=o8YU94mAgwxfOhqB0=`52PymEQ8GTilqmy%Qm=L z{)2}=pA8^(;h*V%`B}>y_K>sA=rLU()dgyU+bHz@1CUE>-#aamXtARX1geV`d0NEE zc3k=>0Twc=oep3+kgPgy%tq}OS&fu@MUUTd8}+d)3bW&5J+E36g!bEwQ4T*ufXrrW zAd~r&Ahc91Vg^LZ%PY>q3@G$*&{YW54xU)Hv^Hto(>s);&b(VF)@OO>tJG#Zu~Z}0 zkaE@42ZOf5LU6KU`ALBfPpyOY!4NdnBeR5?g!*F=(SLup zshd{E5Mlm)JB$SD^9CTrxunaNPb!F!S=*o2Ix7cN`+~T&G zo)t~Y?7{;({2>FSq(vcZMBbN4Q)|fNZg8~Ifq_jkYvJ`?a?1v)1!k@NFC2l>X3mFd zbi2|H+r3KO4)KuvX06P@t-cJk-tyg%v?$1q%4D_^(w^=Cf0KEL1O+7TU{mzO%BF<_ z)|0O7tUbDdI)2>!>e?+%>vbf(&L_Lh%vqZmAx(#6?(Lmnv$W8=dQ6g2lEdt!OPt(qnyy-{ zyXloa5`=U-B_JYXAKOL&Q`Qm9SK=Rn)T?x4#df>^?opdKzhJaO9}0j3jZ1}m`C$*$ zz3d-2J%++S41!GU&GDWbXPdvf2Nnj~m`%YqWAzHY%@5{dh_S^BVPtl#D9XTolk6L; zJeDz=`fu{p7KLw`9}tc4Z|>xV<-sJ)A?!-g zd$kNbH_ghP-9;SFj2GWE9t8TY>n&RFh3PWhaFKsg3FVHIC^#ybd9Ka!H(ra^ER$$b==P+Mur211bY(;J3FPE(lsEHE=|XD#VYSG?T-<`( z@swOoRh5_wSq{=OF3Ev%n_q_8N!BA8%*Ssz!MxW1ZZ zsyeV#1QJbpvn^70lmIa3q9V&+2|#W1W!zVLemp6Ba%uIYdWEATNAv@uTQ`CHl|b8v zKZi1RG+Fcuo$qv%T^Z@J#OR``vfw2{&J~bkRUVf>-LH~Tuh&oFU|^O6XGlvmu)Hjc*~1bRo%EPTOhqM*Fi3G`@M7 zs{`3J19IRoHCjzUpP~ z%^L#c=oyX@oZ8Pf`O9xCk%Bag0A0s<(3laFam%jtYmcLCn&&()cFWGe;A7t1IhX(q zv?K`%=((w)P@T2kZMXn9bpiAGjSNlNcdPs>^jYot76kx6sT+4P{Uy^3V5b*Wk5^W` z94ZEP59}%rBNtXz3+a(ddZI?xQ`hslWnWm*MZhUqpFD3Y?<*s;(ET*J-)3j%<`pvYkR-`rkU~M>QJ0! zdoD19Cogj5bUnRnk#}WVS1G(;oRAEB$db8Ss8S1lAkybwuODu zYCx`L7SG%;g2GobRn`|i=q_@K3ZpwpNXcpCFw7H!Rkl9}nKZROfC@`yfyLe0F?s4x zf6d}vaDmn&giluQ^%3UOCgaKc^Da7YU^I}8 zBk|Gz_gQj~J2jr9=tbHnw%!RwUF07=e5kj)Nd7ja{7bM_ z<#wOiLa1I;c5vlftY><^Ec=pDug52K#@mNW@?qPq-2zXfv(AnBcXbwmEU0s^gK{v; z^2Y)rR&W^!(HxR_A^}thfAJs{G(!aP+pl_&5byQudWi$t)kgVHwX%5Vitna=)B9t=f&s2b_Lt(+JYCTcH4qoHTYQ>k@(vt=^ zfus&nU#G8;l;Vh9Ues{fpO#u7(rYU{IX(;$ni8?4*sHqe^@^IkV(<0jh~3V%z&nj- z<-k`=W+n6V{iy|}2*cJic_t~h{yZ%X{njZ=bT+VFmaeZj(Fp(+ml1sniwA)7(5-nq zTt1wGsb9QGpAYyhnXH*}#KCx_LQ|qIi|@-eJ@C!ofm(Iue_^HZGez5P3H>w`&gdVh z2LQD5Mmt<0U=vPulSmqPz;DVD%2{gh912guhuC*@77L~}C*uyAg?kp70c}V#-*}L7 z8^|m5!K|;t1t#Ipou&tF!$%DgBC!=a`B}ZRR~HuzcbvKtWw&acmQ24?!i-)CvP_rnje4+ zKVojE{6%6%;PR-i^wp;VEKTW`Fds50w^xo+{t`EA)uHN~J(D7bOWH!@&PNvc4!bLE z)9)iFl@>Epa>Wsjo2?54FEKshaI(DaTrlrd~fEv236;<;pY>`hG4;Q!h zNig${y@P{M;=A%tp?5j`#xqnM@-C~7WjtQz1M!hw@}Zux!yhcX&ww%tdjDA+*rZb$ zV(0&5{gSydXO-p9V>9fU9yTde#I_0Vs@rTdGNiq>5Nau2%4zn}`P#F(Lt`yCY-*!I z`ekDwEsKYIwQSR;1i3TZt6_&oSSg+*&o}C4i@QNu@qA}`>;oQ8rk_#1sZ}xQU0o0^ z&?80Pj1Q1a4Xyt(bOd?N2{)IP7Il8zEyA2fbOT-Liri^w@fAMBBOQ-*!K-JKjTXgjC9#m94v#tduMoLf=DU z7UOSOciiJnYlLi|0(>sv0Dc*Mx%9E%f!}A4d5QVaPgv1!5tuj!**xF4f9iv z$0?5QA>PeK&cs(m>$OFs6d3P=LRxs{{HdF3C#wZ&^7qg`Uc#cKefBWmp_vl;L+tC5 zFT-EnBpQARYj1yd|HwCTpi&tMlDc_F8#7#SI>Pbs#u>PN^ZvCtYWn~K!dM&|vB#A2 zeI_THZRL0EY**X*1`&gZ)pt?7D_eJ?i|z7OA+r&=HTD@=m73saFT8tVLZ@Va?OtO! z{)t4Ov>41at(GrN*xKU!yoF{M*Pmfbly5>_3G6u=$l$-YE_=6QCaWG%6X^M(M99Mc z)0No9L|@2~_6AB>SQ(7#YgV4CspWv(qRCcn8SG{dS;y|Q7|ZeWA>Azyu_``itZYLpP!AQ&urx0#I4fBm#SzxkCr8NEOO`M=x|z}HiPP80W4X_s&5 z6q)zyEF>$7m3|XiuO*rWs&V$tHcGXTGDHqWBSTxlxTQ6nkv99SW==G#<-3}7FeaOe z%gf6fjdD)UosK3c)WSz>`ZX-)zPUFkC~43QdB^*h?SezlbRJKLQqXS2+SVE9IeehM z=3AW-%|HY)j~?$7X2|4U8;Rhzn%Zb)525iDdJb)j7dl+Z?JKhPcv!6}q1FXiHqs)3 z{FNx_f0$C1wa8UR39feu9;~ckjt8J`P>aBV0-pU9yGrJ)ii6a-gM6m2>nIgOXSyk< zkE+mkka~2lo>vI(WIM}QgAH8g?bCHEdo#S12Ki;(eXgo`hixbW+@{4S?l;Y zTghDP(xWG50i$0hNB*9R1ZA2_6Wi&@0(XtR4VoQ^x8XOBY{$BCr*Ag-%rIaT%C`wD zvUDoyuf%3dWEM?aQYp93cihR14zif6=l8T)E)1Mp@PQwin;B8i-tcZ(E*xbOJ=t%6 zGhY2^XUS{#>1|pPb#qs4Cj?O7r@h|nd~_IHY2UHjEiEwaj*la^1LaL(58K&>aHlVd ztf4ixGGot*_#6u5?$&{1;J-usKTn|8Iy%+esQWW6m+H*ag1EJeZCTST zl>HgOS8ki1H2W)WxE15VFgBK;l*nYg~; zRc3xZK7E_=f&PAm&6(DK^H=z)Rv*acRoab0lz_fWi@GrN`~A9;FPR@ht%$i!!h8jI27n=v|{GZ~FC+SdQP+pBodOp5dMpRKs>I|@F=6zsxK z&Iou(ME?X|rsCP34S{28fnBUWaJ%wC&p)o=|MXW3Hp=5N{%AVzJwwc7$P`m?Nm@%j z`Q~3w!%uz)`0Rwj1!s`lCXmcdse(-HZx`|}2j=OuNI3>D(QN^4nEzU>v;RG79LNqd z21mhyzp0FWb-C&VD5JxGi}QEy{_uaJesNn1TvGeC#vU%K=m-QNmi-i|bGBu+L0b0= zcLCUGwG&<9&pexr7N%G|s$%DZuQ4lLg(z$A_8&9{vj$8zhk$C|ihp-a4Dgco`On_` z9>wuh?*A~vRV1dQ6cVJ|U6lmCPnfCBUw|st_D7E?*{1c*zQGDUr}5&#qW#lwdi;)u zH~zZb&biuyd<}ilQIluW90%6Ro9ERP^lR92-R=iWg&lNniY<-*opp;c1~@m4Zk-MH z2S`Fq5@B>tChTzxJPEHEk?AqVckRzkfpJh^GETXf5S?maLZ{p4oOOd1Kx{>Lh#bfhte!zPM;8QI#y6=7Zx^0Zu z47;WX4)lDYjwI~T(o!69zoaWTgxRO%YB1zq?yevRePPX+8n>@PB?IV#%FCl~(g#Qm z0tM1|D6i3edwaXFCG_r_zH}8%?fT2u?w-y*7PD=UP}!tSMYD-o4?=HZ5-1%Z_B$Xl zB9OqRJ-Dyl;yyB>*}2(C=vk*z{Smv)xLyD_m@&ZI%a{}JWB#6BeSO!7*a|n1M@+tA zNQ|xd2e|f%>B|0vgn4ew%{BLbx5D~fb+o^k5wAd}&{G>iU$UT}=?H?qK+$CpJsfP; z>Ce>QPwVXKqhEDCIh^*?I*Mgc{!Te7u+ze-R&douBnXSi)@Cr^KJI0tI)tGoXhTN%+*{|Qm1m>tD#8mO?(nc4IK6Cc^Sv7**rzFvP5B?EE*D|)cR zCT*6=Q<%bpV;4BB8e4fsStt%V+^2}5_de=37pl`aL=Scs7g)4gWvw1BX)^TB4x5(B zu$8Ud-yg0-sx59KSDi)s4MAlRR~M8=>InUr@afDOg7BCO`D-&j*$_zp&aJJAR`1Iuc?+g^kfnmX>g$x=rj^`2J5cG+Jy3&MMc<+yi=@} zQA>YAVnSsLl;bAB)v|!{+p~F5akdZUou&n5QO0|)CwqnR&I%&jN;E$rMY5XWc+Fjr zSvYIzCF^2MHqccp0RyiIckqn={L}|HgDiUnF1Qk6at$UTTy;VmU7zlh{3DO=U40Au zvyar2-&AV89slIDVA$_FxHy;R)68SD(Boy3Cj3n@v6-#(3Az0o)$CJkm`_l<^YLcR ztC|Zcl}_bwplDk5YuKr<_F0eERcw-Vt}rg=m?8xDPmVS{rNWTiqR4=V{xiLf0qh?V5D z+Xp*sHHu|%lgj-TwH9(gDbgfAVXP^LIdWbVQZ(*%Go9FVWirxnqtO!h$@y~g1fK!H z!<7koGN~zOcp7N*X2v&xrxx0_t~xIYp5I6(205Wn^y{LL7bi`GDnb7-96LKd`7L z82Aw@Xt+nXH%sf`R!HJE&RB18%-gpMUP;aaSfCVd+A#5sbHRbE!4ub~vu{c5y#vWxE*!3)~;w z+_fiGvsx*xlgcIssc*n=`g41C#i=vzIia`h&m~1Z6#F0_YG@Bwq%zlbi9VC3aR^qS zhj|?i;nhkt5Nu)W-B(cNUManXp2LqW0tT{8aJQ<|9+(mjiHsd~ytXJ~dY6}$0v}RS zt-pn*Aj$0DRH!&nx@Ng*qFnS!3;PyT=o4M*{bbg~yabJuY2Z}53fWiDglzYEA!H&2 zxV1}&K}z4_T_cO@KAmIeT1Y!2n^2da^oMBKX2s~DJbpH0hh6+P_eUNw;geyy(U`k% zV}eA2rYyQ3)i!#^g7DW;a1_hLyDU%k82fP6jnnPU$7rM`U0@lKZrX< zUmzg~Tovbb5%raB5%-L+%tuWs7LS0bCkAJ&J_fdD*TQ>C;Jfc@bgOyAf4nz2Jri!$ zcD;SMO>i}E*p%6+1+hF_s5jOc!GBk~gq&#TX9dKlqpc)Sn=4t9t;GR8ZRjGDPZ`h- z@v0AGL8Z`_m)0C^v@AZC%_s_YR*-XczI5tM@u(~>+)RPH&v`D z{EktjdhcemZn4~=Ri2Kz*i@|Rs4c#&Z9eJEE5H!1Ap_j?qc({MirCI%;9@`W1roi= zkS?j&2^70-eng@%rP;b^WlIhA{m!Pk5{b}80}Z&nOUf4K?q_W7&ql8nj{qlyVefGH zd)pr=jS@2S6>HTvD~~o3*ddbivJqc7e6cYW=w6d!}wHnnZ8dEMlAW~9c`1ZT;qLV<+qft4x(Q=<^NWXMq z_DtpeOt@lUfNe%Tw|WI_4}rw3SNX)H^NHc*^E0mRg%o8V>|p3N61&gLq*=Ho#1a z&&Dai{gneS9-dbnL%Qv79sdt|-~CVZ`^OzoBw0z6Q5ltxy+=y2Wp9OSieqmr86kTU z+0KzY4u?u49NV#pIOee(9OJ%@KBGS0`-l4vxPS5BIPYt}uj~1GUa#ltwa)f)Gvf2! z=9TOcqBCRe#fLvzW*9ay0Tt*#v@{{?i<;yxl?}%^I=~bJBQmEOYw#S!Y+C!SCuXz; zxjzPXZv!`MoPYs{;O|`g5_p;^p~M>ShV;E#DG7kz+XnYp$Khb}k*rXl2m#)8CW#vpuuX%NdgzL};DpgozUl}rXfZ1H;3>B_sIY+lezL^j zJyleOjMBOpD>(u%VxsXGM(^=DPyA^&SSY?yJJy%4In$@97R#m-_u$qANF43nZ4nVY z>)5cFY*=~;FAW!GB#+PxL3i9i$z;>XtLoQ<7;WE|L=FnDpb8Ugf5Or0&#ZZ$7rl_# zB0BRGOjM!O#4T@jD!PxH`Tol5HyP$4UvBg~mcj%50C|5pp52amQ3|ombRKY?PZ2${ zMBrej#DV9GM|Yk^8>w|y+3L>pPyQqHa0(jfE6kzvrG066xg*aG;do=2whYxe9#n6# zjXT7x@T8`u_MlK!Tn07HPquH;_u}`ld!|EjRVWzm^jdYrgJ_%G5Vxvbz}@mr+$4^# zsH(CmDs!5?FA984_-1F4ySA32>Uq4SR!FLe*S9XpP$Rj%S`jU9s2lNx50?As=IvZV z!_iSMImh1#Qsw2yIJJskYWnL=y{%h$c1;&Szlhn|BnwpKH*rAbzati)%58|RQA9MU z!nP81O$mx|+@Dl^i^=5WSqvK)$i8HS>S5qR25$wSt-S7@2V1DIXi;gaDhu(+fT?Ds zb3tF|0X_ov7wdqFLQC97B;qe`qF?rIfkP(ksjF$+6Zb;>uxMQ4B_aD~>||``QfYU) zdhme)KqAJymgmtF836|w*06###vAd9x6z(skYJ@3p7nX!i)ZvGf|DH0Ahej88m$B? z;J{SIM!1@`GEaxQk$XZb`X)<2<3}09gwo-LLl|?k57ro`klQRmV;d}ENAeR_{}VA% zEq!u`G)LFOGkdbNI@;>J`U7BvXgs^*{R&{&_CxD>{~1ox)&y?LsUJbiNWEp?=-QD0 zYiq4S06Rk$AY$ zEp7=IQo$`^Pms)M1_6gLlAHit_mWX}(EE`5{-vR^ClkI{R~=(Ob!~{Eg1|{mA+?iH zX8q|oG22_HTelqDlQ0hKqUn(V27z<5@_!|T_m4sV{2^Ybk}Ra63T_0gY7LlYJJWeE zAF>C$5B9d?7##u2OuyKF)hWSAhCS&%$h2$2&BoD#Pq5nI{xDc6c_qpDeg{6Todl{> z$4d4pOs7Zgvenk~`1BXr5>XITK!%ZM$NU~+m#+jX)|i=hh?Y>0x*BZB{c{g4=jafY z=qZl_3rsW!J`-p76C8wYIc3s5fRK#FTYQNJ?^do2TX{bq%1$ z!Rk)xVohOsx;OwuPIdZimt`88=)mWA55b`|DjLSlDvhn_d{XWLxlk)LC*px;_)qIs#BB!P)1w;iDcoV=LbM@UD&5?$^!iMjnZX6=Wruep%NIqMm_W8Wy;dlgcxby6(MIhG3LEe zk1PcMzm=|GN6xBtnE4Qxa3qn4W7fys(gTLbf!#p~?n6#2$j|}{DDXAy z>cZid#ZZ?oY1e7-d{x(N247*xvmdg%@CKatE}pGj{sFBTFQrz--!ogI zBD$@$(!3UKcml4OKRlzg1W=JI<|ZXH4S>P^3GOQ*kARm^vD^RcF#!E*8Y!Jb%-<8 z&#xycf3IpdaIkCG+XeP~hv!+97;vZLx(=MxZll-DEN$sx{Q=27qhz`l<%1TVc;_L3 zvCrZ9Y2r1{W+jGu5ni%gXnLSnE@uL5!(Re$4!Hr2`#a1S&UdcZ*s>Ph;lfk5*`qYG zw^&S9k@XM9^^1S#Qw;#Tsa40jzsyd&?P*O zVGwt1o8qF@vQD2_{!-w%R;ZmI*)ZxZnN#_B1n{^%0ExxxmsxsZ3d+hfrS`fbz87d0 z{si77-;-bVUhsLZ{%uJ2J;Yt-*7^&#tQxmvgI%=$p;h5b2FfzY1-BX`ZC|n1>xAtq zPML!FZ@Y3?`Tsd&B;`>jxTXx|Z6UDV__4kRkAQ=VDmFO#@_GscN#%>Dr+}qgq+>cP zrV4*j4$r^!#Q$rN_hjJd?|?ghvY6d6R_9sfYOj)GgN?zH(%?qx4Lc^}hGwoR?eaot zl3c`l^X7S_mXEy{{;H=Fj%+6Y6ydBj>TFBB$5|x>Xl?hJ>0#|gJ(OqEo@E*YL2Z8OV;`|Dgu|C=dY}wy@aQ{o4)2|K*GSzqCIF7f?_L z!tOn$v@GElGPvi_Y!Xh)i(02ljW}DO$w@!TeZ+(2K9#WwdBZ9i`e5!Blk*%tBf{eU zaSWy;RU1|tjxe@(_~G>F?4$R%N3GOz5_b843|>s$k2=hwyq@X~Fev}S3Xy2t1#>E> zWoVzCWF4ZKXq)P~-T?)n9bx`WL@ym65v8;2LJfDWa<>I3{INPRGy+!#om59-7nZM3 z#h(-qLX(iny*@lEu=5#mm37MbI`2P=WEBwpDMvX=z_EFOl`@e}MV66qpj#7cE`4d( z1udF$f4VG#@UDdKx6cJ%7IB#C)4ZWzKim~^Hs$XyiPgY;yXN)|NOkj;tIKI>o;uYn z(v{kmqB!akdO%>C^dT@3eqU^Xuc^BnHCodA^R>~NjR8xD!ys~P|C@hTc}RBF^&IBN zZ|K_JRMbDW9)Ip#T!8zAW%Kx_MGGG^+qs`X#60+rIi|NoqFWh}d6V)?q9eI@s6js@ z+gh2`VCHoWujFgb{=$S$^>-bXXPCz*(CZP%xZgGrmM$4W9SOb1$~LL6;Jf!spTs47 z%{FLIG?#i?vmb)4uX|33vN1Afoj7yTb-NUx>c5_iWK*9ByED+8a26J@OU;-ps5=}k za=-l2t7qtog|PX1qhVzghEqJy=s;Js(6i?*NSr?SJmJ>oL@mDbj5H#hx_uzV_^<(iC)=uNp(IVDG_ZB`Fqf#PUu7 zXV|vKq<6yxGCEi!N}VdzyV)&eyvJN`dJ2`aagqB6Del2YHyNuCr1E9^!n*-uP6S&){%Sy&ue-F;frmR*CJxS zA#jQ2XW2e!cw-+pqt7i4H<=o*Eb$4r)rHkR%DZ$nRn!_bGFfYtI#W!Z;TRZUsTFSp@}qpN?VVV!Cr` zf8|CPq-c!Y9l_;Wh>4lI5~R@HCn;v0^JaHwt{z9x<2gJuAm()$Ue4_Oyu2g%shZ-- z_Yu}DjH*@AAAu_=B?4Dp2D5LABKO9bnS@Pb*-3MG zXX@Lj!XhVlUK?=%T+Sbt8NoP)#FspS(|xrZ%nRh>FWzT31f zd3X9rZbE|V=o4MB=wRrj?4uu!|Ms$r0e{p?i_uB6A=&5@lNGit=u|T2=~}-jSS6g8 z^%(NrAZI@@oLry8&zL5n&}`!M>R!7&L{sMPSO9eX?%hpkh^v}b4~|}NLfBus_^O1s zCYZlc4y{|tBt;C~3DPSLM5leH3R?J6v&Z>{QJyMvS2$5hZ zoX3FZinQF-+3eOiRSNfIOqwwfXKBwMq(Xo5_RTY)#ke94N4ev@+@@Z17 z((Ixh$wv$;ztHggdQM7fs!T<4rNOX=T|W^sbS)q2rtgY+E2fGmB`hu{HA*ovSf2M2 zd|1G?yPZ6BPH!~)-9e5XI#2pECiBxPhQ&hvjx4^_rn{K0NVH;#AgP_H+SubDh3X1s z*P^nO7}8SNlI<|prB{!m^=N&EzDDF)FZ-_jO)ek5!w53#&-P1l%mNc@K}^l%6dAm6 zQX=$Bc1+*-DQ35%x87`ZeW>^8gX=f=cIWq$tj4sT(^31_I=+0-vxcA>R=l|7-K?F+x@A5Fxt?B>m0DqhV7`ilTmw( zg>m6Ds=`kD1z}nG6l+XkEs43P0d9;(m)E53k;yYfRNjub#njaHv}(-Yl4lQOawTW#NTTEE;B+Gl_4`a3DmqTr-(bX zYV@T68c1=yeT79?Rd-W!!UlH<2Lw@-+!P~D8ZX2(*VFo;4CY7bVCs_OS?=JH$z_`s zvnb)}be~$)aGE@cW2ETbJI^yc+{^MjF4QX9bxYWD8=I;4XVs+HRSG4z_SpQw=p}}1 z9IUoe2TLEG>d<_?)^X_`sy}liLKN4TlDkoq_~q!HipEENPLeK;b2Ns+O3u|Y+8*tC zHp$)}yvw4b?ACizUtU_*d#XH;JZbim%%`1*JBhM9RO)@MYj-AT8M3#tgie2j;lz+a zwF}>s`a;adv~wMLHXfPNFMEp~X>)AtB(%90p-O&UL_uI*qRs?r5P+SzHCuDU-3;F-1w{dIN#$J*+xz*eDFk14Fm1(L?b*t^#uoli-{tAZy$dzXX$mM zufWN{tvNsPq7yN^&51BUm5-TfV_V~{q4k4aE4LNy!mS^njviET%(Sy0;h#Zqkc5jo}?jEONcjqHNcIEJH$`i!RSPuUsk? zYXiPcNP2MrnxvnXZ>uRRY?Y?53v7EkRwE4@mcBBV8qCe#OZV*Q@s%v1wcS*+MGr7A z27mbtiF92i9G?2V&~LWm;`Ov)NP?*PlW0w(=QpQOws>(0g@rF~NPB0@5ShVAfmZ2J zALxZ-%-cVDRzDrZNiBtWjtOB&=zX?Z!;Jg*xS_1GWt>oBgZmUImTCsAZE50uPq$(+ z`CTt>BKeVmYm*NO`K@NIN^4Q0<|j_MTz~V;@3bEDi&&?Tt-;kpBW25eiQ8V48_(>& z@4tP2^a*!n-~n^S@-m`L!n$U}K(~d5+5ENTnx`ai$ey=EwKb{%{y4> z1$T3Ug}H`@Zwo#BK>pOJ4gtBD^>5Eb(LZRi|w`!ro6-X&);)u@P?G^QxccYn?VM z76R-Hx1O2al};&JJTD9tz>a&=&mWl@5+$W;)6@||tuvyBYdXBROc4^+{aI4KZ5Tuv zGR!+Jbd1-Aja|a319Rog=v0<|ays$X$#g=6^|^`!NOR@9`!4SEP9rUUIMJkbZl88n z*6gqDyZo^yL2a>UOm+BcNZ?a=tHS4bAI9eySj0$bb+<{?99D7Kil%BisRXe+u2ht! zs_Y*7kY>eJILobja42|aIZ=N_VI$*dgs02^=*86g614JEx8QhsI46eNPXvxd?^d0{ zDiZ7{+cN}&9Cuh>x zcELC9O|0DlHrZbnY`a<8onv3xjK#i_NV(njTov_JmDxrvPwg|ggme!zvYyBH%+qHF z5RAT@|2ae79SW#a^)`Ly@+ngWn{3Of^)lT79Va(xZFpRP9P+SLVTa;80H8Lhmt z5=R`-E{5@t3b6BZ%w1)-P1`REWkzlzHjSg}fF{dQiil^|64iUoWoaJxCy}&@7`|}- z%@C<0E(+o>reF&9otU|+z>fNO%84e_z$u?L8MZ!}SSh} z+Qz?~e`Bf2*B?T-MqaF#d@jR2N^{RvyeEv_%A^&mY}W3w^`5J+oLJ-GS6Y!L#ZNh2 z7VYL3eqG8mju@1~WketoY${}XVFF9>aM~oBla|G48)FzUY47*ms}>H4xnp`Bv{;rO zkNH@a6Bb;7pQG(oQ`hq+s8CWkA;Vn>u_LMWp=MWjEeq8l;9nq;xAOhS%$r z_1Mq|dq^}2-*h9BZx>`RWn2!5c!xD(jOx@u?WQD;Pd$7vl{m#H& zBfknHzG#JBF*1RgK5sM^pJgj?Ch2YXJ!K6J=S@V9-zH(VQ(w7uoOi|(@A}M%vf8@# zid45N-G2KD1Mv`ezX5FPW+Gi$xoPt(FMpgNubg4bkQ~hmH+tG{Lu5BSPzNLCKjs|; zNID#s(@Lj>DhE@*1O`xKv%_Yt0$_6l72w-is7csCnCS)OC~p)DChTS^Opl zXG;~oN3~Hit?{+XX<(ll@zs@`ci65r{j&d1EWLJLN&Q#Wp_^R8uKC7AL17&eLgh*t z(kaUaLf^r;n`Lxk;W%V!mm=Qon72iX>N0lPN6>q7x(!n4bkT@T%H-nikKK}Rt8+>5 zsEuV=tjD?mXXI}*FFf50doq#yow1eI!s#pCx-sVm%PgC)nnhW=^;q?V$RR(r$d~D= zC9F+{+{pj&DFf;n1h4Gt92U%8`Ra8OVo{v5q?j0IMXVHlqWKK+>RybMWp@e1_j4v= zaA#?o1>5<#&@!9|!O`VJDU?gwUepJivnfZE6G20O7R5ijTE-z^j&_RMjeK*j7R!$4 zJo-ZMB`*>U^Ro`TboHBO%Eld{2J1dy#ZWi7X-&-#MJ8tDY#E&({2{jxWIJzo{-V3s zt$Sa;GR7|;mlDS4&>?oZy^NH}12M@je#!jR@j0a>Z8No6|RC#EGslzUYSryEXoG20t^PO@HjRQBO%*np#K+l|!RbgrRK(U@Gw8gUP# zjYkr0e`8w@vvxQ-O||pG$e!_eDHDO3I*e%RU;4$`f`E~yoq^xp^}$uk%L@tYvzE*l zi5PlYyZCTTxJ9vNLSLsw8;GKj(8(BIa5oR~fsU@hXg_4}b2OHY(fJCzyhBoBs<`3S zaI_K|+fN3ms}51lm2a7mv3-RmBh<oHGx$odLQOM;^CZs``@K>B5DR{NQCO3BE| zfG?bv;<@eR0=CDmTe=!Vt?fj(hR-+^(@%FCJj$(T+){i0o^#Z^Qy=G45m`7mq#Y3@ zeP14{s<|%peH7e#DT@gGa+u9HL-^z7Qnay9RKR@BdteQ{HsMIM*-B3XLCO(bV}#tL zors}AEs15!h}13yHfq|~F?a%yAX+{}VCXuk$87AR#T*TAPIm*1V2pO|DU7rElD=|5 zr^!)U0?$5%VzL~NRNEHKaCY92_KV&#=gBWD^#C4=ei{oj*OG3lS5z5CRD72+@7M&{ zkXyxniXYH7-zS$isFrN=y819(xm_2qTC7PQJG6_uYHG9k2uhdn$3(H&gkZ2;{IfvT z!g>?nA2mtp8jc;UaMP^u*>&!mx%KbjmC5KQDwn??F7&oT)OmA0`6BG7Ri1A)$TwWY zjy`@NSc_ppAJ_GA=9!JS4+8xngW=L}@NMq{v+;vBi<^(TzC9aFRAtWwcf<4EkFr1P z;fij~?aZZGr9@YWi80kkJxD5R8vfvbd5FyXS^2J7%WCh((o)?Uiuq4>thiL7Ubj$4 zjxM$r{WuNquo>j~FEFtVocP*a^qvXru^cUnDRhnZyThcdoXO_gsAc8mj1os$HAq#r z-0aQW868uc;}u}((edQvzc2IktAt>J4^G8Oc$p)iWF+-c;mGo2~bp*m}WjzB^;|)1}?`aK&c*lL^VJsm2=c ziP)IJ5w*2XL*woxEBRw9qJ{T!64Ldf>!7;b0l9^RZ5yks(-Jm%|MIY1&k0+r$ z;_WVX^CcrJXyGlC-fp%iRa_R&ir`x%mA9O4cQ-#wKMiXN5UYe&a<|RJW^?hY{FGhYN)@_5*Y3W(-r7$?^LiEc^ z5R4o(AKE5<=XLyY!~h&`u-&@-zN7?x&#K);ZD60j#HOQFl;syAn?(fjtt~v%etf^4 z2*`Jq($}v*QM*+2|AIOAItD z?1um61h?vu& zzinLaLAyL_zArD$b)y$*_1)!4Ci)_;6b5r=kI7ilb|WNJewp{^tb3FQpOY!Q|JVK; zWDfC%dAEwAcQ|MO&k@tUajk9tP5nBL5*K%O^k@nDba|H;HwOj(zgI&B4`ia*h7A;_ zr5p|fFr}Pw-lJWPH+tt|?sa^Le#ozrV!R3WN5IZ5Zz*~fyBZlNv|FX4CdoPzFk*C~ z6z>+gX!ckfY*(1N#|<~%_ZpeM9vPwAFmW@of;~Y`ZKO`y3CClY45o_fSk}I@KRaV- z@5RNK%k!>D2vXmXuvQb&oNkUZxH%u)S#D4>|E0tWwl`T|ym@g=cVHy?ugMp992K9R zCOP|*@a|U6z#9ko%jQhaF}x*pydLZCWxZE@MN9HmJ}gllpZ|eTz-HB@)`h@Ev6G6u zE4UG4s|{h+L7sW`7gnobm8uX7Yl_>`IZRH^P~K;Xy-jnU=|!Z8KuLD%CqRGml9tfx z+>Qf)D$pv;Wg!3Oi*VsC_akHWJJF?XS*CM>Lw$bx!96yJK6vyq(&(3q#-59LU&y5z zniXS)$OOIxGdd^Ce|F9R?j&Lym2v6SOYoI*z2vhGbgwZ)F7&=nI$S>*@XyIJvJCle6`o%k^U5e%PEA92izx42N^H7leHp%y73>CH# z6Pmeoe4bE072`I_wdtl6cM0E;$>2u8j=Wat$4vUGum&yJ25Zh{h%lP)mW~&3?uuem z?w0cG%fg$_=Cx>)l7o0EtfRWa{ZnZ}_thPZQ19zI&r;B5v|tv6Q0^k9swl>)ys|y5 zU98^(ew-(j_jX&m*$N2WPyz0N1TDkvn&@W?a06+&FLxhSV>L_4yshMX1|Y7lo4QUv z>X^7B!2R~}?d&(u!m=|ZHm*iPoWEZ2uXxYt`}spCUei(U!<|cZt9Ih^QNG{z${$bM z#p<^fE8t*Y}_e`h$3nHIY4PdwgAq{&p)5 ze6H7GD0{GW&TWkjrP#^r=fGA_dXhSI05Y(*IqGGOA!zInt9KtaGia6+D?sj9n2fs6 z%f%e z5)Wp4rpX7hw_vqAlY<^e=3`Rq+hsUC|B>jx-%HsGQ?2h}eFlD>)^vz785=b%#;+_h zvi}Zq`*g?MidM#Jckd0tRk4^@g#1bsN5_lwvLYtS6#6gB8ac5>om>r3pL9*zFPhv1 zenaJc+3(DG1r<*w{zKIywkJ&U~Xe|+YQr4xKNEkgYG>sjGo8n%lDH_ z<>!@>?#H2ma1Brevkzv&#XW0QaMblh|*QwLdH}ok9|H zoT&LI_%K5bgf-L=ruh&-)qx;i47Rx8dkr1dSBb`9Smg;KlT@5MybeQ*IhYwyy=>le+Ro%bEkkkxis7WDtd;Eh zPN{BkV%wjks|t#Pt0kq6N+37IUS~8Nj zh)>^EAU=SYs_rZ1Z1tne3lB(l%18M&uHfEIP`@>J+Q4{bxpVUt zUC0H4;29`K27zi%-GaWx7h~1FqI2veGfWBapVLW?K4vb5N@Q%urVA9;(oV++Obf&q z-*n~1HEvC))7ZN;?3+|{-!pzc05KWi1OA+U-x~+!Pkn;gMvxsZG0Xw1?CQ zCu3VIoSgvcI6%wkSE_-Yia1{<7zz^f8k$fSTNte(-!S|#%_w~cdVPX`6EyJ3B_j}@ zY<0h-(wvP`w6{rk8g%PFYBQNeHg#qDf>seTO*#hVU;AaA3@~MgJ?!zR>0)lmG-?)H zT>Os)bcYM1%&fYO`&C*auzgah$dy3*0}Dv38_Vp6O6zd5^Iz&DZvAV!9|PMwb%5bz zWrTidwlW?NG{51b4(X?~W()k}+WEk_#c<#TN2R@Owr25A4Yw&`RBRxGmQx}^p{uXSX&+spd@w2)tLvWw#Gsp@Za+LW#>G0#fZfx8;MYXp9>^e6m@X-g_qMtmC`%Ag z8Fp>bse0#MgnIm&Ix)dpl`qjbUYJCSe`S!r*qtwgprnjS!dFpFL=qMCOqa?3U2fnv zYq`MH4>qjR25X-o$YYvU8>I5l6*O=I0TxMOxEp(evXe{xn~A0YWTM&MvUqy@02JT9 ziehOi;3-LP{yO_)_e;kA(JR`5#A7;2aRjK{8Vlq8Yhxf8}`3-D{anwZ36l)>lN5$#{oDK7}Pq)GOPvg3~i z8Gd|v(yo4q#SgDU?KZk{s+^Hl;6DNlCkWJBi%U?x*nGiGEo|d|RN~ewU_9fw3}^Mt zy&Tp)JSO;c&d36x(D7&z;$7Z-kkEQ+2~l0{+nOiSfRLrLg2dyO@t|`3*_fJt+(yxU z-O8-Q2|VHqyg zUkCa$@@F-OtyKuQx>5mfIdKf+S8E^p++VMHBmG)0ntC;cQ0GUj^sYZjxB4`AsR9ki zc6=wGt_PwZh}^GR8}(orE+*>!J*YuN$Y6OC+1EdSXw5A`kne}SGWOf(V1FZ4tTA6yH0;f(Ws^*!IKTv$`F6wuW*Ck~xs@A93 z7t96NgOx=~3Os&+d?m zi}xc|bTpnGE1DS|HZJ1crM0m&)KpQa1&dwl>Lc9pc%XGBSmj{1CZ1 zbVQV^DgmeO`DWSYGroI6HaLUC2C|goR^x^Pg86D9+GOsVX4WY}x3Mj0lj0+vih{ise?Sq! zt+jHqi8ir36Pk@leXUsWC5{UO5zAu_@llDQRA1V^Bq=LA>o;jufS zK!FS*DE5ugrL6*u6-%0L@h^KJP;Jy$Q%>xW#S9!x9j$|DW5VnDOsQ3|riI=xX7pz% zTfAs{dzLt&8YoI)-5JTfws9y^eNxdVXU}}SH!@!d_nL|k0=q(+-dLk?-+?{pu_tYD zGtf@3;adQIXHV4Q_Yd9!mwhRYlv*I4dKWXzaVl`(%$#P!UeC|u!|@qA(p9kOptuDej514N1!cuZ2NwqK>Gx5J$n>Edi3DrncN=w!B3lJv=Em|(~DQkLhk z2G*H?(yskZhA8p#LvQVfJsYEfiRkrMo)oY0iXF0CWM>rvnC1*Z0$&-eQ1(yDohIhi8HuT4f4_MKM*J1!*0Ek<@$c9oi8YIObArw1Hi^nN_$YH=U~_B`R{UD3?+TMj$e;d~@bx>3PwgkA|Bx&-2N$jPMp ztX(73RhZ?U290zpyo{@_yH)~X*6*;X7kk4@x2&Wqs50!n z4~xXRo?o`jkK#!9E_VIWl>nJm!aZ+;6J_h%k3@kiq(nau zmslwl@O!$EWqcSsoqv4e)<+(tQ`u0)2RBg{yU*;F^f!qRNX4 zIKa`z;2bNE(Z?s$9hxp2x4-q7VeR))Bo=3g=CLt@0vp3IVzeLZ^liuVnFDpGfWS)| zR_>2h0P$Vg3NWpX*`JtS7k+O?Qe9VUJnYmtx=2+n+c-Tz`vb9<&gLx(+!k3T+&LwM zEAhyKL9P)&M-wgdoi-~C%9_y-e@7{fCRt~j=9^>{pEOveNrZRm&^jAFLsim*@5^r- zE`3k0C-MwvVCi2jJdzS{8(nBgzbW#pQd;+QJxf;-;oc02dyg6FJ)E|Jg)?K5Ea_QC zD99RM6jn*6)&gW8Q}%y{WcN9mPv2k z*12Zw+zTW>FOcl5Xj3k>d{M0Nk4p#*m>{h1(Q91+@iv-s{KR+G_&-UaDCD*SWgZ6_ zj6*7f?$k}3*RdV-qJud4!LmH7^U}pFOpgz}PgGrxDcoevSCL`_hyUU;lA-`N zvrc4CrNs)mw&@?U2r|E;Lg$y#Etb z;<_vVB_ymm$Aw!P;I-u5Hln=X*+&lo0)o_b{pUu7grzkgQv${M$)l9~}jE>(>sZ0f^p^&Z3@3`XU z_yqP3bYIls`zUz+`uWu&y%O06&Hby-pZ!HI|21M6#2}ue;c6!2JHGXyV5sa$$dL1R z)aXb+$W(PI{XacfxB|LP;TnE?X{XK!x>6A}D}sMm1`HLU*W;tU!cu8z0lH$-zmX7{qrO2M9B^5Q^%9rflnMc1iOyLKcd9~W}bj4`CoVF zOB6Pk2&r#EgtnZwKO09;R;+a zbkSsuKk$Q{K}NYsba3XO^lxJYCS(^(fASFnu+YgH~l*yJ}w~ybXE)l%>hcU&#clqM)(?e

    %3@0TzLz;r8)d;Fs91 zxm8aAQ{<#tblewpfBftGSG*o;5dv}#5XmpDYCMcSK0fHxA2k734XV;1`8Ip}u^Xq|p)79nWdE!=UWpgg>u zBm24iaV;!F68n_@siMyiR57j_Fnw&$|Db&ye942-u_c z&VI)nzk4_FYmN7c`0*DI-&e);T#to!e}H@&cu$J3Oa8TRxE!hk)n>xf zWP^dUN9T-h|5P`5SOg9#CrT|1CP;B6HYahB*0#v*`{-*{B1D%Ph z(Q%wDZZ$-*Hk!^{d8oR<6gg1}M*rQ`lnp!4vT@h4qq2T?F#3IJlg!fYZ?*WZ=&pFb zO)q8G^dOe-O2H#-mFE7D=QvopH2D%);5w*$wz3c_$>$DD2$eA>1N_^~>s!;}8s4D(>H@nTRO>hdK9_1gEjI#TjEw zodUhA<%Yc#RI4cSL3!lMLMoA8qR0C}g=kTlP?)|qaHRuceZ}n*cZ9y;zXyr4=N;9?&Th zFdJIC;%$UPLh0T4&LXbjOMcklw}<0tu08&qT$Tu`t^ljGsjod_W2wD&4V!JcbI;ou zht#a3xpfh&ztW7TEcm~v`=3p8{EAG}NEc{!oK#-6LzVLSb#Mkp=FQQD1mjsKMUS5H zOzeD1$k$OI4nByqWF|R(frXpPzKP*0*~^}s^9^2GsIU(Gm<^V-`1E$shB$sZZDGbY zf)oCA3_tgGf+q53@*%+Fzj&dm5lHDV_+{)fQ(T6mnDlNAqr!Q|%&H%KEL< z2Kwh~S8Ovjf+*6Bf9RKi;QBGwGCrvxUK(*uxZAl#+(OrafDCVMQf^KZ8lZ~1K)Ctd z`v(fS;~gyO9fRcq*d#BL2-y-Fn1xZr;eCBzC>j}brk@WVTOjNY2bM2lByS70)d~4_ zzBZ?*!i%baPDED=S=8AG99|VP1ewVB`&V$1D*6RxxDrRLAPTuAAAFu^ zNV_Empgf2ofas+bL^*+1P2(}1p_z=3LEy6w9NX^SX<3B0bh#mL&rZnsPeCZxxXaTZMT7I6rvM}cW*v$X8{ z0C{+Rkz8XmXv3#)zae~J0$=cNuLZ-q;dnWKK>Dw$JKXCPDY*DUttcFLDV z#n2XeHI!n@i#FYYU2yS(gZb6qyYrD~ z-Ib=zS6B}fG7SDSC;N%RS2Bo-8y3zzQ36V?Fe$M?{Hi1lRFVrSNl${7@!)AS=vFX3 zfjyCCW1)5ms@MLLMCjf?L1CfN=KL2QxoD?(gygQynrotz>x1La9v&4T(}+v$i*iu# zQr3XeKY7v1S*Qb#xp%j22?;G@FtGhca!Y~L$3{VhDv0yt`uyqSmngtJelNBT05>(hk^)jt z`c7)tM(s+^fqXwVzxZDaY2Yc!UvqJSF5)Jz(!S!EYCraAK%O;a)0!%};ZIW{e4vlN zwzd1`^*r{0gnW5RP7M9M-GvA&F#&osx;F^;!gR{K3Yk2J z1bDZzAQE_pp@0t@H~(s;91o%HUtpm}ebbGo{qt&~ z_M8AuPBCEW@^1f)((uCtIx$TG&>gZOGY9_%C1@w{T7fzT9vl+LXQ(&!0!DwBFC34- zeupc++ed-3AY>wHACn*RM|D@v!uBxlz!(7~qf1ZjT*+9@!x*er^zY@Tg5{Hm`2<*l&&RG|V{10A?#beo< z-?s6U*pb{boE8vO!VhbpB|gYUsIHkx^s^d+PeU7Uz+Wxz;!U1^Y!>|2oQZF~RcVvt z9y!Kz0^vYT{kKVQBJftAtQxZ=6v_JMw45MB;8CRN_j7m~B?q8R6#f9l8$>f#HRLX#Dw-E|KrLDz*P<1M>1L0 zd2X;*ssTrzXjQVVBN(mTlkm6z(@vu zcwLY**P{H5-5PVs%``uco542>L8E4XI{s>t-74iv*_OTSSM@P>!?Th_1>&Ckf0`01CBLa zRKyZ1L#)#Z+^C7Qz5Fc9d$=x*QD18RhS)UayvVMymEVeFmEuCZ@qkMUJ^F#mcY#F+ zUSPToUG8z)@d-|FWX?K`jEp3$?OCg4ofzjn*(q@&_-(N$u665*#A9X~aj(E8OJrQv zAh@!3RpOY68m`c!u8rVe7Lul@&k`b68#isfg=kDlAKk6ogw|?i23=<7)x^$)C0=dL z!D6Hptx%o=VpL@aW3daVHyt)+Db#YZ(MUs9hxVe(oU&O(;Oz0hqsNaLbJ@J3a-rO+ z>WSE+QHy8KJiNz$e;HZ60n$YqA(q{oog)T3G4-524BV0wVUctny=6A_ZHdov9(T|D zK36aoP=CI2T!g4kX+Xo^+`be=z{j zoJjJBOFUPAV^L6AZ@r{yUFs;}>oCs|*f%`E2KgOg&Uw6V1e~#+%;H&Qx@Y>TIkFdX z<61902drymNn*o@rtY$&t>8fvtQfeJEBb63&BKoz6(}~7zp-7(mt4^gW*^I&{>dHu z)W?^VG@C!`VfVr77PYP$8^lK{c9-lv@6#mu1bC9ORb0iX@Zv<;SP3{rLZR*|&GR1i zAQSioka4W3H#>V@t?Oi^cb}J6=ByR+9_ZT6(KfX>x-p;>ZRT6J@~&x-$Et{iiGLtJ zhwBnSh{?mHk zjR|op2n=O!Tgp_M%YJQ`VDq_XV-vC@Hvm5+<3`P$s{CN3K>VIYK9IBqgQh~Fo_Xyz z%?h*bj>EU7-XCy@=x63s7`k=;L?vwSxJEtOvq+cvA+j^IK|N7-r{_yc-wDh1A^%wk z;>3F9)PWC4;-nxUFmp_~LbJ`HjBiaSW+W|r7(U`fs2OkfOEv15lo!^pU22}Q_)2Ax zf3{9(^V%Qx*bKmhp)WmD$I&QhcrEVi{yc9kbuF`s)R0`X3L{6_W3}&-ZHxCMQ*3z7 zDjF4R&5Lw5^}<+FkV?seec{rs!`voL`P|H9o75bnxh2l)sa0=3kwtmfCHLP%j?1@v z)?~G1YRo=fwdHR(JKqAh33gJ32QumR`pl%?<)tsjIlHLkGmNvD)xKU5$!^nsJXs^y zb2Mn%>%%1~z;ih+JQ^9}w!xYf_e(NZ;hM~zyT@Fo+nnOP)>WEVR&>F64P{aaJUdTV z#2<1n2^rr*n5)hDs3n_kOFoNeGBsMf711FoE7mF2xgd7e%2qhud%~k%AI4lhqi-># zPInFXBI*c74R>T4msgA>Y$yw}PP6Obx;K{V7cY1_A+bN<;!heSsPb^#asKSG?<9qd ztf1oCQxy7Kn=YA0-MA-R2~X&jO?tV5p~=~3e}B8E$S*mTcSd+aKlDmeE&G_cW*V<_*lodS^Fui z+{=&Ny>$lprC@K>;j@H{A^_4o#K4)dmfr9fwh4R;NDb@dsQ(Pi+q`JU8ym)|2boaw ze4B2LsiI>c9#X(`zhcElrOu{1gZ*zI9%3)ELiuQ3#3A@)yQVk8jtEZBw}V1z<7HVm z>h$t<4^T&JP0cFQ6QC-o$XVOlZVbE5{2#u)JD%#kk2@kHE2QkQ35m$j;WUtuQAXiV zWF>oL6RFIs%)`kRj_f^3_B!_7>o~?Sj=i3*uKT{P`?{a!_59PTUY+0X`}?lXc)vg6 z{>Qr!9@*MDEuyCW`(MU3(wCIHIEMoMBnGnPbyO!FztGKHt=@3`9Ixe*HJ+}Sxc`E` zxXc&P$683G4QSXqdn*y)w~seQxe893Y2$b9{evz5*4xUEm}|_EJ}9q-`;a3h)L&LS z7_^pil+_t(*Ri8xY4V!XQtK>B)=@Z)5)kPEqinE~63? zgIP;9`b}{UdO3Uf2jyRv|BCH=`5E~K&N1y-HB5{5g_&!7emuBE=9f0BqMeH?RKCKw zQJvGm^G?g6ZY-Ac3<$_WhnzbePxkgqS7pfFtaJ1reIh`KiL}l*0`pwsY=VN8RLtZ>9bL51Z zwZP!AY2CkT0SN3bOsIk)BU(^L{kLoC+)ObAmjkp6t>4K$vGH%bL@ZY!{f`SDcgDk< zI&O0aJ&EOvC;=8SSFuBkNcp(2&3Iw>P|;7QMc*Rwcbbj}G&%;ZN^g@B%CMypW{2$X>?}ZY&IVcm@Z#`x=eefnzDduQaqzA|J-sAicI2LqEva! zR}hq0fyQpES>@9wmRUF6UQK@56vz~Blq0&j(1UPq9YWs;V^qm0H<4uPHPI9I_|OO% zZhufN=n^_eO@TYys}Nz(d!riSS{h+f0qeT1y(y3@IN07B8TEC~UpuYio==t_>2|+U z>NU~Ms&L$DLB#?3tKZ1X|9B`x72a&NS&Z1T+%lm`GWtJ z&U%y(4;!PXN#DwEf9;$B>+8Lmsg;*d9wX2fJEp6R>a>0##Zeq zWmCAFr2AYQ6P!vES}h|_K5E6p=}WP9;nQnEW24LOtZLk2okM%Y@1-MHn^-?~N}K$0s8&~2SRIGAzoE~KxHbNI zO~k09fl}(%Mb{f7%ovB1EGf!R7Ga=Gtf0BWZn#yIL3VnU$l!m@NAB{vh){oTNJjz` zyJA$h5I10u7ZL}0;$8?+YA6a=u3pr#c)LB6<8t|qQF^6YV0Vm;Zo8~-^{*V~cusL^ z-q;Tm)UdisDTMzngwFm&iG93MyXX9|N?>kHzq9T*_kvC-x(XB_V2z^dwS+JjutQ6a z3T8wGf1&QD7Yis?ayqa5kO|O@)okx-8+% zKD{4xes@T_r!K~pI4#{RR@#aCsBl(w^sHZ6Psh+7f9aghSDMFb@05~zsY*H}Zku~` zJ<>B9BQlqoF|(+4Fvz!v=EMVMxs0&h&Wq*J zF}8hGip>s5Sqe8R&FtEQTAU{eWE5XdwsbD)u^DbHD82jj$U^;Sp26Kbiy>fc@4Im# z?^}u4`dH=oYt}-1=T&81v4~pgFf|U!n1bB=si8n=Go1zsg!b!^pP(!eTio^;$US({V&dV?KAs9M*()8 z^^b|y3+YTWt!<)|egtWc_;BzhzP2vW(nA4vD|dS3_FfOfqP@^3gRRu`g(@dD;`6@Z zp-3R-oQPGtrm(hiBpL0!VXpF7OVLZPk+{B%rIAokvRk3a?mi*l` zx;MI|y6*A_n~650xh9WX{_$}3=~lp5N>q9L^NPPxqCP(ny%z&RJ_l&VpW4iSd@(`Z zFGB_0lFRF58$H90Cu&MUrl?pk0;R(Zj&V0m@eq9Oq|3t746pr2mp3Q_lAq$nFFR5U zotd)!%>2MFS^KPBkx^HzgT`$k*XCErKfnAdeMm}i6(EuUj>GY2KM+m*FR%$yc1Zy=^o^u7sU$FRAA$UuHSv z_(v5EfTNIUFh(Mos*5yH!`J$T$uI>BhYG*|&rra=mT-{%0A0a5^IV{3K6ltjf@2<+h-RsLhsNsMkRI z=t~dtK@{z%a&*YI9VZfwnpeCu45F%oO9!Bu#JavkyQIYkXCTrB&0{lE;c+}dy->HDn%(WC zhk~nHp`{lZRo+gRRqSE%by(y~+uJdU&nk(({Wp>Z;5uG41wHjDxVg8jGfc6KRL_$d6?7fOKGcO zU^HF@cZOJO5)cp-0JO_E@YcW4k86aW4F2`&y^Mn2b1Rj*g)s+3379s`oJSD~B}>sx zEQPsSeO#SSS@uT8`;&zp*MJgKNxDV{@oaCp(`VC{riNNdOQc4J*~B)|6=a^I%kIP* zR<9WE9C{CH|6=B;G?Ur&*qt;!I5K!cY%n0L{8U(bH{!2k8;czw#ki2#y(-QH=4sB| zqQHnB3OVN2o{Or4bJGlhjGt^>x=IVWCqqjD+TCmS3RmrNo6gYFKXA-nn9K{}*3U}) z$q(IRud&{SxydXo*Ng`BBU)c#b*rByv&FrFzGPg(6d5*6NO>jYhp33@>V3~8JY1{Q zZM-Q0?TbmSm;9|f^Tc473woCq6dy>0#)D81z)n2U3scMS{{s-gvM*w2t)ZpkW;#}a zUrJI!qSnjEuF;KT)Rf#>1`ehIWoXxc)9-vO>J))D%LURA$TrP`q<4>#)B6*+zZDh7 zanHKqt|3U7o^O3hrB%q@jtSw(SRUare^26(EOIiV<2I1#3{QufVXoS$Z>6TVxluGe z1%`}H*0#)!_h(f2wb6{(jWF}w#jx_&BxRWcSIUw3*DF&mF5iZ`jx;Ip`0pC;^lmjx zUhSIAiRQjm7$|Ld7t|YDj;ap&XWR=iiU=&1V~lsnev_oKF9280(moi5$a|{Y(s13U zwbfES6Di+uGM==g?2lWKM}i|5OI_^ zjPn?GSreUfGc9j_yN5l0A;M&J{a(thhfIX4EG*u3iT#y$Vglo}#;@YEOSQ5yJz0)g zLgfwjAddN%11a2_0nMYhcQZlaJujT%YAY^Pt!Pje`wt5-a?_8J%{Cx*o_$_bL&d%;GkT^O`^xv4-*;7- zn5r0Tp?s7wCM0ak7u=sjjt4B~u^#oh?L=~qYu=^bzrg^s z+x(v#)AbSSWiAWd@)__PNCB-JfGEG5W3l}Z^PKcwsxrqrG3pfGS3BUJ^|E!sMM8@) zePEE|?gK`;se91vP#_Eb`!cbg^*@=+#9t1G%1FFY576y?@*EG;v+Yrnz<;ghKOy|T zr50YxJEUyl3iR3^cH+M-=-*fM>~ERRz|SC#!{YyiyZ+yAhFKH8lh?BjObq&q51}rp zYvn49$%qza;3r%0qw2k;@8~cBL`Wbpog!yCvdWrl!?f=)`@QPrL`t%a9 z|HD}P&!U)}{VY8d#4-Ng3-C8A4a);#F8@D1ExImyckGV+fU&kgyySoP93;i7*ExEw zK$6x{)NgnFbo^a@u~qX&OZ)n@zc<@QnzXDl7{1u!aqD{M{|@iRORXob_vvx!S>wh3 z`RM=tr%wT)!|y11BbWd0uLy`PlA>v}&wZBt|GkTVD5(61VHl^j;7B$sPr_n5ER?J4 z$dWp!f7+(iQ?GKyU7*Ozu;T(a{ekVA2oPiSNw01@q)dA?-(pUZW1qB0UJ+p=(EiPBD{gYye>o~h9soK*`OFGHT;6@q59WgUC3UVFWg%sm zct*vX<=5Z8jTeF*kyjtqIjaA>8gCeNg-E3W>!mv+PzgH7C=g{j^K=`*!GN-Lj@|d4 zF`zETH`=9sQ6U<*7zvfP>*qlon64&P?6yRiMyvM&!c{Y{07`^q;MHB^mqkGuG)elC z$amB}g44~V0Vko#Z*CK=Tcojoy6!*0y=g4Jh%=)2Vyo3kM^$`cCCs68Z<@Mzmf?N%7u8m6~0FRnIYt-HSp~^47~?IMur2Jw$n?|bI)Z~H-v7B_ zY8DPGtt}ft4J5h@WG!=TkJGUWt(ue%AhM?*GkP-8ps!Jhz-h%M+SyQZ*XP#`4eX@` zY*K!Z{v#@TSgy40IWHl~Ra=|wKn3+cL1qbt6ec{7u0TOjCsy0kVbU~!ZvAU|5ovWb z)XfZFdXB}l0nTuefFQ&Len-yC$4hIKHX3*q&?N+PIB+HZFvEIoP_^e+Ep-sO?&3|~w_ z*)O>&ea_`TQxOdAqFTlzY^`2zJ$(CyYcCKML+IF$e!b7}+owwG7^Ej!d+~Dgz_(6u z8eSnaQ$59M-7027{o7?BciBuvNz$@krC_tE95qWV3^vty>ek#=rOKp2q~_e^FKtoB z=uM3*dzs@^?}yYl*D&-gyGgacuPIvgZ@0`5u3G)nWYAiT)W-2+KyS=)-^$sthp2I* z_tnk@u+YhF8Dhx6#8ROJa>B9Y7C1T)$^{+I2Ro$XZ-@sMzgBP#B7D-2jw&hT zmP#|cnZ)0daQ3lU-`d5gdAZTA^%#w+mlZ`9yVqtd;JZ0H3z>C#ABOe6$Bh@F)@oL; z`xA(fO1l*U+^=?UPe+KqmkHEs?ruEPa@(L-``(i6yb71xu|Cc+qyl^;Eykj{&H4i_ z^Cc_cHW}stvZSj;?@+rM>4n;l$fS-}Np7kjE#gpXkbDZ6ckB~h^6(R1YqcMBL{hIO`VZR`yaM=Uwh`+#OU!~*cLO9V34aUAY8*tAn~KR|jAXx|(0Y(3($xihyn$S+>nT2VU0`T`^!wqESphxV6T zlOxWwmxo?`h_ohj94IfiR6Q|2gWi;P|H5MXjm_*uyW{b>QDH}TX_?9V2Xo;1E5jz+ z6!P5p3NoHE`3B5ch8zD&kj~H{C*aSlP=URM_dbSfJ?8B9ptremS8TOg>+R#WBMu6M zGDHF78jc}z6_2ytDOQwnI$G_vMlxT&l?BrTQ>L+~W@>fWNHrdwJGk%kyZqb(-l-B# z@9kd$QA$^+CSJ{LzJh>Q367Uc*YVXkcA42mS*>Cv`GVMS@xEam!dGN!xR9^%q>H~( z_zW-kSdsZnWUVSDVg0$WpvZ_73vQ0v%7?!z%zay4SL6k0+Cv1>1HLa8z=mcPbZUlF zJjYdF+ixlU7}scj4!u(p@3&Ij6<4XdaLG}w7+Tgd@Eai_jw>nE(|&YVQ{?1wa>Kjs z9bpR_H8ESu#zsj@_1BuF!>;%wNl`ynVbNC;A29JMcGHi18Jzu#9hm4hpG zhc|?Fb++5Di{~~)<=#LS!L)c1dSJ?LE~sX-ezzaJE1c44c@Y2eggR4|eZX~a`@7#k zYDNBhChL=H7hRI9Td2}5JtiT9)7n74hVRzr@|k1DQ7jvCBtARDhCs$qNQ;FRdnt## zw<0U6d4{Ush#bofvFWG%mKx>&0PA;zH5I|n+b6FnAebgLlmnA-}qhNFJ1LLf;MQ>~q?zC_zc=_<#SNVg+Ttcu+KSn{fQfx<|%?PYP+c z?}cS*S))zdd|JK1Bbo!9Fj?H&46E)J49=m}i z207bfWyTGzr0ln9yz^>o8eY!g1SjrjE>>XSNw2CTx#s~o;}64t1w6Jv9pqEgtLo-S>|@@c%sbu`>31{^Z^ggnm`D}``TVDs2` zls*X9)msewF0AJ$cWk!;92LXMHVCG2Y;??~mZJ{5mb|ZAQ>hS2R3U)eY8D!(1Orxb zy^uD&@BF(wQGii#Y8Lx#fu!unr_ll$+>!jHIT+)mEK-B z;uC6XA-$Jj^W^>7C;RPG|I2%OD9Rat>b|Lfz>%S%dbvV+vO(4x6<=ayRR@DG~K!a^5N1 zJDC;OD7-kRr?B~LVs)A3ehm#TH?>>!veYgYhR^BJ4?4nG(%}aO5u_ve`hQBd7$R`wBah;m2xGZPRg9#%9 z%^qKq?O{m_s>6$OYo2WF@97;bJn}rV)#|im_OMaixkVBS*iYogdgV}&ogmm9LevFq z1H{vg#5T(Y&CskhQo-DLyT%?M^*?&&=>z-JF_mM$@*v2)B!3~=fy29Q4qVuu+8N3< zi`WvFG9=b`oyAs!NE#)Eox*QBsA@Un08A_Y;vSY*+vBo@{<30e+h^BOAzPpV#|Ysl zXO}BpV@=#yq1tu*)ZJPk4pwv67;P4e#gVx{#|i@h(Fl#22YTKE(J@8O6^JKk7F+$Zr{HaalJM#Q6tZboHe2+`TL4f0qHDGw{XEYf zQ{tHAJSs%66AHUW2+wkc2V+q7=y!^rB5zQac;lq5JXPkGHQ)%r{zCUKIkiMUh${Gy zu62WC^7nMA#)4I-;Tkrd;#>65p3~4|4~k zf8;<36$A`pK&(Nv=UL{y{n((xeSgPA9d|Gn^AYAnRPhvdz}W7-*m}%oZWjQnCPd+4 zaaE`ZI%8}2l0%xN#rj(ETZ!}=XdVIsC{pFFK~O(W#tj!ukLBgE$UBxkvYan2_^e&b zq+D;Xs|jp3Qs3zJ&7b?0<#1t=hKE%97R}SChv?qE>usLYpS0^3@VrygP1$rBq2UIc25EW7HZw<OqF{NU~IZE)%wvL?0>Qg$@*Ir6+qA?kJX5eJOiBdr-dSc@_B z>dS@|qnn1cqz-V{s^l-THfc1oWr*FkMlznF*G@!T6` z#tFDz#oixT(^yQR_~W>YnaTUQdZCz>K@GD;yCYDY3c|P3?8xPQ7CTv zWqXoylB;w2LL>qVZ-kMG-B3%N2+n1>_1k`2?qcB#B4;Zr>)VXwKtVZ6$#`SJaW$Lr z=#KSTwHNwR^=yJ=dWVI@`zL|f)4x?hSCt08@*K*tYxz`Npd`Lmb0Pfs*0@{+xDRTk zNkIIg8(eqSjY8V|23$B7!(+viA0e+0AS`%8o*l)qiV6S5rSymyHO4imr}yUA-uZ6k z`N_OXQMebls0pyJ#+Q)k*4Tru>7Cn4x2SwPNiph0FOC_+cSZD%kdv3S<@Me=?+n|S z&EjmIJrn~srA{8<@Qs*C99q&V2(NRhfD1vBY{4GuA_IeeevAH;t7>^1r;>-TidO0} zWhNk+xq>mQ&b~KXApnO&yN@1AhvN$X3I${p*3bUVBW5RzPyb5lOfIHMe*3W&NwQPg zk(p=aQeKR2$8{has~TVSQ=GcaWVp_DYsdu-l)q^zVu%or65j!lkHBGY$9k|qJ6#ep zp-n?%84i^b_E|^h8t!j~zK|J{oJ7U|!HJWn{Chcd@*h~=P~`}8Xo=1hjf!mo+0;t5 zUM;++x!tp;E!et4O=^J5o;-YE@7(r#F$Al8h5LqDrMIDazr+jQneH!49Gh5$c|QET z*X5$1I))sv`?iK%3ovYXcf9DQ;ee~)kI=;CHu%p9ykid3t`6#3~z?Qs?KDq z$D;q6vR7x}qQrflX0UI2jocov>g7A!?s#*T63W(s-vj`Z95Mdb5q|S!bWpL#P`b?5 zS{IS5&9GgukIy$jbt+6DY!Eo#y)X0@>pi5e1k6Fdhm<`(pOkz*)YI=8i2G6@@bDsY z{lzDR9k*$HAIYv3efEI1Jo9~3xIEpcEHcJ>nYn%&Az0FX5dC!gn?UpymYUIjS(tnW zB#sikxzuU#tY^3OF^M|M((guWpsYfgnp`J>+(ZVRbGVp<+r)nTgNGc9YxoubLE7YqwI?}vQgW+CNhibY8kL@`x|LKI9mjl`rA=ste$0j;xO=tUynA2$zGpi~GoD#$ zQph@E>rA0CphIzo(yn7jTA^$q#q=*KWvOyfNtVzz%bME!EAEw57Lw`UdTkMUWIMw{ zAp_PI9W}O^*xVkm3;w7t8I38d-Sx%K{oD(>#vOpm!d_0(+|IWC5v-Cw2U^uHTt^f8 z++KXtnz!~|Y6X41^2>JS780U;s#nV*ZqILU{0x!37r7-ZP;WpqLW==7K;=^1faz+~ z;{-Qpt2duEL+*MI=uyx?T0TvGY;{hvKd?!@E+hEsF5xO^=Gw;b)wE~yN*<9a&e}~` z&Xqa^bJ=Z#9y!h)j`WLuaOsg2-M56O${D)AM*LoB!FEpdy5=?wu!6*R)OISrS9~IW z7blQ(+s*FS^R*?AtV+P272~+2Qq9yWiBt`%oOXxIcJ#8?0`rZp62*p!AxtVTFk@O7 zVw>3rKokRkmjfHd@!wZ{F}?6rSodpRWk(c++XHq!@b-~3*Fk!Grt}Q+EtkmkYd%iI z8mbMRwq{R%V2pahmors*+ypd9M1ha9O}OL!Sa)DC`JwWvq3l*)@qDIzrB?D>h5%v7 zgy6G9ejJ3SU{yX}cMqE(G+(%67f`{WV(sz}S+Bu=unPt%T;4 zJY67(L7Lv)O&>^~usj39)%3hw2QN(wrq=OTAUTB2rdcF_{t);u;(d*O^qk#`q}30E zaC5)bR*qikOx?Oh#ExEF2*Y@uq57znh6Br23)Q;B!>@v-X)%z}YCEH@@ykCZWq{u^ zkFB^BsZv%fH|u`(QP5&r% zeLEELqe|J5mhydeB7%M+c==tiB}s*k3F9i8$?D^GOi6?|i!UFoW_v?V^Y;hc>Q=-K z2Rj9}qA&E^)_EnUgkaKg`4YEAmb$Dpx|E)fz8KZw!LSTB@aSte`pSn>hz_m+OY{{* zEFB?sY4rSqpTQJP;XW#xh48Pnq<97P9;E4v2 zpW@VJx-f>T4y!ZhBEH~ybDBmPGXrnA?wvLyicW2DfUC`k=Bfcqq7;*4ScsymsrWdV z(4tgL%VGt~dd#!x<%(MxHw2<@vVM4#Qv~CI z%tS6r6G@8?<;(m|Q;Met)dp_(NFVR15cPkH%5s*{hsa7Zm-!iB>tnwRlCc$cR$Ci zO7zSY$OzP?Sq=%b*2h3bQ^_b82q3G zzZaUjX7);n)Q@Yb;;u%PUXjiHj4m!q7P%(9=S`ehpPzxl3?ccEG}}&=g;Mf{lnT8l z8Pz7x9o#9OnE!=72tDa^`2G2No=@m%Zn_cbMEH^$rskYX&A9L^zun_%-48cLVK0Nv zZKFhppzci67=JK7wSGrz&~h6)c|(R1x{M_}X;tjx%sO{lGXk+(fbUpPDaGHbyyQi* zPNuea06>X4Jo$2#vwY~qHGihZW=FNa5!Oglv+WzD(NZBIkdL5mg9% zp?piXK4iXQJxI}Puua%k*&kofuto}wI(EFfVBV@sovM-g6C9bx@UcT{H4ve191;0!*7Zm$e7OSd<)d*Jt874rA%4tS=3J&$PHQT@!!L7m}U zQ@_--Ig)^>%qxrhFV@pXocNPM{2)$4K4GuVX!)XhU$4?s@6r86*1yBk1}@rcPwS9N zUq{Ol7<{b29Zt%UZ~Fa*qND!mjdPEzoW=@dcRK$598i~0bKV~?PyhZwkZ@hp$Jx<* zZo+(0%rCXUhHv+hZ-0HtiTF3~2HPd-BU=nm=qFoT)?P}KE0m1OcYR~#6y336dFrx- zaIRhQPIfMhwRHRIPUx!8Jo4a(&HJ2*kN)O59aI3)yd-74%KS*_8Wuo2gh^GMWnbHe1^?dmE+Y5)p}2%% zQ{~qTJ}P9G$_@^GDb3W^fDrm?mp(nMb2fUaFQXWFjq+AEV)au*v8$bjPw28sCc#%) zg|l3vfBjGcwwQp*5W(n0Wm9noJ`WFu5fB*2SK@4oiN!l^{Jkcma~CsdI|r#|qzpR7 z?~>D9!2pPY&Yk4x#Qo>b!~ROp`sAPyv4%wO)K+KqziJw_5O1t zed>sSnU+V}>CEx$<%`eKk_g%1)%+g8_!`}0!l;|}frRiC0xa7wDF z-6F=>Itm0aP^Wn3%|i2fr5lgJP5$#TpzrBI5A?%eJbH!my@4Tj!w#$p+bd-Yhi|;H z{(C|4$@f?#qIWYZhPG4=0OF-vH#iiwdv$b0@HYLwR~Ja@CE(j^LBo5T^&uV>%2^MP z5-hgAHd$WQu+H!!hcEoSr!qvFjC~%9smIheqDk@3*z;&727M+~MeP0^Ndy$K7CSDT zW0FG2c(-A*f!MX5CVID~?0B)mc>?t!A(HFwgL%OJW%rY$+Z!i zGF}Q&db9C2#m}w;Q{8P__S@R_eQga(9rx$ zGkw-GTSMym$Rdy!WHeOq3WU>|myIBjY}q6Eed;8 zq+)%kq=#0jwmK&?Sv~h&Z8b{Nbm!<-yZ#s}8MU&gIbOAP8*GW&AMWxTIGW>;xz-7p z_M>f=K3oxHUVJD)gz*c^W;Rv!$RBhsoKiFEa-!&U5BaKmh_HhKX0iM5~@~j&v z&*E{5=tjg!VhOIK&{-&9I>+eW-1-PG6zpk5;KDvpN zn!tNf4N6Iui3&~pItGoxO{GCk?^p{|IkzZng|v>dIcAbxTkloPMzD1ZXy@F|Y~L>8 zE5!jmDz;qT?oU^6ozBn`(C|1&@1@weYhQ=}@nulwok_ji+2XUrQK7%fPLM2IHPwa4ajmnNsH6q@%id|pke%-j6~P( z`}}aDS^~NRqGE5J0Z)QrjjzXC);@vqi+j4nvX!Mrx=6NgF!ilH_~|Y!+Mtk z%e^TQnY(3Mt99q8m{k_f6Uq{~FMifBz|k+*F357m8~BHmMY`xuJv7z3cymhqed57I zd2M@sqyt4QxSt)Lc@oV!B^Q}fLeS$*qaDS(z$`*?5L#Y<=DCe8UvQq~<|A_rXLLs? z0!^lJpGs9VQ!##(9c|!K6{F!8CdArrcWD}!6d3eFsyhsDhwh?BHTX2)J8z0KeSZiU z_Q9`WAN8=bVwIs#ld2VO7w3H_u_VDI3`PA-K`51^x z$@fUeZjX{Mi{ARJ+Z=b2D?<%70<G2~9M#udeLZ{PgC$H8s z$L3gUDg;z(X|bHI0ek2DPE3{eXn&uIkxGWDI8}`x81=@9LVVq}!_>8s5QUU4jhcKn z8|=$af~~eX$Uj7Xlz#0HHQke@)R8P3*uD^NEgk-j7|N4i^Q*b2issHmi|RQUV$zFO z@DHpUO)Lvqxa&KH1YSAFD>VC}E<+A5De~cEUDAhFi33{)pGb_0#U#1QBv{qT9Ie;O z54-}o+kiH^=!y@Cct|Pvdl=10Obm793@1#QLn3)xWDV`~W4pvL-5iK;Ukcao-n}4i zC2wRCS3PioGM^zID?=>StU8;X55$U>tTLdDnnPqx_QF2kBdHNgSu_LvWEu23qt>-Q z5w=pes;%Y&gM*&vpqWN~|Cp}O6QCbF*7r)!&WhOC6n=imCVtPZiKBSMFr_YXGNZmb5gAlz57Al z^lEI6CiW8#Zk63_3;1vt9dcK7n0oJ>4Pl}z`kBEr1{|bmV_~;pB&~{Y!+>bX=}Om0 zyZgyjQqmO%efN!qd;FYaO{PQ;ZWy{ZpO8!HA%WX;)JesK_pH&E&aUD`f`(L=(5jBg z=g0tuAHjBn74OtTTT1G|JY~u-_Jtkdg%MR=&LieLTlH)wHgmS1I*y>IHrz4Fxw=5q z=q+8%_JD4%{)aED`de{>F(7zb3K|Q*eT^sTd#}+m7~{WW<>)i`2Qe--5F>bGdBlJ$aD6x)DjW+ZOKh1h;CIRa=3+CpiYp znU?O)GRwb$=Ds+-x^E|)bO<Kqt#>d zEUh6Z_oKn5mMW^U!3)mK_s~*71QVBm5(5`R zwE|X;%yw3W7aD{;H>ObZ4KZ&xy}H~cMD2E8%8ADUf12frT6H`#<=v@sg(nBgJUsL# z?&$j@6p-z{{0uMGU(cfIp}S+E1(N;dmq=5I9i*cNyVO!^Z&bq@N9muJj*mALI=c}k z3;)R9S}E9=LrL-FLNE#^u#iKJwccPp%zmeHjc9ljUh@HmXolR*kWuJQ`p==zouQWl z9TJmo@v5PD!07i`BkiNXKS#uhf{sSKFET9Hh@e_=wF*8rD28Pje@BeYo%K#v``}%D zw{P%Pic3WdtnFaJMPJr+NUj8SF!ngq3tvquRi3d1|%T+ zOX42q+|}CC6Q_aaUXHddaNh~~IA#ZJ;q!!AutgiJ!8$ZbwylTEv|;8zNnY>sW&z9F zl6pZ4&5p_=hC#az4p&sp_k!(vShHF(>d&OjUJhi^rKUn@OjuOjy9NO{*0$sb4#-SF zQH(zzW{990Us?Fr=y@6OXfg?G<5d6{3Y_iS_OP*bS4LR+aPqY^%cY*Q`mO_{bM%LM zXkKQ!r7z+j1>y#_`a?#zSV>}lfo#8TkuDWa#j`50r>K6 zgQErmqId&I%8L|}d?78)^MQI7ik%PsLtgg-Oa~y0UIM=~Sxt;=` z`D1}Ve*Kz}I+pP9x%PS4!BY+_OHlA`=rAJ$Ic73fxx22=r$yf?%<7%y zqO#J0@OZ0=r6O}j)w?S{>eKsoRiWFF<+CAD?I1Bn!gAO3Xdaln#)WGYVjZgWOWl6S zs>a=t%!_s#Y**b!n{GaIL@g;Hvokz=!Truuthp(j_GWi7A&>kA#!W4>g@q0hhOwrZRFOG#3{7At_*r%8C$Z^`-$8!A!TG2 z^?kZpxdKT-kwwn6#+IEIq_IJ>Yh^dTy-5$I#g91-!WR%YchhMi_jO-3F|f;KYnyNj zIu&Q+nW`?opu=9R=PLC<{MQ2jn&iNE?{@(mtugconQ==qgJDjnOoc%>_94=lFhKOq z9%MFrYjx8%YA^4p4>(r9v9Qw0)3>Z0Gi-8$SV)J7YE2 z(RT>o=W2dlV2a^Ma<_{qCSh|&w8{Lms`- z5E{q~Kd zo*AfH6@%4QBN2M(xnJUb7&Zj-MH;5eqO{*G-@Jo@byrTkqnj`L^(s$ZUQc#D1tw>3 z+2dPZ^DHB7JIsum9ezh<2HS z?oj_ijm0(Kxz^hbevD9ov;f~w2sQj6Bd_-4fTIn&WT4pV&t!G?Hq`o&>NzFKrB&|5 zjdD+)+YEko`5MRb-2<$cPoMz!1!-AESIn1eYU%u}g8QMY2n@L^s%lel%j6 z;aV}kTQ&bxmy!a1U)%A@wVt-L(r2%kZUVyp6*=&2-_{=Qm;gP~Onmnk-%I9d7j;kx zi$#7UPdji+_+o$d{T0f-Gd)#5g^UJUbQAaY3GEl!ubESyA)b3nvO$DMSCJA$gKggRp+?b8*cV-*oW!Y(aJik4d;QfpUVNH1Ge zhY?@Rl0_=t{7!@>(^e88DpZHeyy3m*VJ?-Yj%n`@?_8~mZx~tQ?Z@K-aM#Tg-Fy^N`7Dn zJ+L0H`$$wTJ>l415iA`u6so}7u%GfF`KkgU_``j1Aq#VMnjQljzDb8;i*b`DJEiqc zP7VfZh5BiBfp^@QnxvMtqv~%$+_zvY{H}FWRMohJ7Tb6fJKik!u3U|}5CmZVoaNto znjyT9Q(j4bVz*$K*R@5D=OiJ9MqK(v1Z}y;XOE1z&(rf~k&*@7Rp zs*n9lC3CvpMQhZErjhvdlavrat%i*MvK6_SfnJJsy(~9GIL#3L+Je0))oP68(q(rPYRHxUCt4ZONXaI^v#7 zhFmNj1CFQ!?QD#=i}@X_o<^yGUFXkGcJO(9UBuGLe+>^guyJj=$Y2_2)JaT?~2*lsy4Ei?gPDa=mA5h1&5V&P2 zfn=qM*BK=iAQ~fdANkN~^A|fN*6O)t%KBE~Zq;<7V_+4b;v|xm6*?j+`MSEEPco_N zjLJE8zsoF^1|Q@q1~u_bOb%{_$P2hmYxKIjIxF}E(i~RndL0#ST{8@f z&aNDt;y14=tmmLOm(cwVYfn+V*D1l`x2;{;bp2u@KGrGRmH^&SY(22qAfv>xvhi{};4g#*c*shA?j`MVWJ)tho(%%|^ zpp@QwSvYJGADlYBUQZ=439tqE^fY;^vgr#cjuDo&q9*NEk4{pfjjJF&wnpkgq`9a| zhQpV^guh0Wht)!Y*+v8URC;a_OeQ2~H#C3jkqZiQO(Kt?pO>ZX8E%9OF0jKheo&hd2RfRqkWX zbaJ7xIt-aA&I9L0!t!F!z^nQXvnh*Fkl+qRs#O9`Kk^FdEZ@N z9>mA=s>*o0rcx~QWWX!37B|6Nm(omtmRSIpbL{?tM6LM=fMEtO{gf5!9Fp!+=hQ{0Pl?vX+CWk&LFc)?s zPJ!ozdi5{a1K26(_3;_Pvp>U7YgRJpc^I~|L_-Wc+vSdDs?a+LX~;!@-LZsZ$YADk zE6%r5zWFk491UN=Jo4IpuZa-52XKY@RiNy|7r|mO2i6LOJIK5pcmIi1AnT`e`m+2(Qn1QFtMg zNZmIwQK=Sq&hclaG6uo(J(TVxXWme6#YI<*Qr>edjMyCeVi~w(tYgEz!EwI<1s2|K zd?4tu{*xpaE1tH!3=oE*?6E)C(EqW{m7HA@M^=DQaJg{WpDEo`MB1nB?xKpTN7Rh^u`wp2P9y&4 z5@bf_|Fn0dQB7T27}A8n3J)#QLnbL@6cj}akzfMEn8=VQg9!K-!cYh(lVU(k!b7MS zkRTL%AqE06Rm3o=^#K+^<}ip%S{bU$hzNpO+LI7%ZGHZIf5?x!va;?yx%WGJ?|b&Q zhhrR#7@O1=qq8Y0agk{Z^+Kj<#O`)LU+xKjyKrB*`e*stTg=;(fmPA4_l?tSLHi*& z!y7FDviVRcO0S|W(2_)heT?uYHvGfB(xspg@_0@yTm~OTh#x9m>Py@IO$R!Ad7N=P zhI?z|f}zF175@o)8D!G7IV%U1xQT=X4O}aRvkzkEQldIEam}T|V!r;XxLuZD5l{!4 zp>(Cs-IEM1{SM*swlj-DvGLBOBi)LOU_J+Q1k2X-Qw zC#4ojlYa$g;PJO-aQSk;!zYRZ@IcW+NQg2rKyt<-4LNL4tOxusmds4Uqzt5 zdTaVyA!&?us9=)?dupAKzBsrwe1TULs z&4xErjoqHBRSQBiouiD%&>yE1Oz;oV-9pTxcr9M>YJSI-c;Cx(H^k3F<>Liet&$F2 z`U~2MGYr}4`PMr>DuTyMo2^z0YnDiMYc{q;9%FLy-|QD~@S}jnT(LEyco&rn^n@2L zlxX9BxQ7KZlYq)XcL3$sQ{c8OO+CC2yWDFc_o^rRN{!u?@d9Gs+4g@0wQYtXF?mO1 z0RRsa36G1>^WXNPBTKKxx0DPn{<1hGwp$8t<(W?;hJlCCGw$>y0FtxZmX@OOp{WmBXlZ&GcNX`MHRu_#}gg*sKeLsF^9`-j7h^i!xp}%|g>)g5H z^4!U*2*pUHt1n9*I=f{WRrY!BP`C4g*cFU)gpLq=8`)n>0D=Gq3NM)|;%A&|eWHh2 zbPlS8sYGD3emY9K9m}>f+c0$@2UwQK7bWzHt{_X~bMp?N6Wj`%u^_(=b{4x04k%b# z;BEWEEYT;WDLS=2!aXCrX*k5#W%(_s1IiU6W0^PayB#w@h8 zIr`X{u5gL)D>+)(!g!@D4L3pvX3D}Px248|QeYYVd+$7NpvuJw{f-0FMhTSfqq)hK zP~^k(X|+>BCe@p<;<*xc$uYA-xBOyd%nD9og+hknqyncqYn$^<5GGOETsmLCaA<70 zqEhfG`BXE5+lUYaM+tdUZ-UI|lNG;iD@j(`)8lOLiau>cYpoJbmZ3J_(q}H0iSyxu zo@(!7yq<@|>1-U#RglCIfURxdnptLPD=c{v5zlmpBv6VgjPdGe5u7##zQzd0Bm#sl z;9)$LOcYtn6vrbsvSg?_{fbi`!pp|uBKlH{76qGD7^_UWYN~5yoaYmA2-o|*_&xfp zcZW-nT=v5C4fj_=Vw0*9T8?Ffjs6#FY&clyleG64FCt^DwS*W117o6-dvA_n9(5c| z;^+z;O)5$m5;(?FVmN2!kjQvRk2D_nOYjtcpIU~IS(#*ws_vrZETqz~MXo{TYDj!i zb<*X#bhg1P^I&)ZkBH&cIa^HlDvR!iwkoat1#8UwK-imH1H@}SCoyjNIj*YdjS$*J z>As(15r|)yg-iFZnVqd@S$OLON&;w5KQq>&Vb!j^_+?qUMuK&*ysukC zedM%qsQ~)PX0`!0g33MP3p01^p2RB+iVELp^H6u1ZP;xYIxyx2Xln}tFfz*Qzh;Mbm?6K}&H5<+RBw=AD7O4Nv=?`s`A1G*!H+|zR zfI;)9xJRv*#ulg7_fFfQldNgr-75HP10UAkyDkukNHje53rGXb`!Q2(8>!Q{qiDVjcz2nt1+Z}O;{ zt~@xJfDxdSm< zDpw0>F3jqE?4DYt3wcDZ3>A-5{j*j z9tZyLBtH)j;$FMABiYB;fFu{nzLd45VY;ou;pxJyNaRJ;OB_}DM zlVH}aEvL=e1e?_f?lu5jtMH%?g~jn&`f7}{Q$0SYg-A(3IFKmK-=yRensP*Rjqt+1 z9cV|OJ)8FIO0Q?yfA%~<-4{qIz1F35{g1!+)VEI1WkAV}m_qpMaI_52P9d?d{D5dT z;!lkPenXi87%+z&wdI9M_rDIBIzTwQ4UD=3UA6zn1o`VXIR+5MPI(3C&o_ifzyTjC z1D{y_2~htuy3N-pth*@&VF1?DF!|`jJ-{Xl_&jo#>vPx literal 0 HcmV?d00001 diff --git a/doc/en/injection_tutorial.md b/doc/en/injection_tutorial.md new file mode 100644 index 0000000..655163e --- /dev/null +++ b/doc/en/injection_tutorial.md @@ -0,0 +1,328 @@ +# Tutorial: Inject Operator Step by Step + +> Author: Azure-Tang + +## TL;DR +This tutorial will guide you through the process of injecting custom operators into a model using the KTransformers framework. We will use the DeepSeekV2-Chat model as an example to demonstrate how to inject custom operators into the model step by step. The tutorial will cover the following topics: +* [How to write injection rules](#how-to-write-injection-rules) + * [Understanding the structure of the model](#understanding-model-structure) +* [Multi-GPU](#muti-gpu) +* [How to write a new operator and inject it into the model](#how-to-write-a-new-operator-and-inject-into-the-model) + +## How to Write Injection Rules +The basic form of the injection rules for the Inject framework is as follows: +```yaml +- match: + name: "^model\\.layers\\..*\\.*$" # Target module name + class: torch.nn.Linear # Target module + replace: + class: "default" + kwargs: + generate_device: "cuda:0" + # your_op_param_1: 1234 + # your_op_param_2: 5678 + recursive: True +``` +* match: This field marks the matching rules, which can appear in two forms, name and class. These two matching rules can appear together or separately; they only match when both criteria are met. +* replace: + * class: Python class that can be imported to replace the target module. If no replacement is desired, set to default. + * kwargs: List of parameters needed for module initialization. + * generate_device: The device for this module, can be set to “cpu”, “cuda”, “cuda:1”, etc. +* recursive: Whether to recursively inject this module’s submodules, default is True. + +For the recursive field: Some modules contain multiple submodules, such as the Self-attention module typically includes q/k/v/o four linear modules. If we replace the self-attention module but do not want the internal linear modules to be covered by other rules, set this rule to False. + +## Understanding Model Structure +Using [deepseek-ai/DeepSeek-V2-Lite-Chat](https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat) as an example, we can follow the above rules step by step to inject our custom module and run it. KTransformers offers a high degree of flexibility, allowing you to replace/experiment with basic operators. However, it also requires users to clearly understand the structure of the model they are running. + +Fortunately, knowing the structure of a model is very simple. Open the file list on the [deepseek-ai/DeepSeek-V2-Lite](https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/tree/main) homepage, and you can see the following files: +

    + + Inject-Struction + +

    + +From the `.saftensors` file, we can see the name of each layer’s weights, corresponding to the match.name attribute in the injection rules. +From the `modeling_deepseek.py` file, we can see the specific implementation of each module class, with the class name corresponding to the match.class attribute in the injection rules. + +The structure of the DeepSeekV2 model from the `.saftensors` and `modeling_deepseek.py` files is as follows: +

    + + Inject-Struction + +

    + +Supported operators and their corresponding classes are as follows: + +| match | replace | backends | descriptions | +| --------- | ---------------------- | ----------------------- | -------------------- | +| Linear | KTransformersLinear | KLinearMarlin | Marlin as backend | +| | | KLinearTorch | pytorch as backend | +| | | KLinearCPUInfer | llamafile as backend | +| experts | KTransformersExperts | KExpertsTorch | pytorch as backend | +| | | KExpertsMarlin | Marlin as backend | +| | | KExpertsCPU | llamafile as backend | +| Attention | KDeepseekV2Attention | KDeepseekV2Attention | MLA implementation | +| MoE | KMistralSparseMoEBlock | KQwen2MoeSparseMoeBlock | MoE for Qwen2 | +| | KDeepseekV2MoE | KDeepseekV2MoE | MoE for DeepseekV2 | +| Model | KQwen2MoeModel | KQwen2MoeModel | Model for Qwen2 | +| | KDeepseekV2Model | KDeepseekV2Model | Model for DeepseekV2 | +| RoPE | RotaryEmbedding | RotaryEmbedding | RoPE module | +| | YarnRotaryEmbedding | YarnRotaryEmbedding | RoPE module | + +Then we start step-by-step injection of custom modules, our targets are: + +* Replace the linear module with custom Marlin linear module. +* Replace the self-attention module with a custom Absorption-based MLA module. +* Replace the experts module with a custom Experts module. +* Replace the MoE module with a custom MoE module. +* Replace the RoPE module with a custom RoPE module. +* Set the running device for each module. + +The full implementation of the injection rules can be found in the [here](https://github.com/kvcache-ai/ktransformers/blob/main/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml). + +## Matrix Absorption-based MLA Injection + +For the injection of the Attention module, we only need to use a regular expression to match the module names used in transformers and replace them with our own MLA module implementation. The YAML injection rule is as follows: +```yaml +- match: + name: "^model\\.layers\\..*\\.self_attn$" # Regular expression + replace: + class: ktransformers.operators.attention.KDeepseekV2Attention # Optimized MLA implementation +``` +As you can see, each rule in the YAML file has two parts: match and replace. The match part specifies the module to be replaced, and the replace part specifies the module to be injected into the model along with the initialization keywords. + +## Injection of Routed Experts +For Routed Experts (corresponding to the exps in the diagram), the module we inject is CPUInfer, which is wrapped in the wrapper module KTransformersExperts. KTransformersExperts has multiple implementations, and we need to specify keywords to tell the wrapper module which implementation we want to use and how we plan to use it. + +In the source code of the transformer, MoE is implemented using nn.ModuleList. We do not want KTransformers to traverse all submodules in the list and inject them one by one, so in this rule, we set recursive: False to prevent recursive injection into the submodules of this module. The YAML rule is as follows: + +```yaml +- match: + name: "^model\\.layers\\..*\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersExperts # Custom MoE kernel with expert parallelism + kwargs: + generate_device: "cpu" + generate_op: "MLPCPUExperts" + out_device: "cuda" + recursive: False # Don't recursively inject submodules of this module +``` + +If we inject Routed Experts as a custom module, we cannot use the interfaces in the original `nn.ModuleList`. Therefore, it is necessary to modify the forward function in the FFN module. The simplest method is to implement a new module with a custom forward function and inject it. +```yaml +- match: + class: ktransformers.models.modeling_deepseek.DeepseekV2MoE + replace: + class: ktransformers.operators.experts.KDeepseekV2MoE # MLP module with custom forward function +``` + +## Injection of Linear Layers + +For the remaining linear layer modules, we aim to use quantized operators to save storage space while improving performance. Since there is no current research on using MLA and quantization together, we do not want to inject linear into the MLA operator. Therefore, we can modify the regular expression and add a type check in the match part of the rule. Only modules that match both the name and class simultaneously will be injected. We also need to pass some keywords similar to the injection of Routed Experts. The YAML rule is as follows: + +```yaml +- match: + name: "^model\\.layers\\.(?!.*self_attn).*$" # Regular expression + class: torch.nn.Linear # Only match modules matching name and class simultaneously + replace: + class: ktransformers.operators.linear.KTransformersLinear # Optimized kernel on quantized data types + kwargs: + generate_device: "cuda" + generate_op: "QuantizedLinearMarlin" +``` +## Injection of Modules with Pre-calculated Buffers + +To avoid occupying resources when initializing the injected original model, we use torch’s meta device to initialize the original model. The RoPE module pre-calculates some buffers during initialization, but no calculations are performed when using the meta device. Therefore, we need to compensate for the calculation of the buffer when loading the model. Simply, we inject a custom module into the rotary embedding module, which performs pre-calculation during loading. The YAML rule is as follows: +```yaml +- match: + class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding + replace: + class: ktransformers.operators.RoPE.YarnRotaryEmbedding +``` + +## Specifying Running Devices for Modules + +Finally, we set a fallback basic attribute generate_device for all modules: +```yaml +- match: + name: "^model\\.layers\\..*\\.|^lm_head" + replace: + class: "default" + kwargs: + generate_device: "cuda" + +- match: + name: "^model.embed_tokens" + replace: + class: "default" + kwargs: + generate_device: "cpu" +``` +Through these two rules, we place all previously unmatched layers (and their submodules) and lm_head on cuda, and the embedding on cpu. Note that the properties of a module will be determined by the first rule it matches. For example, if you later set a new replace.kwargs.generate_device in an injected module, the device set earlier will take precedence. If your computer has multiple cards, you can also configure the model to multiple cards. + + +## Muti-GPU + +If you have multiple GPUs, you can set the device for each module to different GPUs. +DeepseekV2-Chat got 60 layers, if we got 2 GPUs, we can allocate 30 layers to each GPU. Complete multi GPU rule examples [here](ktransformers/optimize/optimize_rules). + + +

    + + Inject-Struction + +

    + +First of all, for multi-GPU, we have to inject an new operator `KDeepseekV2Model`. And set division of the layers to different GPUs. For our case, we have to set the `transfer_map` in the `KDeepseekV2Model` operatoras as follows: + +```yaml +- match: + name: "^model$" + replace: + class: "ktransformers.operators.models.KDeepseekV2Model" + kwargs: + transfer_map: + 30: "cuda:1" +``` + +And we have to set the device for each module in the model. + +For example, for `routed experts`, the yaml for one GPU is: +```yaml +- match: + name: "^model\\.layers\\..*\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersExperts # Custom MoE kernel with expert parallelism + kwargs: + generate_device: "cuda:0" + generate_op: "MLPCUDAExperts" + out_device: "cuda:0" + recursive: False # Don't recursively inject submodules of this module +``` +But for two GPUs, we need to set the device for each module in the model. + +```yaml +# allcate 0-29 layers‘s out_device to cuda:0 +- match: + name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism + kwargs: + generate_device: "cpu" + generate_op: "KExpertsCPU" + out_device: "cuda:0" + recursive: False # don't recursively inject submodules of this module + +# allocate 30-59 layers‘s out_device to cuda:1 +- match: + name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$" + replace: + class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism + kwargs: + generate_device: "cpu" + generate_op: "KExpertsCPU" + out_device: "cuda:1" + recursive: False # don't recursively inject submodules of this module +``` +For other modules, we can set the device in the same way. + +## How to Write a New Operator and Inject into the Model + +In this section, we will explain how to write an operator that can be injected, using the implementation of a new linear as an example. + +First, all injectable operators need to inherit from the BaseInjectedModule class, which inherits some attributes required by our injection framework. Its initialization function needs to meet the following basic format: + +```python +class LinearTorchInject(BaseInjectedModule): + def __init__( + self, + key: str, + gguf_loader: GGUFLoader, + config: PretrainedConfig, + orig_module: nn.Module = None, + generate_device: str = "cuda", + **kwargs, + ): + super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs) +``` +If users have other parameters that need to be passed to this class, they can also be included in the init function and re-passed in the kwargs parameter in the yaml file. For example, if our operator wants to pass a parameter `my_param`, the init function can be written as: +```python +class LinearTorchInject(BaseInjectedModule): + def __init__( + self, + key: str, + gguf_loader: GGUFLoader, + config: PretrainedConfig, + orig_module: nn.Module = None, + generate_device: str = "cuda", + my_param: bool = True, + **kwargs, + ): + super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs) + self.my_param = my_param +``` +Then our injection rule can be written as: +```yaml +- match: + name: "^model\\.layers\\..*$" # Regular expression matches the module name. + class: torch.nn.Linear # Type restrictions can be added. + replace: + class: ktransformers.operators.linear.LinearTorchInject # Inject module path + kwargs: # Extra parameters + generate_device: "cuda" + my_param: True +``` +For the linear module, it is also necessary to read weights from a gguf file. We provide the `KLinearBase` class to help users read weights from gguf files. Users only need to inherit and implement the load, unload, and forward functions. Therefore, a fully injectable linear class would look like this: +```python +class LinearTorchInject(BaseInjectedModule, KLinearBase): + def __init__( + self, + key: str, + gguf_loader: GGUFLoader, + config: PretrainedConfig, + orig_module: nn.Module = None, + generate_device: str = "cuda", + **kwargs, + ): + super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs) + KLinearBase.__init__(self) + self.has_bias = False + self.dtype = torch.get_default_dtype() + self.w = None + self.has_bias = False + + def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = None): + if device is None: device = self.device + if w is None: w = self.load_weight(device=device) + + if isinstance(w, nn.Parameter): + self.w = w.to(dtype=self.dtype).view(self.out_features, self.in_features).T + self.has_bias = False + elif isinstance(w, tuple): + self.w = w[0].to(dtype=self.dtype).view(self.out_features, self.in_features).T + self.bias = w[1].to(dtype=self.dtype) + self.has_bias = True + else: + raise ValueError("Invalid weight type") + self.w = self.w.to(device) + if self.has_bias: + self.bias = self.bias.to(device) + + def unload(self): + if self.w is not None: + self.w = None + if self.has_bias: + self.bias = None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + out_device = x.device + x = x.to(device=self.device, dtype=self.dtype) + x = x @ self.w + if self.has_bias: + x = x + self.bias + x = x.to(dtype=dtype, device=out_device) + return x +``` +Note that the `self.load_weight` function is provided by the KLinearBase class to help users load weights from a gguf file into the module. The implementation details of KLinearBase can be found on [GITHUB](https://github.com/kvcache-ai/ktransformers/blob/44f57270c9514d79fab224186d90ccf61059331a/ktransformers/operators/linear.py#L31). diff --git a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml index 9d029a9..a2701e1 100644 --- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml +++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml @@ -5,17 +5,6 @@ kwargs: generate_device: "cuda" prefill_device: "cuda" -#- match: -# name: "^model\\.layers\\.([1-5][0-9])\\.mlp\\.shared_experts.*$" # regular expression -# class: torch.nn.Linear # only match modules matching name and class simultaneously -# replace: -# class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types -# kwargs: -# generate_device: "cpu" -# prefill_device: "cuda" -# generate_op: "KLinearCPUInfer" -# prefill_op: "KLinearTorch" -# out_device: "cuda" - match: name: "^model\\.layers\\.(?!.*self_attn).*$" # regular expression class: torch.nn.Linear # only match modules matching name and class simultaneously @@ -52,14 +41,6 @@ kwargs: generate_device: "cuda" prefill_device: "cuda" -- match: - name: "^model$" - replace: - class: "ktransformers.operators.models.KDeepseekV2Model" - kwargs: - generate_device: "cuda" - prefill_device: "cuda" - per_layer_prefill_intput_threshold: 0 # 0 is close layer wise prefill - match: name: "^model.embed_tokens" replace: