From 588126068b45f55a74a5e6d23d3fef4348322620 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Mon, 17 Oct 2022 12:32:46 -0400 Subject: [PATCH] Lewis model (#58) * Add skeleton lei stuff * Add lei notebook for now and poisson process fun study * Add stuff for ben to see * Add current changes for Ben once again (so needy :P) * Fixed the hang problem. * Commit what I have * Add currently broken code once again * Simplify notebook * Add working version of lewis * Add current progress * Add batching method and current logic of lei * Move lei stuff to its own package * Working on linear interpolation for the Lei problem. * inlaw -> outlaw. * Add lei test n_config * Fix settings.json * Add unit tests and update notebook with correct simulations * Update test, add simulation tests, add point batcher, share RNG * Update comment * JAX implementation of scipy.interpolate.interpn (#47) * JAX Interpolation. * JAX implementation of scipy.interpolate.interpn * Update todo list. * Add current version lol * Fix bugs and integrate good version * Fix small bug in stage 2 and clean up code * Modify interpn to work with multi-dimensional values * Add current version of notebook * WTF * Finish final lei * Fix test in outlaw * Add python notebook (weird vscode lol) * Add lei simulator batching method * Remove unnecessary files cluttering up space * Add current state * Add upper bound logic to lei example * Add ignore to frontend and update lei flow * Clean up lewis code and include some of Ben's changes * Add new script * Add new changes to make memory ok * Add full changes to everything except key * Add checkpointing * Add modified version * First pass at holder-odi bound in binomial.py * Holder-ODI, feeling more confident. * Add analyze lei example * Move lewis into confirm * Fix analyze notebook with new import structure * Add np.isnan check for holder bound and update lei analyze scripts * Moving files, small tweaks. * Pre-commit fixes. * Most tests passing. * Fix test stage1. Co-authored-by: James Yang --- .vscode/launch.json | 2 +- .vscode/settings.json | 249 ++-- confirm/confirm/lewislib/__init__.py | 0 confirm/confirm/lewislib/batch.py | 84 ++ confirm/confirm/lewislib/grid.py | 23 + confirm/confirm/lewislib/jax_wrappers.py | 48 + confirm/confirm/lewislib/lewis.py | 993 ++++++++++++++ confirm/confirm/lewislib/table.py | 133 ++ confirm/confirm/outlaw/interp.py | 98 ++ confirm/tests/lewis/test_hash.py | 85 ++ confirm/tests/lewis/test_n_configs.py | 235 ++++ .../tests/lewis/test_permute_invariance.py | 86 ++ .../tests/lewis/test_posterior_difference.py | 33 + confirm/tests/lewis/test_simulation.py | 77 ++ confirm/tests/test_interp.py | 29 + imprint/.vscode/build.sh | 4 +- imprint/frontend/.gitignore | 3 + imprint/frontend/tsconfig.json | 2 +- install.sh | 2 +- research/berry/berry_part1.ipynb | 12 +- research/berry/berry_part1.md | 2 +- research/lei/.gitignore | 1 + research/lei/analyze/analyze.ipynb | 321 +++++ research/lei/analyze/analyze.md | 196 +++ research/lei/analyze/download_data.sh | 10 + research/lei/lei.ipynb | 1177 +++++++++++++++++ research/lei/lei.md | 712 ++++++++++ research/stat/poisson_process.ipynb | 171 +++ research/stat/poisson_process.md | 111 ++ 29 files changed, 4763 insertions(+), 136 deletions(-) create mode 100644 confirm/confirm/lewislib/__init__.py create mode 100644 confirm/confirm/lewislib/batch.py create mode 100644 confirm/confirm/lewislib/grid.py create mode 100644 confirm/confirm/lewislib/jax_wrappers.py create mode 100644 confirm/confirm/lewislib/lewis.py create mode 100644 confirm/confirm/lewislib/table.py create mode 100644 confirm/confirm/outlaw/interp.py create mode 100644 confirm/tests/lewis/test_hash.py create mode 100644 confirm/tests/lewis/test_n_configs.py create mode 100644 confirm/tests/lewis/test_permute_invariance.py create mode 100644 confirm/tests/lewis/test_posterior_difference.py create mode 100644 confirm/tests/lewis/test_simulation.py create mode 100644 confirm/tests/test_interp.py create mode 100644 research/lei/.gitignore create mode 100644 research/lei/analyze/analyze.ipynb create mode 100644 research/lei/analyze/analyze.md create mode 100755 research/lei/analyze/download_data.sh create mode 100644 research/lei/lei.ipynb create mode 100644 research/lei/lei.md create mode 100644 research/stat/poisson_process.ipynb create mode 100644 research/stat/poisson_process.md diff --git a/.vscode/launch.json b/.vscode/launch.json index 306f58eb..0f49b091 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "program": "${file}", "console": "integratedTerminal", - "justMyCode": true + "justMyCode": false, } ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index b267766f..8c914ab4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,127 +1,126 @@ { - "bazel-cpp-tools.compileCommands.targets": [ - "//...", - ], - "jupyter.jupyterServerType": "local", - "files.associations": { - "functional": "cpp", - "*.evaluator": "cpp", - "*.traits": "cpp", - "fft": "cpp", - "openglsupport": "cpp", - "regex": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "any": "cpp", - "array": "cpp", - "atomic": "cpp", - "bit": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "cctype": "cpp", - "chrono": "cpp", - "cinttypes": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "codecvt": "cpp", - "complex": "cpp", - "condition_variable": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdint": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "deque": "cpp", - "forward_list": "cpp", - "list": "cpp", - "map": "cpp", - "set": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "vector": "cpp", - "exception": "cpp", - "algorithm": "cpp", - "iterator": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "string": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "utility": "cpp", - "hash_map": "cpp", - "fstream": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "ostream": "cpp", - "shared_mutex": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "typeinfo": "cpp", - "valarray": "cpp", - "variant": "cpp", - "filesystem": "cpp", - "locale": "cpp", - "mprealsupport": "cpp", - "nonlinearoptimization": "cpp", - "dense": "cpp", - "__bit_reference": "cpp", - "__bits": "cpp", - "__config": "cpp", - "__debug": "cpp", - "__errc": "cpp", - "__hash_table": "cpp", - "__locale": "cpp", - "__mutex_base": "cpp", - "__node_handle": "cpp", - "__nullptr": "cpp", - "__split_buffer": "cpp", - "__string": "cpp", - "__threading_support": "cpp", - "__tree": "cpp", - "__tuple": "cpp", - "compare": "cpp", - "concepts": "cpp", - "ios": "cpp", - "queue": "cpp", - "stack": "cpp", - "__functional_base": "cpp", - "alignedvector3": "cpp", - "typeindex": "cpp", - "*.ipp": "cpp", - "*.inc": "cpp", - "core": "cpp", - "geometry": "cpp", - "qtalignedmalloc": "cpp", - "matrixfunctions": "cpp", - "bvh": "cpp" - }, - "C_Cpp.errorSquiggles": "Enabled", - "C_Cpp.clang_format_fallbackStyle": "{ BasedOnStyle: LLVM, UseTab: Never, IndentWidth: 4, TabWidth: 4, AllowShortIfStatementsOnASingleLine: false, IndentCaseLabels: false, ColumnLimit: 100, AccessModifierOffset: -4, NamespaceIndentation: All, FixNamespaceComments: false, PointerAlignment: Left}", - "cmake.configureOnOpen": false, - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.enabled": true, - "python.formatting.provider": "black", - "autoDocstring.docstringFormat": "google-notypes", - "r.bracketedPaste": true, - "r.plot.useHttpgd": true + "bazel-cpp-tools.compileCommands.targets": ["//..."], + "jupyter.jupyterServerType": "local", + "files.associations": { + "functional": "cpp", + "*.evaluator": "cpp", + "*.traits": "cpp", + "fft": "cpp", + "openglsupport": "cpp", + "regex": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "any": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "cinttypes": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "codecvt": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "forward_list": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "utility": "cpp", + "hash_map": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "shared_mutex": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "typeinfo": "cpp", + "valarray": "cpp", + "variant": "cpp", + "filesystem": "cpp", + "locale": "cpp", + "mprealsupport": "cpp", + "nonlinearoptimization": "cpp", + "dense": "cpp", + "__bit_reference": "cpp", + "__bits": "cpp", + "__config": "cpp", + "__debug": "cpp", + "__errc": "cpp", + "__hash_table": "cpp", + "__locale": "cpp", + "__mutex_base": "cpp", + "__node_handle": "cpp", + "__nullptr": "cpp", + "__split_buffer": "cpp", + "__string": "cpp", + "__threading_support": "cpp", + "__tree": "cpp", + "__tuple": "cpp", + "compare": "cpp", + "concepts": "cpp", + "ios": "cpp", + "queue": "cpp", + "stack": "cpp", + "__functional_base": "cpp", + "alignedvector3": "cpp", + "typeindex": "cpp", + "*.ipp": "cpp", + "*.inc": "cpp", + "core": "cpp", + "geometry": "cpp", + "qtalignedmalloc": "cpp", + "matrixfunctions": "cpp", + "bvh": "cpp" + }, + "C_Cpp.errorSquiggles": "Enabled", + "C_Cpp.clang_format_fallbackStyle": "{ BasedOnStyle: LLVM, UseTab: Never, IndentWidth: 4, TabWidth: 4, AllowShortIfStatementsOnASingleLine: false, IndentCaseLabels: false, ColumnLimit: 100, AccessModifierOffset: -4, NamespaceIndentation: All, FixNamespaceComments: false, PointerAlignment: Left}", + "cmake.configureOnOpen": false, + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.enabled": true, + "python.formatting.provider": "black", + "autoDocstring.docstringFormat": "google-notypes", + "r.bracketedPaste": true, + "r.plot.useHttpgd": true, + "python.analysis.extraPaths": ["./outlaw", "./imprint/python"] } diff --git a/confirm/confirm/lewislib/__init__.py b/confirm/confirm/lewislib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/confirm/confirm/lewislib/batch.py b/confirm/confirm/lewislib/batch.py new file mode 100644 index 00000000..bf9f0229 --- /dev/null +++ b/confirm/confirm/lewislib/batch.py @@ -0,0 +1,84 @@ +import numpy as np + + +def pad_arg__(a, axis, n_pad: int): + pad_element = np.take(a, indices=0, axis=axis) + pad_element = np.expand_dims(pad_element, axis=axis) + new_shape = tuple(a.shape[i] if i != axis else n_pad for i in range(a.ndim)) + return np.concatenate((a, np.full(new_shape, pad_element)), axis=axis) + + +def create_batched_args__(args, in_axes, start, end, n_pad=None): + def arg_transform(arg, axis): + return pad_arg__(arg, axis, n_pad) if n_pad is not None else arg + + return [ + arg_transform( + np.take(arg, indices=range(start, end), axis=axis), + axis, + ) + if axis is not None + else arg + for arg, axis in zip(args, in_axes) + ] + + +def batch(f, batch_size: int, in_axes): + def internal(*args): + dims = np.array( + [arg.shape[axis] for arg, axis in zip(args, in_axes) if axis is not None] + ) + if len(dims) <= 0: + raise ValueError( + "f must take at least one argument " + "whose corresponding in_axes is not None." + ) + + dims_all_equal = np.sum(dims != dims[0]) == 0 + if not dims_all_equal: + raise ValueError( + "All batched arguments must have the same dimension " + "along their corresopnding in_axes." + ) + + dim = dims[0] + batch_size_new = min(batch_size, dim) + n_full_batches = dim // batch_size_new + remainder = dim % batch_size_new + n_pad = batch_size_new - remainder + pad_last = remainder > 0 + start = 0 + end = batch_size_new + + for _ in range(n_full_batches): + batched_args = create_batched_args__( + args=args, + in_axes=in_axes, + start=start, + end=end, + ) + yield (f(*batched_args), 0) + start += batch_size_new + end += batch_size_new + + if pad_last: + batched_args = create_batched_args__( + args=args, + in_axes=in_axes, + start=start, + end=dim, + n_pad=n_pad, + ) + yield (f(*batched_args), n_pad) + + return internal + + +def batch_all(f, batch_size: int, in_axes): + f_batch = batch(f, batch_size, in_axes) + + def internal(*args): + outs = tuple(out for out in f_batch(*args)) + return tuple(out[0] for out in outs), outs[-1][-1] + + return internal diff --git a/confirm/confirm/lewislib/grid.py b/confirm/confirm/lewislib/grid.py new file mode 100644 index 00000000..0bd3a11e --- /dev/null +++ b/confirm/confirm/lewislib/grid.py @@ -0,0 +1,23 @@ +import numpy as np +import pyimprint.grid as pygrid + + +def make_cartesian_grid_range(size, lower, upper): + assert lower.shape[0] == upper.shape[0] + + # make initial 1d grid + center_grids = ( + pygrid.Gridder.make_grid(size, lower[i], upper[i]) for i in range(len(lower)) + ) + + # make a grid of centers + coords = np.meshgrid(*center_grids) + centers = np.concatenate([c.flatten().reshape(-1, 1) for c in coords], axis=1) + + # make corresponding radius + radius = np.array( + [pygrid.Gridder.radius(size, lower[i], upper[i]) for i in range(len(lower))] + ) + radii = np.full(shape=centers.shape, fill_value=radius) + + return centers, radii diff --git a/confirm/confirm/lewislib/jax_wrappers.py b/confirm/confirm/lewislib/jax_wrappers.py new file mode 100644 index 00000000..91686676 --- /dev/null +++ b/confirm/confirm/lewislib/jax_wrappers.py @@ -0,0 +1,48 @@ +import jax.numpy as jnp + + +class ArraySlice0: + def __init__(self, a, start, end): + self.array = a + self.start = start + self.end = end # TODO: unused + + def __getitem__(self, index): + return self.array[self.start + index] + + +class ArrayReshape0: + def __init__(self, a, shape): + self.array = a + self.shape = shape + self.mask = jnp.flip(jnp.cumprod(jnp.flip(self.shape[1:]))) + + def __getitem__(self, index): + i = index[-1] + jnp.sum(self.mask * index[:-1]) + return self.array[i] + + +def slice0(a, start, end): + """ + Slices an array along axis 0 from start and end. + + Parameters: + ----------- + a: array to slice along axis 0. + start: starting position to slice. + end: ending position to slice (non-inclusive). + """ + return ArraySlice0(a, start, end) + + +def reshape0(a, shape): + """ + Reshapes a given array along the 0th axis + with a new shape. + + Parameters: + ----------- + a: array to reshape along axis 0. + shape: new shape of array along axis 0. + """ + return ArrayReshape0(a, shape) diff --git a/confirm/confirm/lewislib/lewis.py b/confirm/confirm/lewislib/lewis.py new file mode 100644 index 00000000..f257d246 --- /dev/null +++ b/confirm/confirm/lewislib/lewis.py @@ -0,0 +1,993 @@ +import jax +import jax.numpy as jnp +import numpy as np + +import confirm.outlaw.berry as berry +import confirm.outlaw.inla as inla +import confirm.outlaw.quad as quad +from confirm.lewislib import batch +from confirm.lewislib.table import LinearInterpTable +from confirm.lewislib.table import LookupTable + + +""" +The following class implements the Lei example. +See research/lei/lei.ipynb for the description. + +We define concepts used in the code: + +- `data in canonical form`: + `data` is of shape (n_arms, 2) + where n_arms is the number of arms in the trial and each row is a (y, n) + pair for the corresponding arm index. + The first row always corresponds to the control arm. + +- `n configuration`: + A valid sequence of `n` parameters in `data` + that is observable at any point in the trial. + +- `cached table`: + A cached table is assumed to be many table of values + row-stacked in the same order as a list of n configurations. + +- `pd`: + Posterior difference (between treatment and control arms): + P(p_i - p_0 < t | y, n) +- `pr_best`: + Posterior probability of best arm: + P(p_i = max_{j} p_j | y, n) +- `pps`: + Posterior probability of success: + P(Reject at stage 2 with all remaining + patients added to control and selected arm | + y, n, + selected arm = i) +""" + + +class Lewis45: + def __init__( + self, + n_arms: int, + n_stage_1: int, + n_stage_2: int, + n_stage_1_interims: int, + n_stage_1_add_per_interim: int, + n_stage_2_add_per_interim: int, + stage_1_futility_threshold: float, + stage_1_efficacy_threshold: float, + stage_2_futility_threshold: float, + stage_2_efficacy_threshold: float, + inter_stage_futility_threshold: float, + posterior_difference_threshold: float, + rejection_threshold: float, + sig2_int=quad.log_gauss_rule(15, 2e-6, 1e3), + n_sig2_sims: int = 20, + dtype=jnp.float64, + cache_tables=False, + **kwargs, + ): + """ + Constructs an object to run the Lei example. + + Parameters: + ----------- + n_arms: number of arms. + n_stage_1: number of patients to enroll at stage 1 for each arm. + n_stage_2: number of patients to enroll at stage 2 for each arm. + n_stage_1_interims: number of interims in stage 1. + n_stage_1_add_per_interim: number of total patients to + add per interim in stage 1. + n_stage_2_add_per_interim: number of patients to + add in stage 2 interim to control + and the selected treatment arms. + futility_threshold: probability cut-off to decide + futility for treatment arms. + If P(arm_i best | data) < futility_threshold, + declare arm_i as futile. + pps_threshold_lower: threshold for checking futility: + PPS < pps_threshold_lower <=> futility. + pps_threshold_upper: threshold for checking efficacy: + PPS > pps_threshold_upper <=> efficacy. + posterior_difference_threshold: threshold to compute posterior difference + of selected arm p and control arm p. + rejection_threshold: threshold for rejection at the final analysis + (if reached): + P(p_selected_treatment_arm - p_control_arm < + posterior_difference_threshold | data) + < rejection_threshold + <=> rejection. + """ + self.n_arms = n_arms + self.n_stage_1 = n_stage_1 + self.n_stage_2 = n_stage_2 + self.n_stage_1_interims = n_stage_1_interims + self.n_stage_1_add_per_interim = n_stage_1_add_per_interim + self.n_stage_2_add_per_interim = n_stage_2_add_per_interim + self.stage_1_futility_threshold = stage_1_futility_threshold + self.stage_1_efficacy_threshold = stage_1_efficacy_threshold + self.stage_2_futility_threshold = stage_2_futility_threshold + self.stage_2_efficacy_threshold = stage_2_efficacy_threshold + self.inter_stage_futility_threshold = inter_stage_futility_threshold + self.posterior_difference_threshold = posterior_difference_threshold + self.rejection_threshold = rejection_threshold + self.dtype = dtype + + # sig2 for quadrature integration + self.sig2_int = sig2_int + self.sig2_int.pts = self.sig2_int.pts.astype(self.dtype) + self.sig2_int.wts = self.sig2_int.wts.astype(self.dtype) + self.custom_ops_int = berry.optimized(self.sig2_int.pts, n_arms=n_arms).config( + opt_tol=1e-3 + ) + + # sig2 for simulation + self.sig2_sim = 10 ** jnp.linspace(-6, 3, n_sig2_sims, dtype=self.dtype) + self.dsig2_sim = jnp.diff(self.sig2_sim) + self.custom_ops_sim = berry.optimized(self.sig2_sim, n_arms=self.n_arms).config( + opt_tol=1e-3 + ) + + ## cache + # n configuration information + ( + self.n_configs_pr_best_pps_1, + self.n_configs_pps_2, + self.n_configs_pd, + ) = self.make_n_configs__() + + # diff_matrix[i]^T p = p[i+1] - p[0] + self.diff_matrix = np.zeros((self.n_arms - 1, self.n_arms)) + self.diff_matrix[:, 0] = -1 + np.fill_diagonal(self.diff_matrix[:, 1:], 1) + self.diff_matrix = jnp.array(self.diff_matrix) + + # order of arms used for auxiliary computations + self.order = jnp.arange(0, self.n_arms, dtype=int) + + # cache jitted internal functions + self.posterior_difference_table_internal_jit__ = None + self.pr_best_pps_1_internal_jit__ = None + self.pps_2_internal_jit__ = None + + # posterior difference tables for every possible combination of n + if cache_tables: + self.pd_table = self.posterior_difference_table__( + batch_size=kwargs["batch_size"] + ) + self.pr_best_pps_1_table = self.pr_best_pps_1_table__( + key=kwargs["key"], + n_pr_sims=kwargs["n_pr_sims"], + batch_size=kwargs["batch_size"], + ) + _, key = jax.random.split(kwargs["key"]) + self.pps_2_table = self.pps_2_table__( + key=key, + n_pr_sims=kwargs["n_pr_sims"], + batch_size=kwargs["batch_size"], + ) + + # =============================================== + # Table caching logic + # =============================================== + + def make_canonical__(self, data): + # we use the facts that: + # - arms that are not dropped always have + # n value at least as large as those that were dropped. + # - arms that are not dropped all have the same n values. + # This means a stable sort will always: + # - keep the first row in-place + # - only the treatment rows will be sorted + n = data[:, 1] + n_order = jnp.flip(n.shape[0] - 1 - jnp.argsort(jnp.flip(n), kind="stable")) + data = data[n_order] + data = jnp.stack((data[:, 0], data[:, 1] + 1), axis=-1) + n_order_inverse = jnp.argsort(n_order)[1:] - 1 + return data, n_order_inverse + + def make_n_configs__(self): + """ + Creates two 2-D arrays of all possible configurations of the `n` + Binomial parameter configurations throughout the trial. + Each row is a possible `n` configuration. + The first array contains all possible Phase II configurations. + The second array contains all possible Phase III configurations. + """ + + def internal(n_arr, n_add, n_interims, n_drop): + n_arms = n_arr.shape[-1] + out_all_ph2 = np.empty((0, n_arms), dtype=int) + + if n_interims <= 0: + return out_all_ph2 + + n_arr_new = np.copy(n_arr) + for n_drop_new in range(n_drop, n_arms - 1): + n_arr_incr = n_add // (n_arms - n_drop_new) + n_arr_new[n_drop_new:] += n_arr_incr + rest_all_ph2 = internal(n_arr_new, n_add, n_interims - 1, n_drop_new) + out_all_ph2 = np.vstack( + ( + out_all_ph2, + n_arr_new, + rest_all_ph2, + ) + ) + n_arr_new[n_drop_new:] -= n_arr_incr + + return out_all_ph2 + + # make array of all n configurations + n_arr = np.full(self.n_arms, self.n_stage_1, dtype=int) + n_configs_ph2 = internal( + n_arr, self.n_stage_1_add_per_interim, self.n_stage_1_interims, 0 + ) + n_configs_ph2 = np.vstack( + ( + n_arr, + n_configs_ph2, + ) + ) + n_configs_ph2 = np.unique(n_configs_ph2, axis=0) + + n_configs_ph2 = np.fliplr(n_configs_ph2) + + n_configs_ph3 = np.copy(n_configs_ph2) + n_configs_ph3[:, :2] += self.n_stage_2 + + n_configs_pr_best_pps_1 = n_configs_ph2 + n_configs_pps_2 = n_configs_ph3 + n_configs_pd = np.copy(n_configs_ph3) + n_configs_pd[:, :2] += self.n_stage_2_add_per_interim + + return n_configs_pr_best_pps_1, n_configs_pps_2, n_configs_pd + + def table_data__(self, ns, coords): + """ + Creates a data array used to construct internal tables. + + Parameters: + ----------- + ns: n parameter. + coords: result of calling jnp.meshgrid(..., indexing="ij") + + Returns: + -------- + data used for table construction. + """ + data = jnp.concatenate([c.flatten().reshape(-1, 1) for c in coords], axis=1) + n_arr = jnp.full_like(data, ns) + data = jnp.stack((data, n_arr), axis=-1) + return data + + def make_grid__(self, ns, n_points): + """ + Creates a 2-D array of shape (d, n_points) + where d is n.shape[0]. + Each row is a 1-D gridding of points for each entry of n + by creating evenly-spaced gridding from [0, n[i]). + The gridding always includes 0 and n[i]-1. + If n_points is greater than the min(n) it is clipped to be min(n). + """ + + def internal(n): + n_points_clip = jnp.minimum(jnp.min(n), n_points) + steps = (n - 1) // (n_points_clip - 1) + n_no_end = steps * (n_points_clip - 1) + return jnp.array( + [ + jnp.concatenate( + (jnp.arange(n_no_end[idx], step=steps[idx]), n[idx][None] - 1) + ) + for idx in range(len(n)) + ] + ) + + return jnp.array([internal(n) for n in ns]) + + def posterior_difference_table__( + self, + batch_size, + n_points=None, + ): + def internal(data): + return jax.vmap(self.posterior_difference, in_axes=(0,))(data) + + if n_points: + grid = self.make_grid__(self.n_configs_pd, n_points) + + def process_batch__(i, f, batch_size): + f_batched = batch.batch_all( + f, + batch_size, + in_axes=(0,), + ) + + if n_points: + meshgrid = jnp.meshgrid(*grid[i], indexing="ij") + else: + meshgrid = jnp.meshgrid( + *(jnp.arange(0, n + 1) for n in self.n_configs_pd[i]), indexing="ij" + ) + + outs, n_padded = f_batched( + self.table_data__(self.n_configs_pd[i], meshgrid) + ) + out = jnp.row_stack(outs) + return out[:(-n_padded)] if n_padded > 0 else out + + # if called for the first time, register jitted function + if self.posterior_difference_table_internal_jit__ is None: + self.posterior_difference_table_internal_jit__ = jax.jit(internal) + + tup_tables = tuple( + process_batch__( + i, self.posterior_difference_table_internal_jit__, batch_size + ) + for i in range(self.n_configs_pd.shape[0]) + ) + + if n_points: + return LinearInterpTable( + self.n_configs_pd + 1, + grid, + jnp.array(tup_tables), + ) + + else: + return LookupTable(self.n_configs_pd + 1, tup_tables) + + def pr_best_pps_1_table__(self, key, n_pr_sims, batch_size, n_points=None): + unifs = jax.random.uniform( + key=key, + shape=( + n_pr_sims, + self.n_stage_2 + self.n_stage_2_add_per_interim, + self.n_arms, + ), + ) + _, key = jax.random.split(key) + unifs_sig2 = jax.random.uniform( + key=key, + shape=(n_pr_sims,), + ) + _, key = jax.random.split(key) + normals = jax.random.normal(key, shape=(n_pr_sims, self.n_arms)) + + if n_points: + grid = self.make_grid__(self.n_configs_pr_best_pps_1, n_points) + + def internal(data): + return jax.vmap(self.pr_best_pps_1, in_axes=(0, None, None, None))( + data, normals, unifs_sig2, unifs + ) + + def process_batch__(i, f, batch_size): + f_batched = batch.batch_all( + f, + batch_size, + in_axes=(0,), + ) + + if n_points: + meshgrid = jnp.meshgrid(*grid[i], indexing="ij") + else: + meshgrid = jnp.meshgrid( + *(jnp.arange(0, n + 1) for n in self.n_configs_pr_best_pps_1[i]), + indexing="ij", + ) + + outs, n_padded = f_batched( + self.table_data__(self.n_configs_pr_best_pps_1[i], meshgrid) + ) + pr_best_outs = tuple(t[0] for t in outs) + pps_outs = tuple(t[1] for t in outs) + pr_best_out = jnp.row_stack(pr_best_outs) + pps_outs = jnp.row_stack(pps_outs) + return ( + (pr_best_out[:(-n_padded)], pps_outs[:(-n_padded)]) + if n_padded > 0 + else (pr_best_out, pps_outs) + ) + + # if called for the first time, register jitted function + if self.pr_best_pps_1_internal_jit__ is None: + self.pr_best_pps_1_internal_jit__ = jax.jit(internal) + + tup_tables = tuple( + process_batch__(i, self.pr_best_pps_1_internal_jit__, batch_size) + for i in range(self.n_configs_pr_best_pps_1.shape[0]) + ) + pr_best_tables = tuple(t[0] for t in tup_tables) + pps_tables = tuple(t[1] for t in tup_tables) + if n_points: + return LinearInterpTable( + self.n_configs_pr_best_pps_1 + 1, + grid, + (jnp.array(pr_best_tables), jnp.array(pps_tables)), + ) + else: + return LookupTable( + self.n_configs_pr_best_pps_1 + 1, (pr_best_tables, pps_tables) + ) + + def pps_2_table__(self, key, n_pr_sims, batch_size, n_points=None): + unifs = jax.random.uniform( + key=key, + shape=( + n_pr_sims, + self.n_stage_2_add_per_interim, + self.n_arms, + ), + ) + _, key = jax.random.split(key) + unifs_sig2 = jax.random.uniform( + key=key, + shape=(n_pr_sims,), + ) + _, key = jax.random.split(key) + normals = jax.random.normal( + key=key, + shape=(n_pr_sims, self.n_arms), + ) + + if n_points: + grid = self.make_grid__(self.n_configs_pps_2, n_points) + + def internal(data): + return jax.vmap(self.pps_2, in_axes=(0, None, None, None))( + data, normals, unifs_sig2, unifs + ) + + def process_batch__(i, f, batch_size): + f_batched = batch.batch_all( + f, + batch_size, + in_axes=(0,), + ) + + if n_points: + meshgrid = jnp.meshgrid(*grid[i], indexing="ij") + else: + meshgrid = jnp.meshgrid( + *(jnp.arange(0, n + 1) for n in self.n_configs_pps_2[i]), + indexing="ij", + ) + + outs, n_padded = f_batched( + self.table_data__(self.n_configs_pps_2[i], meshgrid) + ) + out = jnp.row_stack(outs) + return out[:(-n_padded)] if n_padded > 0 else out + + # if called for the first time, register jitted function + if self.pps_2_internal_jit__ is None: + self.pps_2_internal_jit__ = jax.jit(internal) + + tup_tables = tuple( + process_batch__(i, self.pps_2_internal_jit__, batch_size) + for i in range(self.n_configs_pps_2.shape[0]) + ) + if n_points: + return LinearInterpTable( + self.n_configs_pps_2 + 1, + grid, + jnp.array(tup_tables), + ) + else: + return LookupTable(self.n_configs_pps_2 + 1, tup_tables) + + def get_posterior_difference__(self, data): + data, n_order_inverse = self.make_canonical__(data) + return self.pd_table.at(data)[0][n_order_inverse] + + def get_pr_best_pps_1__(self, data): + data, n_order_inverse = self.make_canonical__(data) + outs = self.pr_best_pps_1_table.at(data) + return tuple(out[n_order_inverse] for out in outs) + + def get_pps_2__(self, data): + data, n_order_inverse = self.make_canonical__(data) + return self.pps_2_table.at(data)[0][n_order_inverse] + + # =============================================== + # Core routines for computing Bayesian quantities + # =============================================== + + def sample_posterior_sigma_sq(self, post, unifs): + """ + Samples from p(sigma^2 | data) given by the density + (up to a constant), post. + Assumes that post is computed on the grid self.sig2_sim in the same order. + The sampling is approximate as it samples from the discrete + measure defined by normalizing the histogram given by post. + """ + dFx = post[:-1] * self.dsig2_sim + Fx = jnp.cumsum(dFx) + Fx /= Fx[-1] + i_star = jnp.searchsorted(Fx, unifs) + return i_star + 1 + + def hessian_to_covariance(self, hess): + """ + Computes the covariance from the Hessian + (w.r.t. theta) of p(data, theta, sigma^2). + + Parameters: + ----------- + hess: tuple of (H_a, H_b) where H_a is of + shape (..., n) and H_b is of shape (..., 1). + The full Hessian is given by diag(H_a) + H_b 11^T. + + Returns: + -------- + Covariance matrix of shape (..., n, n) by inverting + and negating each term of hess. + """ + _, n_arms = hess[0].shape + hess_fn = jax.vmap( + lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1]) + ) + prec = -hess_fn(hess) # (n_sigs, n_arms, n_arms) + return jnp.linalg.inv(prec) + + def posterior_sigma_sq_int(self, data): + """ + Computes p(sigma^2 | data) using INLA on a grid defined by self.sig2_int.pts. + + Returns: + -------- + post: p(sigma^2 | data) evaluated on self.sig2_int.pts. + x_max: mode of x -> p(data, x, sigma^2) evaluated + for each point in self.sig2_int.pts. + hess: tuple of Hessian information (H_a, H_b) such that the Hessian + is given as in hessian_to_covariance(). + iters: number of iterations. + """ + + n_arms, _ = data.shape + sig2 = self.sig2_int.pts + n_sig2 = sig2.shape[0] + p_pinned = dict(sig2=sig2, theta=None) + f = self.custom_ops_int.laplace_logpost + logpost, x_max, hess, iters = f( + np.zeros((n_sig2, n_arms), dtype=self.dtype), p_pinned, data + ) + post = inla.exp_and_normalize(logpost, self.sig2_int.wts, axis=-1) + return post, x_max, hess, iters + + def posterior_sigma_sq_sim(self, data): + """ + Computes p(sigma^2 | data) using INLA on a grid defined by self.sig2_sim. + + Returns: + -------- + post: p(sigma^2 | data) (up to a constant) evaluated on self.sig2_sim. + x_max: mode of x -> p(data, x, sigma^2) + evaluated for each point in self.sig2_sim. + hess: tuple of Hessian information (H_a, H_b) such that + the Hessian is given as in hessian_to_covariance(). + iters: number of iterations. + """ + n_arms, _ = data.shape + sig2 = self.sig2_sim + n_sig2 = sig2.shape[0] + p_pinned = dict(sig2=sig2, theta=None) + logpost, x_max, hess, iters = self.custom_ops_sim.laplace_logpost( + np.zeros((n_sig2, n_arms), dtype=self.dtype), p_pinned, data + ) + max_logpost = jnp.max(logpost) + max_post = jnp.exp(max_logpost) + post = jnp.exp(logpost - max_logpost) * max_post + return post, x_max, hess, iters + + def posterior_difference(self, data): + """ + Computes p(p_i - p_0 < self.posterior_threshold | data) + for i = 1,..., d-1 where d is the total number of arms. + + Returns: + -------- + 1-D array of length d-1 where the ith component is + p(p_{i+1} - p_0 < self.posterior_threshold | data) + """ + post, x_max, hess, _ = self.posterior_sigma_sq_int(data) + + post_weighted = self.sig2_int.wts * post + cov = self.hessian_to_covariance(hess) + + def post_diff_given_sigma(mean, cov): + loc = self.diff_matrix @ mean + # var = [..., qi^T C qi, ..., ] where qi = self.diff_matrix[i] + var = jnp.sum((self.diff_matrix @ cov) * self.diff_matrix, axis=-1) + scale = jnp.sqrt(jnp.maximum(var, 0)) + normal_term = jax.scipy.stats.norm.cdf( + self.posterior_difference_threshold, loc=loc, scale=scale + ) + return normal_term + + normal_term = jax.vmap(post_diff_given_sigma, in_axes=(0, 0))(x_max, cov) + return post_weighted @ normal_term + + def pr_best(self, x): + """ + Computes P[X_i > max_{j != i} X_j] for each i = 0,..., d-1 + where x is of shape (..., d). + """ + x_argmax = jnp.argmax(x, axis=-1) + compute_best = jax.vmap(lambda i: self.order == i) + return jnp.mean(compute_best(x_argmax), axis=0) + + def pps(self, data, thetas, unifs): + # estimate P(A_i | y, n, theta_0, theta_i) + def simulate_Ai(data, arm, new_data): + new_data = jnp.where( + self.diff_matrix[arm].reshape((new_data.shape[0], -1)), new_data, 0 + ) + # pool outcomes for each arm + data = data + new_data + + return self.get_posterior_difference__(data)[arm] < self.rejection_threshold + + # compute p from logit space + p_samples = jax.scipy.special.expit(thetas) + berns = unifs < p_samples[:, None] + binoms = jnp.sum(berns, axis=1) + n_arr = jnp.full_like(binoms, unifs.shape[1]) + new_data = jnp.stack((binoms, n_arr), axis=-1) + + simulate_Ai_vmapped = jax.vmap( + jax.vmap(simulate_Ai, in_axes=(None, 0, None)), + in_axes=(None, None, 0), + ) + Ai_indicators = simulate_Ai_vmapped( + data, + self.order[:-1], + new_data, + ) + out = jnp.mean(Ai_indicators, axis=0) + return out + + def pr_best_pps_common(self, data, normals, unifs): + # compute p(sigma^2 | y, n), mode, hessian for simulation + # p(sigma^2 | y, n) is up to a constant + post, x_max, hess, _ = self.posterior_sigma_sq_sim(data) + + # compute covariance of theta | data, sigma^2 for each value of self.sig2_sim. + cov = self.hessian_to_covariance(hess) + chol = jnp.linalg.cholesky(cov) + + # sample from p(sigma^2 | data) by getting the indices of self.sig2_sim. + i_star = self.sample_posterior_sigma_sq(post, unifs) + + # sample theta from p(theta | data, sigma^2) given each sigma^2 from i_star. + mean_sub = x_max[i_star] + chol_sub = chol[i_star] + thetas = ( + jax.vmap(lambda chol, n: chol @ n, in_axes=(0, 0))(chol_sub, normals) + + mean_sub + ) + + return thetas + + def pr_best_pps_1(self, data, normals, unifs_sig2, unifs): + thetas = self.pr_best_pps_common(data, normals, unifs_sig2) + pr_best_out = self.pr_best(thetas)[1:] + pps_out = self.pps(data, thetas, unifs) + return pr_best_out, pps_out + + def pps_2(self, data, normals, unifs_sig2, unifs): + thetas = self.pr_best_pps_common(data, normals, unifs_sig2) + pps_out = self.pps(data, thetas, unifs) + return pps_out + + # =========== + # Trial Logic + # =========== + + def unifs_shape(self): + """ + Helper function that returns the necessary shape of + uniform draws for a single simulation to ensure enough Binomial + samples are guaranteed. + """ + # the n-configs used to compute posterior difference + # means that we've reached the very end of the simulation + # so it's sufficient to find the max n among these n-configs. + n_max = jnp.max(self.n_configs_pd) + return (n_max, self.n_arms) + + def sample(self, berns, berns_order, berns_start, n_new_per_arm): + berns_end = berns_start + n_new_per_arm + berns_subset = jnp.where( + ((berns_order >= berns_start) & (berns_order < berns_end))[:, None], + berns, + 0, + ) + n_new = jnp.full(shape=self.n_arms, fill_value=n_new_per_arm) + y_new = jnp.sum(berns_subset, axis=0) + data_new = jnp.stack((y_new, n_new), axis=-1) + return ( + data_new, + berns_end, + ) + + def score(self, data, p): + return data[:, 0] - data[:, 1] * p + + def stage_1(self, berns, berns_order, berns_start=0): + """ + Runs a single simulation of Stage 1 of the Lei example. + + Parameters: + ----------- + berns: a 2-D array of Bernoulli(p) draws of shape (n, d) where + n is the max number of patients to enroll + and d is the total number of arms. + berns_order: result of calling jnp.arange(0, berns.shape[0]). + It is made an argument to be able to reuse this array. + berns_start: starting row position into berns to begin accumulation. + + Returns: + -------- + data, n_non_futile, non_futile_idx, pr_best, berns_start + + data: (number of arms, 2) where column 0 is the + simulated binomial data for each arm + and column 1 is the corresponding value + for the Binomial n parameter. + n_non_futile: number of non-futile treatment arms. + non_futile_idx: vector of booleans indicating whether each arm is non-futile. + pr_best: vector containing probability of + being the best arm for each arm. + It is set to jnp.nan if the arm was dropped for + futility or if the arm is control (index 0). + berns_start: the next starting position to accumulate berns. + """ + + # aliases + n_arms = berns.shape[1] + n_stage_1 = self.n_stage_1 + n_interims = self.n_stage_1_interims + n_add_per_interim = self.n_stage_1_add_per_interim + futility_threshold = self.stage_1_futility_threshold + efficacy_threshold = self.stage_1_efficacy_threshold + + # create initial data + data, berns_start = self.sample( + berns=berns, + berns_order=berns_order, + berns_start=berns_start, + n_new_per_arm=n_stage_1, + ) + + # auxiliary variables + non_dropped_idx = jnp.ones(n_arms - 1, dtype=bool) + pr_best, pps = self.get_pr_best_pps_1__(data) + + # Stage 1: + def body_func(args): + ( + i, + _, + _, + data, + _, + non_dropped_idx, + pr_best, + pps, + berns_start, + ) = args + + # get next non-dropped indices + non_dropped_idx = (pr_best >= futility_threshold) * non_dropped_idx + n_non_dropped = jnp.sum(non_dropped_idx) + + # check for futility + early_exit_futility = n_non_dropped == 0 + + # check for efficacy + n_effective = jnp.sum(pps > efficacy_threshold) + early_exit_efficacy = n_effective > 0 + + # evenly distribute the next patients across non-dropped arms + # only if we are not early stopping stage 1. + # Note: for simplicity, we remove the remainder patients. + do_add = jnp.logical_not(early_exit_futility | early_exit_efficacy) + add_idx = jnp.concatenate( + (jnp.array(True)[None], non_dropped_idx), dtype=bool + ) + add_idx = add_idx * do_add + n_new_per_arm = n_add_per_interim // (n_non_dropped + 1) + data_new, berns_start = self.sample( + berns=berns, + berns_order=berns_order, + berns_start=berns_start, + n_new_per_arm=n_new_per_arm, + ) + data_new = jnp.where(add_idx[:, None], data_new, 0) + data = data + data_new + + pr_best, pps = self.get_pr_best_pps_1__(data) + + return ( + i + 1, + early_exit_futility, + early_exit_efficacy, + data, + n_non_dropped, + non_dropped_idx, + pr_best, + pps, + berns_start, + ) + + ( + _, + early_exit_futility, + _, + data, + _, + non_dropped_idx, + _, + pps, + berns_start, + ) = jax.lax.while_loop( + lambda tup: (tup[0] < n_interims) & jnp.logical_not(tup[1] | tup[2]), + body_func, + ( + 0, + False, + False, + data, + non_dropped_idx.shape[0], + non_dropped_idx, + pr_best, + pps, + berns_start, + ), + ) + + return ( + early_exit_futility, + data, + non_dropped_idx, + pps, + berns_start, + ) + + def stage_2( + self, + data, + best_arm, + berns, + berns_order, + berns_start, + ): + """ + Runs a single simulation of stage 2 of the Lei example. + + Parameters: + ----------- + data: data in canonical form. + best_arm: treatment arm index that is chosen for stage 2. + berns: a 2-D array of Bernoulli(p) draws of shape (n, d) + where n is the max number of patients and + d is the number of arms. + berns_order: result of calling jnp.arange(0, berns.shape[0]). + It is made an argument to be able to reuse this array. + berns_start: start row position into berns to start accumulation. + + Returns: + -------- + 0 if no rejection, otherwise 1. + """ + n_stage_2 = self.n_stage_2 + n_stage_2_add_per_interim = self.n_stage_2_add_per_interim + pps_threshold_lower = self.stage_2_futility_threshold + pps_threshold_upper = self.stage_2_efficacy_threshold + rejection_threshold = self.rejection_threshold + + non_dropped_idx = (self.order == 0) | (self.order == best_arm) + + # add n_stage_2 number of patients to each + # of the control and selected treatment arms. + data_new, berns_start = self.sample( + berns=berns, + berns_order=berns_order, + berns_start=berns_start, + n_new_per_arm=n_stage_2, + ) + data_new = jnp.where(non_dropped_idx[:, None], data_new, 0) + data = data + data_new + + pps = self.get_pps_2__(data)[best_arm - 1] + + # interim: check early-stop based on futility (lower) or efficacy (upper) + early_exit_futility = pps < pps_threshold_lower + early_exit_efficacy = pps > pps_threshold_upper + early_exit = early_exit_futility | early_exit_efficacy + early_exit_out = jnp.logical_not(early_exit_futility) | early_exit_efficacy + + def final_analysis(data, berns_start): + data_new, berns_start = self.sample( + berns=berns, + berns_order=berns_order, + berns_start=berns_start, + n_new_per_arm=n_stage_2_add_per_interim, + ) + data_new = jnp.where(non_dropped_idx[:, None], data_new, 0) + data = data + data_new + rej = ( + self.get_posterior_difference__(data)[best_arm - 1] + < rejection_threshold + ) + return (rej, data) + + return jax.lax.cond( + early_exit, + lambda: (early_exit_out, data), + lambda: final_analysis(data, berns_start), + ) + + def simulate(self, p, null_truths, unifs, unifs_order): + """ + Runs a single simulation of both stage 1 and stage 2. + + Parameters: + ----------- + p: simulation grid-point. + unifs: a 2-D array of uniform draws of shape (n, d) where + n is the max number of patients to enroll + and d is the total number of arms. + unifs_order: result of calling jnp.arange(0, unifs.shape[0]). + It is made an argument to be able to reuse this array. + """ + # construct bernoulli draws + berns = unifs < p[None] + + # Stage 1: + (early_exit_futility, data, non_dropped_idx, pps, berns_start) = self.stage_1( + berns=berns, + berns_order=unifs_order, + ) + + # if early-exited because of efficacy, + # pick the best arm based on PPS along with control. + # otherwise, pick the best arm based on pr_best along with control. + best_arm_info = jnp.where(non_dropped_idx, pps, -1) + best_arm = jnp.argmax(best_arm_info) + 1 + + early_exit = early_exit_futility | ( + pps[best_arm - 1] < self.inter_stage_futility_threshold + ) + + def stage_2_wrap( + null_truths, data, p, best_arm, berns, unifs_order, berns_start + ): + rej, data = self.stage_2( + data=data, + best_arm=best_arm, + berns=berns, + berns_order=unifs_order, + berns_start=berns_start, + ) + false_rej = rej * null_truths[best_arm - 1] + score = self.score(data, p) * false_rej + return (false_rej, score) + + # Stage 2 only if no early termination based on futility + return jax.lax.cond( + early_exit, + lambda: (False, jnp.zeros(self.n_arms)), + lambda: stage_2_wrap( + null_truths=null_truths, + data=data, + p=p, + best_arm=best_arm, + berns=berns, + unifs_order=unifs_order, + berns_start=berns_start, + ), + ) diff --git a/confirm/confirm/lewislib/table.py b/confirm/confirm/lewislib/table.py new file mode 100644 index 00000000..83f7d88f --- /dev/null +++ b/confirm/confirm/lewislib/table.py @@ -0,0 +1,133 @@ +import jax.numpy as jnp + +import confirm.lewislib.jax_wrappers as jwp +from confirm.outlaw.interp import interpn + + +class BaseTable: + def __init__(self, n_sizes): + # compute mask to hash n_sizes + n_arms = n_sizes.shape[-1] + n_sizes_max = jnp.max(n_sizes) + 1 + n_sizes_max_mask = n_sizes_max ** jnp.arange(0, n_arms) + self.n_sizes_max_mask = n_sizes_max_mask.astype(int) + + # create hashes + hashes = jnp.array([self.hash_n__(ns) for ns in n_sizes]) + + # reorder data based on increasing order of hashes + self.hashes_order = jnp.argsort(hashes) + self.hashes = hashes[self.hashes_order] + + def hash_n__(self, n): + """ + Hashes the n configuration with a given mask. + + Parameters: + ----------- + n: n configuration sorted in decreasing order. + """ + return jnp.sum(n * self.n_sizes_max_mask) + + def search(self, n): + n_hash = self.hash_n__(n) + idx = jnp.searchsorted(self.hashes, n_hash) + return idx + + def hash_ordered(self, seq): + return tuple(seq[i] for i in self.hashes_order) + + +class LinearInterpTable(BaseTable): + def __init__(self, n_sizes, grids, tables): + """ + Parameters: + ----------- + n_sizes: a 2-D array of shape (n, d). + grid: a 3-D array of shape (n, d, a). + tables: a sequence of N-D arrays + each of shape (n, a^d, ...) where each slice + (a^d, ...) corresponds to values in a cartesian + product of points defined by the same slice of grid. + """ + super().__init__(n_sizes) + if not isinstance(tables, tuple): + tables = (tables,) + + self.grids = grids[self.hashes_order] + + self.tables = tuple(sub_tables[self.hashes_order] for sub_tables in tables) + + n_arms, n_points = self.grids.shape[-2:] + self.shape = tuple(n_points for _ in range(n_arms)) + + def at(self, data): + y = data[:, 0] + n = data[:, 1] + idx = self.search(n) + grid = self.grids[idx] + return tuple( + interpn(grid, values[idx].reshape(self.shape + values[idx].shape[1:]), y) + for values in self.tables + ) + + +class LookupTable(BaseTable): + def __init__( + self, + n_sizes, + tables, + ): + """ + Constructs a lookup table given a list of n sizes + and their corresponding table of values corresponding to + all enumerations of the sizes. + + Parameters: + ----------- + n_sizes: a 2-D array of shape (n, d) where n is the number + of configurations and d is the number of arms. + tables: a list of list of/list of/table of values. + If it is not a list of list of tables, + it will be converted in such a form. + In that form, tables[i] corresponds to the ith table + where tables[i][j] is a sub-table of values + corresponding to the configuration n_sizes[j]. + tables[i][j] is assumed to be of shape + (jnp.prod(n_sizes[j]), ...). + Each row is a value corresponding to a row of + a d-dimensional possible configuration y, where + 0 <= y < n_sizes, where the first index increments slowest + and the last index increments fastest. + """ + super().__init__(n_sizes) + + # force tables to be a tuple of (tuple of sub-tables) + if not isinstance(tables, tuple): + tables = ((tables,),) + if not isinstance(tables[0], tuple): + tables = (tables,) + + tables_reordered = tuple(self.hash_ordered(sub_tables) for sub_tables in tables) + self.tables = tuple( + jnp.row_stack(sub_tables) for sub_tables in tables_reordered + ) + + # reorder based on hash order + n_sizes = n_sizes[self.hashes_order] + + # compute offsets corresponding to each n_size + sizes = jnp.array([0] + [jnp.prod(ns) for ns in n_sizes]) + sizes_cumsum = jnp.cumsum(sizes) + self.offsets = sizes_cumsum[:-1] + self.sizes = sizes[1:] + + def at(self, data): + index = data[:, 0] + n = data[:, 1] + idx = self.search(n) + offset = self.offsets[idx] + size = self.sizes[idx] + slices = tuple(jwp.slice0(t, offset, offset + size) for t in self.tables) + slices_reshaped = tuple(jwp.reshape0(a, n) for a in slices) + return tuple(a[index] for a in slices_reshaped) diff --git a/confirm/confirm/outlaw/interp.py b/confirm/confirm/outlaw/interp.py new file mode 100644 index 00000000..abfc94c3 --- /dev/null +++ b/confirm/confirm/outlaw/interp.py @@ -0,0 +1,98 @@ +import jax.numpy as jnp + + +def interpn(points, values, xi): + """ + A JAX reimplementation of scipy.interpolate.interpn. Most of the input + validity checks have been removed, so make sure your inputs are correct or + go implement those checks yourself. + + In addition, the keyword arguments are: + - `method="linear"` + - `bounds_error=False` + - `fill_value=None` + + The scipy source is here: + https://github.com/scipy/scipy/blob/651a9b717deb68adde9416072c1e1d5aa14a58a1/scipy/interpolate/_rgi.py#L445-L614 + + The original docstring from scipy: + Multidimensional interpolation on regular or rectilinear grids. + + Strictly speaking, not all regular grids are supported - this function + works on *rectilinear* grids, that is, a rectangular grid with even or + uneven spacing. + + Args: + points : tuple of ndarray of float, with shapes (m1, ), ..., (mn, ) + The points defining the regular grid in n dimensions. The points in + each dimension (i.e. every elements of the points tuple) must be + strictly ascending or descending. + values : array_like, shape (m1, ..., mn, ...) + The data on the regular grid in n dimensions. Complex data can be + acceptable. + xi : ndarray of shape (..., ndim) + The coordinates to sample the gridded data at + + Returns: + values_x : ndarray, shape xi.shape[:-1] + values.shape[ndim:] + Interpolated values at input coordinates. + """ + + grid = tuple([jnp.asarray(p) for p in points]) + indices, norm_distances = _find_indices(grid, xi) + return _evaluate_linear(grid, values, indices, norm_distances) + + +# This code is copied from scipy.interpolate.interpn and modified for working with JAX. +def _find_indices(grid, xi): + + # find relevant edges between which xi are situated + indices = [] + # compute distance to lower edge in unity units + norm_distances = [] + + for i in range(len(grid)): + g = grid[i] + idx = jnp.searchsorted(g, xi[i]) - 1 + idx = jnp.where(idx > 0, idx, 0) + idx = jnp.where(idx > g.size - 2, g.size - 2, idx) + indices = indices + [idx] + denom = g[idx + 1] - g[idx] + norm_distances = norm_distances + [ + jnp.where(denom != 0, (xi[i] - g[idx]) / denom, 0) + ] + + indices = jnp.array(indices) + norm_distances = jnp.array(norm_distances) + return indices, norm_distances + + +def _evaluate_linear(grid, values, indices, norm_distances): + d = len(grid) + # Construct the unit d-dimensional cube. + unit_cube = jnp.meshgrid(*[jnp.array([0, 1]) for i in range(d)], indexing="ij") + + # Choose the left or right index for each corner of the hypercube. these + # are 1D indices which get used in will later be used to construct the ND + # indices of each corner. + hypercube_dim_indices = [ + jnp.array([indices[i], indices[i] + 1])[unit_cube[i]] for i in range(d) + ] + # the final indices will be the unraveled ND indices produced from the 1D + # indices above. + hypercube_indices = tuple(hypercube_dim_indices[i].flatten() for i in range(d)) + + # the weights for the left and right sides of each 1D interval. + # norm_distance is the normalized distance from the left edge so the weight + # will be (1 - norm_distance) for the left edge + hypercube_dim_weights = jnp.array( + [ + jnp.array([1 - norm_distances[i], norm_distances[i]])[unit_cube[i]] + for i in range(d) + ] + ) + # the final weights will be the product of the weights for each dimension + hypercube_weights = jnp.prod(hypercube_dim_weights, axis=0).ravel() + + # finally, select the values to interpolate and multiply by the weights. + return hypercube_weights @ values[hypercube_indices] diff --git a/confirm/tests/lewis/test_hash.py b/confirm/tests/lewis/test_hash.py new file mode 100644 index 00000000..8694a944 --- /dev/null +++ b/confirm/tests/lewis/test_hash.py @@ -0,0 +1,85 @@ +import jax +import jax.numpy as jnp +import numpy as np + +from confirm.lewislib.table import LookupTable + + +default_params = { + "n_arms": 3, + "n_stage_1": 3, + "n_stage_2": 3, + "n_stage_1_interims": 1, + "n_stage_1_add_per_interim": 10, + "n_stage_2_add_per_interim": 4, + "stage_1_futility_threshold": 0.1, + "stage_2_futility_threshold": 0.1, + "stage_1_efficacy_threshold": 0.1, + "stage_2_efficacy_threshold": 0.9, + "inter_stage_futility_threshold": 0.8, + "posterior_difference_threshold": 0.05, + "rejection_threshold": 0.05, +} + + +def test_stable_sort_1(): + n = jnp.array([20, 20, 10, 20]) + order = jnp.flip(n.shape[0] - 1 - jnp.argsort(jnp.flip(n), kind="stable")) + expected = jnp.array([0, 1, 3, 2]) + assert jnp.array_equal(order, expected) + + +def test_stable_sort_2(): + n = jnp.array([30, 10, 20, 30]) + order = jnp.flip(n.shape[0] - 1 - jnp.argsort(jnp.flip(n), kind="stable")) + expected = jnp.array([0, 3, 2, 1]) + assert jnp.array_equal(order, expected) + + +def test_hash_undo(): + n = jnp.array([12, 5, 5, 12]) + n_order = jnp.flip(n.shape[0] - 1 - jnp.argsort(jnp.flip(n), kind="stable")) + + # test if this piece of code gives us the correct undoing + actual = jnp.argsort(n_order) + expected = jnp.array([0, 2, 3, 1]) + + assert jnp.array_equal(actual, expected) + + +def test_y_to_index(): + n = np.array([5, 5, 5, 2]) + max_idx = np.prod(n + 1) + y = np.zeros(n.shape[0]) + + def increment(y): + carry = 1 + for i in range(n.shape[-1] - 1, -1, -1): + y[i] += carry + carry = y[i] // (n[i] + 1) + y[i] -= carry * (n[i] + 1) + return y + + for i in range(max_idx): + actual = y[-1] + jnp.sum(jnp.flip(y[:-1]) * jnp.cumprod(jnp.flip(n[1:] + 1))) + expected = i + assert jnp.array_equal(actual, expected) + y = increment(y) + + +def test_hash(): + n = jnp.array([12, 5, 5, 12]) + dims = n + 1 + values = jnp.arange(0, jnp.prod(dims))[:, None] + table = LookupTable(dims[None], values) + y = jnp.array([5, 1, 0, 10]) + + # tests if the at function is jit-able also. + @jax.jit + def internal(): + data = jnp.stack((y, dims), axis=-1) + return table.at(data)[0].squeeze() + + actual = internal() + expected = 2428 + assert jnp.array_equal(actual, expected) diff --git a/confirm/tests/lewis/test_n_configs.py b/confirm/tests/lewis/test_n_configs.py new file mode 100644 index 00000000..26690d79 --- /dev/null +++ b/confirm/tests/lewis/test_n_configs.py @@ -0,0 +1,235 @@ +import numpy as np + +from confirm.lewislib import lewis + + +default_params = { + "n_arms": 3, + "n_stage_1": 10, + "n_stage_2": 10, + "n_stage_1_interims": 3, + "n_stage_1_add_per_interim": 4, + "n_stage_2_add_per_interim": 4, + "stage_1_futility_threshold": 0.1, + "stage_2_futility_threshold": 0.1, + "stage_1_efficacy_threshold": 0.9, + "stage_2_efficacy_threshold": 0.9, + "inter_stage_futility_threshold": 0.8, + "posterior_difference_threshold": 0.05, + "rejection_threshold": 0.05, +} + + +def run_n_configs_test(actual, expected): + def check_equality(actual, expected): + # lexicographical sort to order the rows consistently + actual_sorted = np.lexsort(actual) + expected_sorted = np.lexsort(expected) + assert np.array_equal(actual_sorted, expected_sorted) + + for n_configs, n_configs_expected in zip(actual, expected): + check_equality(n_configs, n_configs_expected) + + +def make_expected( + n_configs_pr_best_pps_1_expected, + n_stage_2, + n_stage_2_add_per_interim, +): + n_configs_pps_2_expected = np.copy(n_configs_pr_best_pps_1_expected) + n_configs_pps_2_expected[:, :2] += n_stage_2 + n_configs_pd_expected = np.copy(n_configs_pps_2_expected) + n_configs_pd_expected[:, :2] += n_stage_2_add_per_interim + expected = ( + n_configs_pr_best_pps_1_expected, + n_configs_pps_2_expected, + n_configs_pd_expected, + ) + return expected + + +def test_3_arms_0_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 3 + default_params["n_stage_1"] = 10 + default_params["n_stage_2"] = 10 + default_params["n_stage_1_interims"] = 0 + default_params["n_stage_1_add_per_interim"] = 4 + default_params["n_stage_2_add_per_interim"] = 4 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [10, 10, 10], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) + + +def test_3_arms_1_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 3 + default_params["n_stage_1"] = 10 + default_params["n_stage_2"] = 10 + default_params["n_stage_1_interims"] = 1 + default_params["n_stage_1_add_per_interim"] = 4 + default_params["n_stage_2_add_per_interim"] = 4 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [10, 10, 10], + [11, 11, 11], + [12, 12, 10], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) + + +def test_3_arms_2_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 3 + default_params["n_stage_1"] = 5 + default_params["n_stage_2"] = 15 + default_params["n_stage_1_interims"] = 2 + default_params["n_stage_1_add_per_interim"] = 7 + default_params["n_stage_2_add_per_interim"] = 4 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [5, 5, 5], + [7, 7, 7], + [8, 8, 5], + [9, 9, 9], + [10, 10, 7], + [11, 11, 5], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) + + +def test_4_arms_0_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 4 + default_params["n_stage_1"] = 10 + default_params["n_stage_2"] = 10 + default_params["n_stage_1_interims"] = 0 + default_params["n_stage_1_add_per_interim"] = 4 + default_params["n_stage_1_add_per_interim"] = 10 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [10, 10, 10, 10], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) + + +def test_4_arms_1_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 4 + default_params["n_stage_1"] = 10 + default_params["n_stage_2"] = 10 + default_params["n_stage_1_interims"] = 1 + default_params["n_stage_1_add_per_interim"] = 4 + default_params["n_stage_2_add_per_interim"] = 20 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [10, 10, 10, 10], + [11, 11, 11, 11], + [11, 11, 11, 10], + [12, 12, 10, 10], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) + + +def test_4_arms_2_interim(): + # re-setting parameters to make it clear which parameters affect this function. + default_params["n_arms"] = 4 + default_params["n_stage_1"] = 10 + default_params["n_stage_2"] = 10 + default_params["n_stage_1_interims"] = 2 + default_params["n_stage_1_add_per_interim"] = 4 + default_params["n_stage_2_add_per_interim"] = 1 + + lewis_obj = lewis.Lewis45(**default_params) + actual = lewis_obj.make_n_configs__() + + # expected values + n_configs_pr_best_pps_1_expected = np.array( + [ + [10, 10, 10, 10], + [11, 11, 11, 11], + [11, 11, 11, 10], + [12, 12, 10, 10], + [12, 12, 12, 12], + [12, 12, 12, 11], + [13, 13, 11, 11], + [12, 12, 12, 10], + [13, 13, 11, 10], + [14, 14, 10, 10], + ] + ) + expected = make_expected( + n_configs_pr_best_pps_1_expected, + default_params["n_stage_2"], + default_params["n_stage_2_add_per_interim"], + ) + + # tests + run_n_configs_test(actual, expected) diff --git a/confirm/tests/lewis/test_permute_invariance.py b/confirm/tests/lewis/test_permute_invariance.py new file mode 100644 index 00000000..ccb8e6f1 --- /dev/null +++ b/confirm/tests/lewis/test_permute_invariance.py @@ -0,0 +1,86 @@ +import jax +import jax.numpy as jnp + +from confirm.lewislib import lewis + + +default_params = { + "n_arms": 3, + "n_stage_1": 3, + "n_stage_2": 3, + "n_stage_1_interims": 1, + "n_stage_1_add_per_interim": 10, + "n_stage_2_add_per_interim": 4, + "stage_1_futility_threshold": 0.1, + "stage_2_futility_threshold": 0.1, + "stage_1_efficacy_threshold": 0.1, + "stage_2_efficacy_threshold": 0.9, + "inter_stage_futility_threshold": 0.8, + "posterior_difference_threshold": 0.05, + "rejection_threshold": 0.05, +} + + +def test_posterior_difference_permute(): + lewis_obj = lewis.Lewis45(**default_params) + n = default_params["n_stage_1"] + + data_1 = jnp.array( + [ + [0, n], + [1, n], + [2, n], + ] + ) + out_1 = lewis_obj.posterior_difference(data_1) + + data_2 = jnp.array( + [ + [0, n], + [2, n], + [1, n], + ] + ) + out_2 = lewis_obj.posterior_difference(data_2) + + permute = jnp.array([1, 0]) + + assert jnp.allclose(out_1, out_2[permute]) + + +def test_pr_best_permute(): + lewis_obj = lewis.Lewis45(**default_params) + key = jax.random.PRNGKey(10) + + thetas_1 = jax.random.normal(key=key, shape=(100, 3)) + out_1 = lewis_obj.pr_best(thetas_1) + + permute = jnp.array([1, 0, 2]) + + thetas_2 = thetas_1[:, permute] + out_2 = lewis_obj.pr_best(thetas_2) + + assert jnp.allclose(out_1, out_2[permute]) + + +def test_pps_permute(): + lewis_obj = lewis.Lewis45(**default_params) + lewis_obj.pd_table = lewis_obj.posterior_difference_table__(batch_size=int(2**16)) + + n = default_params["n_stage_1"] + key = jax.random.PRNGKey(10) + + data_1 = jnp.array([[0, n], [1, n], [2, n]]) + thetas_1 = jax.random.normal(key=key, shape=(100, 3)) + _, key = jax.random.split(key) + unifs_1 = jax.random.uniform(key=key, shape=(100, 10, 3)) + out_1 = lewis_obj.pps(data_1, thetas_1, unifs_1) + + permute = jnp.array([0, 2, 1]) + + data_2 = data_1[permute] + thetas_2 = thetas_1[:, permute] + unifs_2 = unifs_1[..., permute] + out_2 = lewis_obj.pps(data_2, thetas_2, unifs_2) + + assert jnp.allclose(out_1, out_2[permute[1:] - 1]) diff --git a/confirm/tests/lewis/test_posterior_difference.py b/confirm/tests/lewis/test_posterior_difference.py new file mode 100644 index 00000000..7936d02d --- /dev/null +++ b/confirm/tests/lewis/test_posterior_difference.py @@ -0,0 +1,33 @@ +import jax.numpy as jnp + +from confirm.lewislib import lewis + + +default_params = { + "n_arms": 3, + "n_stage_1": 3, + "n_stage_2": 3, + "n_stage_1_interims": 1, + "n_stage_1_add_per_interim": 10, + "n_stage_2_add_per_interim": 4, + "stage_1_futility_threshold": 0.1, + "stage_2_futility_threshold": 0.1, + "stage_1_efficacy_threshold": 0.1, + "stage_2_efficacy_threshold": 0.9, + "inter_stage_futility_threshold": 0.8, + "posterior_difference_threshold": 0.05, + "rejection_threshold": 0.05, +} + + +def test_get_posterior_difference(): + lewis_obj = lewis.Lewis45(**default_params) + lewis_obj.pd_table = lewis_obj.posterior_difference_table__(batch_size=int(2**16)) + n = lewis_obj.n_configs_pd[2] + y = jnp.array([5, 1, 2]) + data = jnp.stack((y, n), axis=-1) + out_1 = lewis_obj.get_posterior_difference__(data) + permute = jnp.array([0, 2, 1]) + data_2 = data[permute] + out_2 = lewis_obj.get_posterior_difference__(data_2) + assert jnp.array_equal(out_1, out_2[permute[1:] - 1]) diff --git a/confirm/tests/lewis/test_simulation.py b/confirm/tests/lewis/test_simulation.py new file mode 100644 index 00000000..a5b541bb --- /dev/null +++ b/confirm/tests/lewis/test_simulation.py @@ -0,0 +1,77 @@ +import jax +import jax.numpy as jnp + +from confirm.lewislib import lewis + + +default_params = { + "n_arms": 3, + "n_stage_1": 1, + "n_stage_2": 1, + "n_stage_1_interims": 2, + "n_stage_1_add_per_interim": 3, + "n_stage_2_add_per_interim": 1, + "stage_1_futility_threshold": 0.2, + "stage_2_futility_threshold": 0.1, + "stage_1_efficacy_threshold": 0.9, + "stage_2_efficacy_threshold": 0.9, + "inter_stage_futility_threshold": 0.8, + "posterior_difference_threshold": 0.05, + "rejection_threshold": 0.05, + "batch_size": 2**16, + "key": jax.random.PRNGKey(1), + "n_pr_sims": 100, + "n_sig2_sims": 20, + "cache_tables": True, +} + +key = jax.random.PRNGKey(0) +lewis_obj = lewis.Lewis45(**default_params) +unifs = jax.random.uniform(key=key, shape=lewis_obj.unifs_shape()) +p = jnp.array([0.25, 0.5, 0.75]) +berns = unifs < p[None] +berns_order = jnp.arange(0, berns.shape[0]) + + +def test_stage_1(): + # actual + ( + early_exit_futility, + data, + non_dropped_idx, + pps, + berns_start, + ) = lewis_obj.stage_1(berns, berns_order) + + # expected + early_exit_futility_expected = False + data_expected = jnp.array([[0, 3], [0, 1], [2, 3]], dtype=int) + non_dropped_idx_expected = jnp.array([False, True]) + _, pps_expected = lewis_obj.get_pr_best_pps_1__(data_expected) + berns_start_expected = 3 + + # test + assert jnp.array_equal(early_exit_futility, early_exit_futility_expected) + assert jnp.array_equal(data, data_expected) + assert jnp.array_equal(non_dropped_idx, non_dropped_idx_expected) + assert jnp.array_equal(pps, pps_expected) + assert jnp.array_equal(berns_start, berns_start_expected) + + +def test_stage_2(): + # expected stage 1 + data = jnp.array([[1, 3], [0, 1], [2, 3]], dtype=int) + best_arm = 2 + berns_start = 3 + + # actual stage 2 + rej, _ = lewis_obj.stage_2(data, best_arm, berns, berns_order, berns_start) + + # test + assert jnp.array_equal(rej, False) + + +def test_inter_stage(): + null_truths = jnp.zeros(default_params["n_arms"] - 1, dtype=bool) + rej, _ = lewis_obj.simulate(p, null_truths, unifs, berns_order) + assert jnp.array_equal(rej, False) diff --git a/confirm/tests/test_interp.py b/confirm/tests/test_interp.py new file mode 100644 index 00000000..7d5ad48e --- /dev/null +++ b/confirm/tests/test_interp.py @@ -0,0 +1,29 @@ +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import scipy.interpolate + +from confirm.outlaw.interp import interpn + + +def test_interpn(): + grid = jnp.array([[0, 1], [0, 1]]) + values = jnp.array([[0, 1], [2, 3]]) + xi = jnp.array([[0.5, 0.5], [1.0, 0.0]]) + result = jax.vmap(interpn, in_axes=(None, None, 0))(grid, values, xi) + np.testing.assert_allclose(result, [1.5, 2.0]) + + +@pytest.mark.parametrize("dim", [1, 3]) +def test_against_scipy_multi_value(dim): + for i in range(3): + np.random.seed(10) + grid = [np.sort(np.random.uniform(size=10)) for _ in range(2)] + values = jnp.array(np.random.uniform(size=(10, 10, dim)).squeeze()) + xi = np.random.uniform(size=(10, 2)) + result = jax.vmap(interpn, in_axes=(None, None, 0))(grid, values, xi) + scipy_result = scipy.interpolate.interpn( + grid, values, xi, method="linear", bounds_error=False, fill_value=None + ) + np.testing.assert_allclose(result, scipy_result) diff --git a/imprint/.vscode/build.sh b/imprint/.vscode/build.sh index dbb89470..43dadd87 100755 --- a/imprint/.vscode/build.sh +++ b/imprint/.vscode/build.sh @@ -1,5 +1,5 @@ #!/bin/bash eval "$(conda shell.bash hook)" conda activate imprint -bazel build //python:pyimprint/core.so -ln -sf ./bazel-bin/python/pyimprint/core.so python/pyimprint/core.so \ No newline at end of file +bazel build -c opt --config gcc //python:pyimprint/core.so +cp -f ./bazel-bin/python/pyimprint/core.so python/pyimprint/ diff --git a/imprint/frontend/.gitignore b/imprint/frontend/.gitignore index 95004389..d4bfd63c 100644 --- a/imprint/frontend/.gitignore +++ b/imprint/frontend/.gitignore @@ -44,3 +44,6 @@ yarn-error.log* npm-debug.log* yarn-debug.log* yarn-error.log* + +# local folders +my-app/ diff --git a/imprint/frontend/tsconfig.json b/imprint/frontend/tsconfig.json index a273b0cf..f199ca8f 100644 --- a/imprint/frontend/tsconfig.json +++ b/imprint/frontend/tsconfig.json @@ -18,7 +18,7 @@ "resolveJsonModule": true, "isolatedModules": true, "noEmit": true, - "jsx": "react-jsx" + "jsx": "preserve" }, "include": [ "src" diff --git a/install.sh b/install.sh index 9bc63458..04508cce 100755 --- a/install.sh +++ b/install.sh @@ -27,4 +27,4 @@ fi if [[ -n "$CONFIRM_IMPRINT_SSH" ]]; then git remote add -f imprint git@github.com:Confirm-Solutions/imprint.git -fi \ No newline at end of file +fi diff --git a/research/berry/berry_part1.ipynb b/research/berry/berry_part1.ipynb index 4cc613e4..7ae8629a 100644 --- a/research/berry/berry_part1.ipynb +++ b/research/berry/berry_part1.ipynb @@ -1243,14 +1243,11 @@ } ], "metadata": { - "interpreter": { - "hash": "a9637099bd81b2ef0895c64d539356b45819bc945d59d426757b1f51ae370d50" - }, "jupytext": { "formats": "ipynb,md" }, "kernelspec": { - "display_name": "Python 3.10.2 ('imprint')", + "display_name": "Python 3.10.5 ('confirm')", "language": "python", "name": "python3" }, @@ -1264,7 +1261,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.10.5" + }, + "vscode": { + "interpreter": { + "hash": "b4c6ec5b2d6c7b38df115d547b82cd53ca25eea58d87299956d35a9dc79f19f1" + } } }, "nbformat": 4, diff --git a/research/berry/berry_part1.md b/research/berry/berry_part1.md index ac0ce915..b70a312c 100644 --- a/research/berry/berry_part1.md +++ b/research/berry/berry_part1.md @@ -8,7 +8,7 @@ jupyter: format_version: '1.3' jupytext_version: 1.13.8 kernelspec: - display_name: Python 3.10.2 ('imprint') + display_name: Python 3.10.5 ('confirm') language: python name: python3 --- diff --git a/research/lei/.gitignore b/research/lei/.gitignore new file mode 100644 index 00000000..afed0735 --- /dev/null +++ b/research/lei/.gitignore @@ -0,0 +1 @@ +*.csv diff --git a/research/lei/analyze/analyze.ipynb b/research/lei/analyze/analyze.ipynb new file mode 100644 index 00000000..b725efb8 --- /dev/null +++ b/research/lei/analyze/analyze.ipynb @@ -0,0 +1,321 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Analyze Upper Bound of Type I Error for Lei Example" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import jax\n", + "import os\n", + "import numpy as np\n", + "from confirm.mini_imprint import grid\n", + "from confirm.lewislib import grid as lewgrid\n", + "from confirm.lewislib import lewis\n", + "from confirm.mini_imprint import binomial" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Configuration used during simulation\n", + "params = {\n", + " \"n_arms\" : 4,\n", + " \"n_stage_1\" : 50,\n", + " \"n_stage_2\" : 100,\n", + " \"n_stage_1_interims\" : 2,\n", + " \"n_stage_1_add_per_interim\" : 100,\n", + " \"n_stage_2_add_per_interim\" : 100,\n", + " \"stage_1_futility_threshold\" : 0.15,\n", + " \"stage_1_efficacy_threshold\" : 0.7,\n", + " \"stage_2_futility_threshold\" : 0.2,\n", + " \"stage_2_efficacy_threshold\" : 0.95,\n", + " \"inter_stage_futility_threshold\" : 0.6,\n", + " \"posterior_difference_threshold\" : 0,\n", + " \"rejection_threshold\" : 0.05,\n", + " \"key\" : jax.random.PRNGKey(0),\n", + " \"n_pr_sims\" : 100,\n", + " \"n_sig2_sims\" : 20,\n", + " \"batch_size\" : int(2**20),\n", + " \"cache_tables\" : False,\n", + "}\n", + "size = 52\n", + "n_sim_batches = 500\n", + "sim_batch_size = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# construct Lei object\n", + "lei_obj = lewis.Lewis45(**params)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# construct the same grid used during simulation\n", + "n_arms = params['n_arms']\n", + "lower = np.full(n_arms, -1)\n", + "upper = np.full(n_arms, 1)\n", + "thetas, radii = lewgrid.make_cartesian_grid_range(\n", + " size=size,\n", + " lower=lower,\n", + " upper=upper,\n", + ")\n", + "ns = np.concatenate(\n", + " [np.ones(n_arms-1)[:, None], -np.eye(n_arms-1)],\n", + " axis=-1,\n", + ")\n", + "null_hypos = [\n", + " grid.HyperPlane(n, 0)\n", + " for n in ns\n", + "]\n", + "gr = grid.build_grid(\n", + " thetas=thetas,\n", + " radii=radii,\n", + " null_hypos=null_hypos,\n", + ")\n", + "gr = grid.prune(gr)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# construct tile informations used during simulation\n", + "theta_tiles = gr.thetas[gr.grid_pt_idx]\n", + "p_tiles = jax.scipy.special.expit(theta_tiles)\n", + "tile_radii = gr.radii[gr.grid_pt_idx]\n", + "null_truths = gr.null_truth.astype(bool)\n", + "sim_size = 2 * n_sim_batches * sim_batch_size # 2 instances parallelized\n", + "sim_sizes = np.full(gr.n_tiles, sim_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# get type I sum and score\n", + "cwd = '.'\n", + "data_dir = os.path.join(cwd, '../data')\n", + "output_dir = os.path.join(data_dir, 'output_1')\n", + "typeI_sum = np.loadtxt(os.path.join(output_dir, 'typeI_sum.csv'), delimiter=',')\n", + "typeI_score = np.loadtxt(os.path.join(output_dir, 'typeI_score.csv'), delimiter=',')\n", + "output_dir = os.path.join(data_dir, 'output_2')\n", + "typeI_sum += np.loadtxt(os.path.join(output_dir, 'typeI_sum.csv'), delimiter=',')\n", + "typeI_score += np.loadtxt(os.path.join(output_dir, 'typeI_score.csv'), delimiter=',')" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "delta = 0.025\n", + "n_arm_samples = int(lei_obj.unifs_shape()[0])\n", + "tile_corners = gr.vertices" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "# construct Holder upper bound\n", + "d0, d0u = binomial.zero_order_bound(\n", + " typeI_sum=typeI_sum, \n", + " sim_sizes=sim_sizes, \n", + " delta=delta, \n", + " delta_prop_0to1=1,\n", + ")\n", + "typeI_bound = d0 + d0u\n", + "\n", + "total_holder = binomial.holder_odi_bound(\n", + " typeI_bound=typeI_bound, \n", + " theta_tiles=theta_tiles,\n", + " tile_corners=tile_corners,\n", + " n_arm_samples=n_arm_samples, \n", + " holderq=16,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# construct classical upper bound\n", + "total, d0, d0u, d1w, d1uw, d2uw = binomial.upper_bound(\n", + " theta_tiles,\n", + " tile_radii,\n", + " gr.vertices,\n", + " sim_sizes,\n", + " n_arm_samples,\n", + " typeI_sum,\n", + " typeI_score,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "# prepare bound components\n", + "\n", + "# classical\n", + "bound_components = np.array([\n", + " d0,\n", + " d0u,\n", + " d1w,\n", + " d1uw,\n", + " d2uw,\n", + " total,\n", + "]).T\n", + "\n", + "# holder\n", + "dummy = np.zeros_like(d0)\n", + "bound_components_holder = np.array([\n", + " d0,\n", + " d0u,\n", + " dummy,\n", + " dummy,\n", + " dummy,\n", + " total_holder,\n", + "]).T" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([-0.98076923, -0.94230769, -0.90384615, -0.86538462, -0.82692308,\n", + " -0.78846154, -0.75 , -0.71153846, -0.67307692, -0.63461538,\n", + " -0.59615385, -0.55769231, -0.51923077, -0.48076923, -0.44230769,\n", + " -0.40384615, -0.36538462, -0.32692308, -0.28846154, -0.25 ,\n", + " -0.21153846, -0.17307692, -0.13461538, -0.09615385, -0.05769231,\n", + " -0.01923077, 0.01923077, 0.05769231, 0.09615385, 0.13461538,\n", + " 0.17307692, 0.21153846, 0.25 , 0.28846154, 0.32692308,\n", + " 0.36538462, 0.40384615, 0.44230769, 0.48076923, 0.51923077,\n", + " 0.55769231, 0.59615385, 0.63461538, 0.67307692, 0.71153846,\n", + " 0.75 , 0.78846154, 0.82692308, 0.86538462, 0.90384615,\n", + " 0.94230769, 0.98076923]),\n", + " array([-0.98076923, -0.94230769, -0.90384615, -0.86538462, -0.82692308,\n", + " -0.78846154, -0.75 , -0.71153846, -0.67307692, -0.63461538,\n", + " -0.59615385, -0.55769231, -0.51923077, -0.48076923, -0.44230769,\n", + " -0.40384615, -0.36538462, -0.32692308, -0.28846154, -0.25 ,\n", + " -0.21153846, -0.17307692, -0.13461538, -0.09615385, -0.05769231,\n", + " -0.01923077, 0.01923077, 0.05769231, 0.09615385, 0.13461538,\n", + " 0.17307692, 0.21153846, 0.25 , 0.28846154, 0.32692308,\n", + " 0.36538462, 0.40384615, 0.44230769, 0.48076923, 0.51923077,\n", + " 0.55769231, 0.59615385, 0.63461538, 0.67307692, 0.71153846,\n", + " 0.75 , 0.78846154, 0.82692308, 0.86538462, 0.90384615,\n", + " 0.94230769, 0.98076923]))" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t2_uniques = np.unique(theta_tiles[:, 2])\n", + "t3_uniques = np.unique(theta_tiles[:, 3])\n", + "t2_uniques, t3_uniques" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "# slice and save P, B\n", + "t2 = t2_uniques[25]\n", + "t3 = t3_uniques[20]\n", + "selection = (theta_tiles[:, 2] == t2) & (theta_tiles[:, 3] == t3)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "bound_dir = os.path.join(data_dir, 'bound')\n", + "if not os.path.exists(bound_dir):\n", + " os.makedirs(bound_dir)\n", + "\n", + "np.savetxt(f'{bound_dir}/P_lei.csv', theta_tiles[selection, :].T, fmt=\"%s\", delimiter=\",\")\n", + "np.savetxt(f'{bound_dir}/B_lei.csv', bound_components[selection, :], fmt=\"%s\", delimiter=\",\")\n", + "np.savetxt(f'{bound_dir}/B_lei_holder.csv', bound_components_holder[selection, :], fmt=\"%s\", delimiter=\",\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.5 ('confirm')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "d8e1ca1b3fede25e3995e2b26ea544fa1b75b9a17984e6284a43c1dc286640dd" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/research/lei/analyze/analyze.md b/research/lei/analyze/analyze.md new file mode 100644 index 00000000..7e1a307b --- /dev/null +++ b/research/lei/analyze/analyze.md @@ -0,0 +1,196 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3.10.5 ('confirm') + language: python + name: python3 +--- + +# Analyze Upper Bound of Type I Error for Lei Example + +```python +%load_ext autoreload +%autoreload 2 +``` + +```python +import jax +import os +import numpy as np +from confirm.mini_imprint import grid +from confirm.lewislib import grid as lewgrid +from confirm.lewislib import lewis +from confirm.mini_imprint import binomial +``` + +```python +# Configuration used during simulation +params = { + "n_arms" : 4, + "n_stage_1" : 50, + "n_stage_2" : 100, + "n_stage_1_interims" : 2, + "n_stage_1_add_per_interim" : 100, + "n_stage_2_add_per_interim" : 100, + "stage_1_futility_threshold" : 0.15, + "stage_1_efficacy_threshold" : 0.7, + "stage_2_futility_threshold" : 0.2, + "stage_2_efficacy_threshold" : 0.95, + "inter_stage_futility_threshold" : 0.6, + "posterior_difference_threshold" : 0, + "rejection_threshold" : 0.05, + "key" : jax.random.PRNGKey(0), + "n_pr_sims" : 100, + "n_sig2_sims" : 20, + "batch_size" : int(2**20), + "cache_tables" : False, +} +size = 52 +n_sim_batches = 500 +sim_batch_size = 100 +``` + +```python +# construct Lei object +lei_obj = lewis.Lewis45(**params) +``` + +```python +# construct the same grid used during simulation +n_arms = params['n_arms'] +lower = np.full(n_arms, -1) +upper = np.full(n_arms, 1) +thetas, radii = lewgrid.make_cartesian_grid_range( + size=size, + lower=lower, + upper=upper, +) +ns = np.concatenate( + [np.ones(n_arms-1)[:, None], -np.eye(n_arms-1)], + axis=-1, +) +null_hypos = [ + grid.HyperPlane(n, 0) + for n in ns +] +gr = grid.build_grid( + thetas=thetas, + radii=radii, + null_hypos=null_hypos, +) +gr = grid.prune(gr) +``` + +```python +# construct tile informations used during simulation +theta_tiles = gr.thetas[gr.grid_pt_idx] +p_tiles = jax.scipy.special.expit(theta_tiles) +tile_radii = gr.radii[gr.grid_pt_idx] +null_truths = gr.null_truth.astype(bool) +sim_size = 2 * n_sim_batches * sim_batch_size # 2 instances parallelized +sim_sizes = np.full(gr.n_tiles, sim_size) +``` + +```python +# get type I sum and score +cwd = '.' +data_dir = os.path.join(cwd, '../data') +output_dir = os.path.join(data_dir, 'output_1') +typeI_sum = np.loadtxt(os.path.join(output_dir, 'typeI_sum.csv'), delimiter=',') +typeI_score = np.loadtxt(os.path.join(output_dir, 'typeI_score.csv'), delimiter=',') +output_dir = os.path.join(data_dir, 'output_2') +typeI_sum += np.loadtxt(os.path.join(output_dir, 'typeI_sum.csv'), delimiter=',') +typeI_score += np.loadtxt(os.path.join(output_dir, 'typeI_score.csv'), delimiter=',') +``` + +```python +delta = 0.025 +n_arm_samples = int(lei_obj.unifs_shape()[0]) +tile_corners = gr.vertices +``` + +```python +# construct Holder upper bound +d0, d0u = binomial.zero_order_bound( + typeI_sum=typeI_sum, + sim_sizes=sim_sizes, + delta=delta, + delta_prop_0to1=1, +) +typeI_bound = d0 + d0u + +total_holder = binomial.holder_odi_bound( + typeI_bound=typeI_bound, + theta_tiles=theta_tiles, + tile_corners=tile_corners, + n_arm_samples=n_arm_samples, + holderq=16, +) +``` + +```python +# construct classical upper bound +total, d0, d0u, d1w, d1uw, d2uw = binomial.upper_bound( + theta_tiles, + tile_radii, + gr.vertices, + sim_sizes, + n_arm_samples, + typeI_sum, + typeI_score, +) +``` + +```python +# prepare bound components + +# classical +bound_components = np.array([ + d0, + d0u, + d1w, + d1uw, + d2uw, + total, +]).T + +# holder +dummy = np.zeros_like(d0) +bound_components_holder = np.array([ + d0, + d0u, + dummy, + dummy, + dummy, + total_holder, +]).T +``` + +```python +t2_uniques = np.unique(theta_tiles[:, 2]) +t3_uniques = np.unique(theta_tiles[:, 3]) +t2_uniques, t3_uniques +``` + +```python +# slice and save P, B +t2 = t2_uniques[25] +t3 = t3_uniques[20] +selection = (theta_tiles[:, 2] == t2) & (theta_tiles[:, 3] == t3) +``` + +```python +bound_dir = os.path.join(data_dir, 'bound') +if not os.path.exists(bound_dir): + os.makedirs(bound_dir) + +np.savetxt(f'{bound_dir}/P_lei.csv', theta_tiles[selection, :].T, fmt="%s", delimiter=",") +np.savetxt(f'{bound_dir}/B_lei.csv', bound_components[selection, :], fmt="%s", delimiter=",") +np.savetxt(f'{bound_dir}/B_lei_holder.csv', bound_components_holder[selection, :], fmt="%s", delimiter=",") +``` diff --git a/research/lei/analyze/download_data.sh b/research/lei/analyze/download_data.sh new file mode 100755 index 00000000..62caf74e --- /dev/null +++ b/research/lei/analyze/download_data.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# directory where current shell script resides +PROJECTDIR=$(dirname "$BASH_SOURCE") +cd "$PROJECTDIR" +cd .. +mkdir -p data +cd data +aws s3 cp s3://imprint-dump/output_lei4d/ output_1/ --recursive +aws s3 cp s3://imprint-dump/output_lei4d2/ output_2/ --recursive \ No newline at end of file diff --git a/research/lei/lei.ipynb b/research/lei/lei.ipynb new file mode 100644 index 00000000..3bb3f140 --- /dev/null +++ b/research/lei/lei.ipynb @@ -0,0 +1,1177 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "import confirm.outlaw\n", + "import confirm.outlaw.berry as berry\n", + "import confirm.outlaw.quad as quad\n", + "import numpy as np\n", + "import jax.numpy as jnp\n", + "import jax\n", + "import time\n", + "import confirm.outlaw.inla as inla\n", + "import matplotlib.pyplot as plt\n", + "import numpyro.distributions as dist\n", + "from functools import partial\n", + "from itertools import combinations\n", + "\n", + "from confirm.lewislib import lewis\n", + "from confirm.lewislib import batch\n", + "from confirm.mini_imprint import grid\n", + "from confirm.lewislib import grid as lewgrid\n", + "from confirm.mini_imprint import binomial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lei Example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following description is a clinical trial design using a Bayesian model with early-stopping rules for futility or efficacy of a drug.\n", + "This design was explicitly requested to be studied by an FDA member (Lei) in the CID team." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> The following is a randomized, double-blind, placebo-controlled two-stage adaptive design intended to identify an optimal treatment regimen \n", + "> from three possible regimens (for example, different dosages or different combinations of agents) and \n", + "> to assess the efficacy of that regimen with respect to a primary binary response endpoint measured at month 6.\n", + "> \n", + "> In Stage 1, one of four experimental regimens will be selected, or the trial will stop for futility. \n", + "> In this stage, a minimum of 200 and a maximum of 400 will be randomized 1:1:1:1 to one of the three experimental arms or one placebo arm. \n", + "> Interim analyses will be conducted after 200, 300 and 400 subjects have been enrolled to select the best experimental regimen and to potentially stop \n", + "> the trial for futility. \n", + "> If an experimental regimen is dropped for futility at an interim analysis, \n", + "> the next 100 subjects to be randomized will be allocated equally among the remaining arms in the study. \n", + "> At each of these three analysis time points (N = 200, 300, 400), \n", + "> the probabilities of being the best regimen (PrBest) and predictive probability of success (PPS) \n", + "> are calculated for each experimental regimen using a Bayesian approach, \n", + "> and the trial will either stop for futility, \n", + "> continue to the next interim analysis, \n", + "> or proceed to Stage 2 depending on the results of these PrBest and PPS calculations.\n", + "> \n", + "> In Stage 2, a minimum of 200 and a maximum of 400 additional subjects will be randomized 1:1 to the chosen regimen or placebo. \n", + "> The two groups (pooling both Stage 1 and Stage 2 subjects) will be compared for efficacy and futility assessment at an interim analysis \n", + "> after 200 subjects have been enrolled in Stage 2, \n", + "> and for efficacy at a final analysis after 400 subjects have been enrolled in Stage 2 and fully followed-up for response. \n", + "> The study may be stopped for futility or efficacy based on PPS at the interim analysis. \n", + "> If the study continues to the final analysis, \n", + "> the posterior distribution of the difference in response rates between placebo and the chosen experimental arm \n", + "> will be evaluated against a pre-specified decision criterion.\n", + "> \n", + "> - Scenario 1: interim analyses are based on available data on the primary endpoint (measured at month 6)\n", + "> - Scenario 2: interim analyses are based on available data on a secondary endpoint (measured at month 3) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook breaks down and discusses the components of the trial." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The notation is as follows:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- $y \\in \\mathbb{N}^d$: Binomial responses.\n", + "- $p \\in [0,1]^d$: probability parameter to the Binomial distribution.\n", + "- $n \\in \\mathbb{N}^d$: size parameter to the Binomial distribution.\n", + "- $q \\in [0,1]^d$: base probability value to offset $p$.\n", + "- $\\theta \\in \\R^d$: logit parameter that determines $p$.\n", + "- $\\mu \\in \\mathbb{R}$: shared mean parameter among $\\theta_i$.\n", + "- $\\sigma^2 \\in \\mathbb{R}_+$: shared variance parameter among $\\theta_i$.\n", + "- $\\mu_0, \\sigma_0^2, \\alpha_0, \\beta_0 \\in \\mathbb{R}$: hyper-parameters." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Bayesian model is described below:\n", + "\\begin{align*}\n", + "y_i | p_i &\\sim \\mathrm{Binom}(n_i, p_i) \\quad i = 1,\\ldots, d \\\\\n", + "p_i &= {\\sf expit}(\\theta_i + \\mathrm{logit}(q_i) ) \\quad i = 1,\\ldots, d \\\\\n", + "\\theta | \\mu, \\sigma^2 &\\sim \\mathcal{N}(\\mu \\mathbb{1}, \\sigma^2 I) \\\\\n", + "\\mu &\\sim \\mathcal{N}(\\mu_0, \\sigma_0^2) \\\\\n", + "\\sigma^2 &\\sim \\Gamma^{-1}(\\alpha_0, \\beta_0) \\\\\n", + "\\end{align*}\n", + "\n", + "We note in passing that the model can be collapsed along $\\mu$ to get:\n", + "\\begin{align*}\n", + "y_i | p_i &\\sim \\mathrm{Binom}(n_i, p_i) \\quad i = 1,\\ldots, d \\\\\n", + "p_i &= {\\sf expit}(\\theta_i + \\mathrm{logit}(q_i) ) \\quad i = 1,\\ldots, d \\\\\n", + "\\theta | \\sigma^2 &\\sim \\mathcal{N}(\\mu_0 \\mathbb{1}, \\sigma^2 I + \\sigma_0^2 \\mathbb{1} \\mathbb{1}^\\top) \\\\\n", + "\\sigma^2 &\\sim \\Gamma^{-1}(\\alpha_0, \\beta_0) \\\\\n", + "\\end{align*}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Probability of Best Arm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first quantity of interest is probability of best (treatment) arm.\n", + "Concretely, letting $i = 1$ denote the control arm, we wish to compute for each $1 < i \\leq d$:\n", + "\\begin{align*}\n", + "\\mathbb{P}(p_i > \\max\\limits_{j \\neq i} p_j | y, n)\n", + "&=\n", + "\\int \\mathbb{P}(p_i > \\max\\limits_{j \\neq i} p_j | y, n, \\sigma^2) p(\\sigma^2 | y, n) \\, d\\sigma^2\n", + "\\\\&=\n", + "\\int \\mathbb{P}(\\theta_i + c_i > \\max\\limits_{j \\neq i} (\\theta_j + c_j) | y, n, \\sigma^2) p(\\sigma^2 | y, n) \\, d\\sigma^2\n", + "\\end{align*}\n", + "where $c = \\mathrm{logit}(q)$.\n", + "We can approximate this quantity by estimating the two integrand terms separately. \n", + "By approximating $\\theta_i | y, n$ as normal, the first integrand term can be estimated by Monte Carlo.\n", + "The second term can be estimated by INLA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def pr_normal_best(mean, cov, key, n_sims):\n", + " '''\n", + " Estimates P[X_i > max_{j != i} X_j] where X ~ N(mean, cov) via sampling.\n", + " '''\n", + " out_shape = (n_sims, *mean.shape[:-1])\n", + " sims = jax.random.multivariate_normal(key, mean, cov, shape=out_shape)\n", + " order = jnp.arange(1, mean.shape[-1])\n", + " compute_pr_best_all = jax.vmap(lambda i: jnp.mean(jnp.argmax(sims, axis=-1) == i, axis=0))\n", + " return compute_pr_best_all(order)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "d = 4\n", + "mean = jnp.array([2, 2, 2, 5])\n", + "cov = jnp.eye(d)\n", + "key = jax.random.PRNGKey(0)\n", + "n_sims = 100000\n", + "jax.jit(pr_normal_best, static_argnums=(3,))(mean, cov, key, n_sims=n_sims)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we perform INLA to estimate $p(\\sigma^2 | y, n)$ on a grid of values for $\\sigma^2$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sig2_rule = quad.log_gauss_rule(15, 1e-6, 1e3)\n", + "sig2_rule_ops = berry.optimized(sig2_rule.pts, n_arms=4).config(\n", + " opt_tol=1e-3\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def posterior_sigma_sq(data, sig2_rule, sig2_rule_ops):\n", + " n_arms, _ = data.shape\n", + " sig2 = sig2_rule.pts\n", + " n_sig2 = sig2.shape[0]\n", + " p_pinned = dict(sig2=sig2, theta=None)\n", + "\n", + " f = sig2_rule_ops.laplace_logpost\n", + " logpost, x_max, hess, iters = f(\n", + " np.zeros((n_sig2, n_arms)), p_pinned, data\n", + " )\n", + " post = inla.exp_and_normalize(\n", + " logpost, sig2_rule.wts, axis=-1)\n", + "\n", + " return post, x_max, hess, iters " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dtype = jnp.float64\n", + "N = 1\n", + "data = berry.figure2_data(N).astype(dtype)[0]\n", + "n_arms, _ = data.shape\n", + "posterior_sigma_sq_jit = jax.jit(lambda data: posterior_sigma_sq(data, sig2_rule, sig2_rule_ops))\n", + "post, _, hess, _ = posterior_sigma_sq_jit(data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Putting the two pieces together, we have the following function to compute the probability of best treatment arm." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def pr_best(data, sig2_rule, sig2_rule_ops, key, n_sims):\n", + " n_arms, _ = data.shape\n", + " post, x_max, hess, _ = posterior_sigma_sq(data, sig2_rule, sig2_rule_ops) \n", + " mean = x_max\n", + " hess_fn = jax.vmap(lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1]))\n", + " prec = -hess_fn(hess) # (n_sigs, n_arms, n_arms)\n", + " cov = jnp.linalg.inv(prec)\n", + " pr_normal_best_out = pr_normal_best(mean, cov, key=key, n_sims=n_sims)\n", + " return jnp.matmul(pr_normal_best_out, post * sig2_rule.wts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n_sims = 13\n", + "out = pr_best(data, sig2_rule, sig2_rule_ops, key, n_sims)\n", + "out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Phase III Final Analysis\n", + "\n", + "\\begin{align*}\n", + "\\mathbb{P}(\\theta_i - \\theta_0 < t | y, n) < 0.1\n", + "\\end{align*}\n", + "\n", + "\\begin{align*}\n", + "\\mathbb{P}(\\theta_i - \\theta_0 < t | y, n)\n", + "&=\n", + "\\mathbb{P}(q_1^\\top \\theta < t | y,n)\n", + "=\n", + "\\int \\mathbb{P}(q_1^\\top \\theta < t | y, n, \\sigma^2) p(\\sigma^2 | y, n) \\, d\\sigma^2\n", + "\\\\&=\n", + "\\int \\mathbb{P}(q_1^\\top \\theta < t | y, n, \\sigma^2) p(\\sigma^2 | y, n) \\, d\\sigma^2\n", + "\\\\\n", + "q_1^\\top \\theta | y, n, \\sigma^2 &\\sim \\mathcal{N}(q_1^\\top \\theta^*, -q_1^\\top (H\\log p(\\theta^*, y, \\sigma^2))^{-1} q_1)\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "posterior_difference_threshold = 0.2\n", + "rejection_threshold = 0.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def posterior_difference(data, arm, sig2_rule, sig2_rule_ops, thresh):\n", + " n_arms, _ = data.shape\n", + " post, x_max, hess, _ = posterior_sigma_sq(data, sig2_rule, sig2_rule_ops)\n", + " hess_fn = jax.vmap(lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1]))\n", + " prec = -hess_fn(hess) # (n_sigs, n_arms, n_arms)\n", + " order = jnp.arange(0, n_arms)\n", + " q1 = jnp.where(order == 0, -1, 0)\n", + " q1 = jnp.where(order == arm, 1, q1)\n", + " loc = x_max @ q1\n", + " scale = jnp.linalg.solve(prec, q1[None,:]) @ q1\n", + " normal_term = jax.scipy.stats.norm.cdf(thresh, loc=loc, scale=scale)\n", + " post_weighted = sig2_rule.wts * post\n", + " out = normal_term @ post_weighted\n", + " return out\n", + "\n", + "posterior_difference(data, 1, sig2_rule, sig2_rule_ops, posterior_difference_threshold)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Posterior Probability of Success\n", + "\n", + "The next quantity we need to compute is the posterior probability of success (PPS).\n", + "For convenience of implementation, we will take this to mean the following:\n", + "let $y, n$ denote the currently observed data\n", + "and $A_i = \\{ \\text{Phase III rejects using treatment arm i} \\}$.\n", + "Then, we wish to compute\n", + "\\begin{align*}\n", + "\\mathbb{P}(A_i | y, n)\n", + "\\end{align*}\n", + "Expanding the quantity,\n", + "\\begin{align*}\n", + "\\mathbb{P}(A_i | y, n) &=\n", + "\\int \\mathbb{P}(A_i | y, n, \\theta_i, \\theta_0) p(\\theta_0, \\theta_i | y, n) \\, d\\theta_i d\\theta_0 \\\\&=\n", + "\\int \\mathbb{P}(A_i | y, n, \\theta_i, \\theta_0) p(\\theta_0, \\theta_i | y, n) \\, d\\theta_i d\\theta_0\n", + "\\end{align*}\n", + "\n", + "Once we have an estimate for $p(\\theta_0, \\theta_i | y, n)$, \n", + "we can use 2-D quadrature to numerically integrate the integrand.\n", + "Similar to computing the probability of best arm,\n", + "\\begin{align*}\n", + "p(\\theta_0, \\theta_i | y, n)\n", + "&=\n", + "\\int p(\\theta_0, \\theta_i | y, n, \\sigma^2) p(\\sigma^2 | y, n) \\, d\\sigma^2\n", + "\\end{align*}\n", + "We will use the Gaussian approximation for $p(\\theta_0, \\theta_i | y, n, \\sigma^2)$\n", + "and use INLA to estimate $p(\\sigma^2 | y, n)$.\n", + "\n", + "\\begin{align*}\n", + "p(\\theta | y, n, \\sigma^2)\n", + "\\approx\n", + "\\mathcal{N}(\\theta^*, -(H\\log p(\\theta^*, y, \\sigma^2))^{-1})\n", + "\\\\\n", + "\\implies\n", + "p(\\theta_0, \\theta_i | y, n, \\sigma^2)\n", + "\\approx\n", + "\\mathcal{N}(\\theta^*_{[0,i]}, -(H\\log p(\\theta^*, y, \\sigma^2))^{-1}_{[0,i], [0,i]})\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# input parameters\n", + "n_Ai_sims = 1000\n", + "p = jnp.full(n_arms, 0.5)\n", + "n_stage_2 = 100\n", + "pps_threshold_lower = 0.1\n", + "pps_threshold_upper = 0.9\n", + "posterior_difference_threshold = 0.1\n", + "rejection_threshold = 0.1\n", + "\n", + "subset = jnp.array([0, 1])\n", + "non_futile_idx = np.zeros(n_arms)\n", + "non_futile_idx[subset] = 1\n", + "non_futile_idx = jnp.array(non_futile_idx)\n", + "\n", + "# create a dense grid of sig2 values\n", + "n_sig2 = 100\n", + "sig2_grid = 10**jnp.linspace(-6, 3, n_sig2)\n", + "dsig2_grid = jnp.diff(sig2_grid)\n", + "sig2_grid_ops = berry.optimized(sig2_grid, n_arms=data.shape[0]).config(\n", + " opt_tol=1e-3\n", + ")\n", + "\n", + "_, key = jax.random.split(key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def pr_Ai(\n", + " data, p, key, best_arm, non_futile_idx, \n", + " sig2_rule, sig2_rule_ops,\n", + " sig2_grid, sig2_grid_ops, dsig2_grid,\n", + "):\n", + " n_arms, _ = data.shape\n", + "\n", + " # compute p(sig2 | y, n), mode, hessian\n", + " p_pinned = dict(sig2=sig2_grid, theta=None)\n", + " logpost, x_max, hess, _ = jax.jit(sig2_grid_ops.laplace_logpost)(\n", + " np.zeros((len(sig2_grid), n_arms)), p_pinned, data\n", + " )\n", + " max_logpost = jnp.max(logpost)\n", + " max_post = jnp.exp(max_logpost)\n", + " post = jnp.exp(logpost - max_logpost) * max_post\n", + "\n", + " # sample sigma^2 | y, n\n", + " dFx = post[:-1] * dsig2_grid\n", + " Fx = jnp.cumsum(dFx)\n", + " Fx /= Fx[-1]\n", + " _, key = jax.random.split(key)\n", + " unifs = jax.random.uniform(key=key, shape=(n_Ai_sims,))\n", + " i_star = jnp.searchsorted(Fx, unifs)\n", + "\n", + " # sample theta | y, n, sigma^2\n", + " mean = x_max[i_star+1]\n", + " hess_fn = jax.vmap(\n", + " lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1])\n", + " )\n", + " prec = -hess_fn(hess)\n", + " cov = jnp.linalg.inv(prec)[i_star+1]\n", + " _, key = jax.random.split(key)\n", + " theta = jax.random.multivariate_normal(\n", + " key=key, mean=mean, cov=cov,\n", + " )\n", + " p_samples = jax.scipy.special.expit(theta)\n", + "\n", + " # estimate P(A_i | y, n, theta_0, theta_i)\n", + "\n", + " def simulate_Ai(data, best_arm, diff_thresh, rej_thresh, non_futile_idx, key, p):\n", + " # add n_stage_2 number of patients to each\n", + " # of the control and selected treatment arms.\n", + " n_new = jnp.where(non_futile_idx, n_stage_2, 0)\n", + " y_new = dist.Binomial(total_count=n_new, probs=p).sample(key)\n", + "\n", + " # pool outcomes for each arm\n", + " data = data + jnp.stack((y_new, n_new), axis=-1)\n", + "\n", + " return posterior_difference(data, best_arm, sig2_rule, sig2_rule_ops, diff_thresh) < rej_thresh\n", + "\n", + " simulate_Ai_vmapped = jax.vmap(\n", + " simulate_Ai, in_axes=(None, None, None, None, None, 0, 0)\n", + " )\n", + " keys = jax.random.split(key, num=p_samples.shape[0])\n", + " Ai_indicators = simulate_Ai_vmapped(\n", + " data,\n", + " best_arm,\n", + " posterior_difference_threshold,\n", + " rejection_threshold,\n", + " non_futile_idx,\n", + " keys,\n", + " p_samples,\n", + " )\n", + " out = jnp.mean(Ai_indicators)\n", + " return out\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "jax.jit(lambda data, p, key, best_arm, non_futile_idx:\n", + " pr_Ai(data, p, key, best_arm, non_futile_idx, \n", + " sig2_rule, sig2_rule_ops, sig2_grid, sig2_grid_ops, dsig2_grid),\n", + " static_argnums=(3,))(data, p, key, 1, non_futile_idx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sampling based on pdf values and linearly interpolating\n", + "n_sims = 1000\n", + "n_unifs = 1000\n", + "key = jax.random.PRNGKey(2)\n", + "\n", + "#x = jnp.linspace(-3, 3, num=n_sims)\n", + "#px_orig = 0.5*jax.scipy.stats.norm.pdf(x, -1, 0.5) + 0.5*jax.scipy.stats.norm.pdf(x, 1, 0.5)\n", + "\n", + "#x = jnp.linspace(0, 10, num=n_sims)\n", + "#px_orig = jax.scipy.stats.gamma.pdf(x, 10)\n", + "\n", + "x = jnp.linspace(0, 1, num=n_sims)\n", + "px_orig = jax.scipy.stats.beta.pdf(x, 4, 2)\n", + "\n", + "px = 2 * px_orig\n", + "dx = jnp.diff(x)\n", + "dFx = px[:-1] * dx\n", + "Fx = jnp.cumsum(dFx)\n", + "Fx /= Fx[-1]\n", + "_, key = jax.random.split(key)\n", + "unifs = jax.random.uniform(key=key, shape=(n_unifs,))\n", + "i_star = jnp.searchsorted(Fx, unifs)\n", + "\n", + "# point mass approx\n", + "#samples = x[i_star+1]\n", + "\n", + "# constant approx\n", + "#samples = x[i_star+1] - (Fx[i_star] - unifs) / px[i_star]\n", + "\n", + "# linear approx\n", + "a = 0.5 * (px[i_star+1] - px[i_star]) / dx[i_star]\n", + "b = px[i_star]\n", + "c = Fx[i_star] - unifs - px[i_star] * dx[i_star] - a * dx[i_star]**2\n", + "discr = jnp.sqrt(jnp.maximum(b**2 - 4*a*c, 0))\n", + "quad_solve = jnp.where(jnp.abs(a) < 1e-8, -c/b, (-b + discr) / (2*a))\n", + "samples = x[i_star] + quad_solve\n", + "\n", + "#plt.plot(x[1:], Fx)\n", + "#plt.plot(x[1:], jax.scipy.stats.norm.cdf(x[1:]))\n", + "plt.hist(x[i_star+1], density=True, alpha=0.5)\n", + "plt.hist(samples, density = True, alpha=0.5)\n", + "plt.plot(x, px_orig)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Design Implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 2.2 s, sys: 1.19 s, total: 3.39 s\n", + "Wall time: 4.23 s\n" + ] + } + ], + "source": [ + "%%time\n", + "params = {\n", + " \"n_arms\" : 4,\n", + " \"n_stage_1\" : 50,\n", + " \"n_stage_2\" : 100,\n", + " \"n_stage_1_interims\" : 2,\n", + " \"n_stage_1_add_per_interim\" : 100,\n", + " \"n_stage_2_add_per_interim\" : 100,\n", + " \"stage_1_futility_threshold\" : 0.15,\n", + " \"stage_1_efficacy_threshold\" : 0.7,\n", + " \"stage_2_futility_threshold\" : 0.2,\n", + " \"stage_2_efficacy_threshold\" : 0.95,\n", + " \"inter_stage_futility_threshold\" : 0.6,\n", + " \"posterior_difference_threshold\" : 0,\n", + " \"rejection_threshold\" : 0.05,\n", + " \"key\" : jax.random.PRNGKey(0),\n", + " \"n_pr_sims\" : 100,\n", + " \"n_sig2_sims\" : 20,\n", + " \"batch_size\" : int(2**20),\n", + " \"cache_tables\" : False,\n", + "}\n", + "lei_obj = lewis.Lewis45(**params)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 2**12\n", + "key = jax.random.PRNGKey(0)\n", + "n_points = 20\n", + "n_pr_sims = 100\n", + "n_sig2_sim = 20" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 5.58 s, sys: 453 ms, total: 6.04 s\n", + "Wall time: 6.46 s\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "lei_obj.pd_table = lei_obj.posterior_difference_table__(\n", + " batch_size=batch_size,\n", + " n_points=n_points, \n", + ")\n", + "lei_obj.pd_table" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 17.7 s, sys: 4.76 s, total: 22.4 s\n", + "Wall time: 20.3 s\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "lei_obj.pr_best_pps_1_table = lei_obj.pr_best_pps_1_table__(\n", + " key, \n", + " n_pr_sims,\n", + " batch_size=batch_size,\n", + " n_points=n_points,\n", + ")\n", + "lei_obj.pr_best_pps_1_table" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 13.3 s, sys: 3.29 s, total: 16.6 s\n", + "Wall time: 14 s\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "_, key = jax.random.split(key)\n", + "lei_obj.pps_2_table = lei_obj.pps_2_table__(\n", + " key, \n", + " n_pr_sims,\n", + " batch_size=batch_size,\n", + " n_points=n_points,\n", + ")\n", + "lei_obj.pps_2_table" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "n_arms = params['n_arms']\n", + "size = 52\n", + "lower = np.full(n_arms, -1)\n", + "upper = np.full(n_arms, 1)\n", + "thetas, radii = lewgrid.make_cartesian_grid_range(\n", + " size=size,\n", + " lower=lower,\n", + " upper=upper,\n", + ") \n", + "ns = np.concatenate(\n", + " [np.ones(n_arms-1)[:, None], -np.eye(n_arms-1)],\n", + " axis=-1,\n", + ")\n", + "null_hypos = [\n", + " grid.HyperPlane(n, 0)\n", + " for n in ns\n", + "]\n", + "gr = grid.build_grid(\n", + " thetas=thetas,\n", + " radii=radii,\n", + " null_hypos=null_hypos,\n", + ")\n", + "gr = grid.prune(gr)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "theta_tiles = gr.thetas[gr.grid_pt_idx]\n", + "null_truths = gr.null_truth.astype(bool)\n", + "grid_batch_size = int(2**12)\n", + "n_sim_batches = 500\n", + "sim_batch_size = 50\n", + "\n", + "p_tiles = jax.scipy.special.expit(theta_tiles)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class LeiSimulator:\n", + " def __init__(\n", + " self,\n", + " lei_obj,\n", + " p_tiles,\n", + " null_truths,\n", + " grid_batch_size,\n", + " reduce_func=None,\n", + " ):\n", + " self.lei_obj = lei_obj\n", + " self.unifs_shape = self.lei_obj.unifs_shape()\n", + " self.unifs_order = np.arange(0, self.unifs_shape[0])\n", + " self.p_tiles = p_tiles\n", + " self.null_truths = null_truths\n", + " self.grid_batch_size = grid_batch_size\n", + "\n", + " self.reduce_func = (\n", + " lambda x: np.sum(x, axis=0) if not reduce_func else reduce_func\n", + " )\n", + "\n", + " self.f_batch_sim_batch_grid_jit = jax.jit(self.f_batch_sim_batch_grid)\n", + " self.batch_all = batch.batch_all(\n", + " self.f_batch_sim_batch_grid_jit,\n", + " batch_size=self.grid_batch_size,\n", + " in_axes=(0, 0, None, None),\n", + " )\n", + "\n", + " self.typeI_sum = None\n", + " self.typeI_score = None\n", + "\n", + " def f_batch_sim_batch_grid(self, p_batch, null_batch, unifs_batch, unifs_order):\n", + " return jax.vmap(\n", + " jax.vmap(\n", + " self.lei_obj.simulate,\n", + " in_axes=(0, 0, None, None),\n", + " ),\n", + " in_axes=(None, None, 0, None),\n", + " )(p_batch, null_batch, unifs_batch, unifs_order)\n", + "\n", + " def simulate_batch_sim(self, sim_batch_size, i, key):\n", + " start = time.perf_counter()\n", + "\n", + " unifs = jax.random.uniform(key=key, shape=(sim_batch_size,) + self.unifs_shape)\n", + " rejs_scores, n_padded = self.batch_all(\n", + " self.p_tiles, self.null_truths, unifs, self.unifs_order\n", + " )\n", + " rejs, scores = tuple(\n", + " np.concatenate(\n", + " tuple(x[i] for x in rejs_scores),\n", + " axis=1,\n", + " )\n", + " for i in range(2)\n", + " )\n", + " rejs, scores = (\n", + " (rejs[:, :-n_padded], scores[:, :-n_padded, :])\n", + " if n_padded\n", + " else (rejs, scores)\n", + " )\n", + " rejs_reduced = self.reduce_func(rejs)\n", + " scores_reduced = self.reduce_func(scores)\n", + "\n", + " end = time.perf_counter()\n", + " elapsed_time = (end-start)\n", + " print(f\"Batch {i}: {elapsed_time:.03f}s\")\n", + " return rejs_reduced, scores_reduced\n", + "\n", + " def simulate(\n", + " self,\n", + " key,\n", + " n_sim_batches,\n", + " sim_batch_size,\n", + " ):\n", + " keys = jax.random.split(key, num=n_sim_batches)\n", + " self.typeI_sum = np.zeros(self.p_tiles.shape[0])\n", + " self.typeI_score = np.zeros(self.p_tiles.shape)\n", + " for i, key in enumerate(keys):\n", + " out = self.simulate_batch_sim(sim_batch_size, i, key)\n", + " self.typeI_sum += out[0]\n", + " self.typeI_score += out[1]\n", + " return self.typeI_sum, self.typeI_score\n", + "\n", + "\n", + "simulator = LeiSimulator(\n", + " lei_obj=lei_obj,\n", + " p_tiles=p_tiles,\n", + " null_truths=null_truths,\n", + " grid_batch_size=grid_batch_size,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-09-12 13:40:16.727209: W external/org_tensorflow/tensorflow/core/common_runtime/bfc_allocator.cc:479] Allocator (GPU_0_bfc) ran out of memory trying to allocate 1.09GiB (rounded to 1173514496)requested by op \n", + "2022-09-12 13:40:16.729045: W external/org_tensorflow/tensorflow/core/common_runtime/bfc_allocator.cc:491] *********************************************************************************************_______\n", + "2022-09-12 13:40:16.730879: E external/org_tensorflow/tensorflow/compiler/xla/pjrt/pjrt_stream_executor_client.cc:2130] Execution of replica 0 failed: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 1173514448 bytes.\n", + "BufferAssignment OOM Debugging.\n", + "BufferAssignment stats:\n", + " parameter allocation: 1.21MiB\n", + " constant allocation: 128.20MiB\n", + " maybe_live_out allocation: 12.89MiB\n", + " preallocated temp allocation: 1.09GiB\n", + " preallocated temp fragmentation: 0B (0.00%)\n", + " total allocation: 1.23GiB\n", + " total fragmentation: 128.21MiB (10.16%)\n", + "Peak buffers:\n", + "\tBuffer 1:\n", + "\t\tSize: 546.88MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/reduce_sum[axes=(2,)]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=704\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: pred[100,4096,350,4]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 2:\n", + "\t\tSize: 250.00MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/gather[dimension_numbers=GatherDimensionNumbers(offset_dims=(2, 3), collapsed_slice_dims=(0,), start_index_map=(0,)) slice_sizes=(1, 4, 20) unique_indices=False indices_are_sorted=False mode=GatherScatterMode.PROMISE_IN_BOUNDS fill_value=None]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/table.py\" source_line=66\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,4,20]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 3:\n", + "\t\tSize: 62.50MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,20]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 4:\n", + "\t\tSize: 62.50MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,20]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 5:\n", + "\t\tSize: 62.50MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,20]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 6:\n", + "\t\tSize: 62.50MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,20]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 7:\n", + "\t\tSize: 36.62MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f64[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 8:\n", + "\t\tSize: 25.00MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/concatenate[dimension=3]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=703\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096,4,2]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 9:\n", + "\t\tSize: 18.31MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f32[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 10:\n", + "\t\tSize: 18.31MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f32[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 11:\n", + "\t\tSize: 18.31MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f32[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 12:\n", + "\t\tSize: 18.31MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f32[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 13:\n", + "\t\tSize: 18.31MiB\n", + "\t\tXLA Label: constant\n", + "\t\tShape: f32[10,160000,3]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 14:\n", + "\t\tSize: 12.50MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/select_n\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=963\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: f64[100,4096,4]\n", + "\t\t==========================\n", + "\n", + "\tBuffer 15:\n", + "\t\tSize: 3.12MiB\n", + "\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=54\n", + "\t\tXLA Label: fusion\n", + "\t\tShape: s64[100,4096]\n", + "\t\t==========================\n", + "\n", + "\n" + ] + }, + { + "ename": "ValueError", + "evalue": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 1173514448 bytes.\nBufferAssignment OOM Debugging.\nBufferAssignment stats:\n parameter allocation: 1.21MiB\n constant allocation: 128.20MiB\n maybe_live_out allocation: 12.89MiB\n preallocated temp allocation: 1.09GiB\n preallocated temp fragmentation: 0B (0.00%)\n total allocation: 1.23GiB\n total fragmentation: 128.21MiB (10.16%)\nPeak buffers:\n\tBuffer 1:\n\t\tSize: 546.88MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/reduce_sum[axes=(2,)]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=704\n\t\tXLA Label: fusion\n\t\tShape: pred[100,4096,350,4]\n\t\t==========================\n\n\tBuffer 2:\n\t\tSize: 250.00MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/gather[dimension_numbers=GatherDimensionNumbers(offset_dims=(2, 3), collapsed_slice_dims=(0,), start_index_map=(0,)) slice_sizes=(1, 4, 20) unique_indices=False indices_are_sorted=False mode=GatherScatterMode.PROMISE_IN_BOUNDS fill_value=None]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/table.py\" source_line=66\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,4,20]\n\t\t==========================\n\n\tBuffer 3:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 4:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 5:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 6:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 7:\n\t\tSize: 36.62MiB\n\t\tXLA Label: constant\n\t\tShape: f64[10,160000,3]\n\t\t==========================\n\n\tBuffer 8:\n\t\tSize: 25.00MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/concatenate[dimension=3]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=703\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,4,2]\n\t\t==========================\n\n\tBuffer 9:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 10:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 11:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 12:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 13:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 14:\n\t\tSize: 12.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/select_n\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=963\n\t\tXLA Label: fusion\n\t\tShape: f64[100,4096,4]\n\t\t==========================\n\n\tBuffer 15:\n\t\tSize: 3.12MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=54\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096]\n\t\t==========================\n\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m:2\u001b[0m\n", + "Cell \u001b[0;32mIn [10], line 77\u001b[0m, in \u001b[0;36mLeiSimulator.simulate\u001b[0;34m(self, key, n_sim_batches, sim_batch_size)\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtypeI_score \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mp_tiles\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, key \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(keys):\n\u001b[0;32m---> 77\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulate_batch_sim\u001b[49m\u001b[43m(\u001b[49m\u001b[43msim_batch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtypeI_sum \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m out[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtypeI_score \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m out[\u001b[38;5;241m1\u001b[39m]\n", + "Cell \u001b[0;32mIn [10], line 44\u001b[0m, in \u001b[0;36mLeiSimulator.simulate_batch_sim\u001b[0;34m(self, sim_batch_size, i, key)\u001b[0m\n\u001b[1;32m 41\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mperf_counter()\n\u001b[1;32m 43\u001b[0m unifs \u001b[38;5;241m=\u001b[39m jax\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39muniform(key\u001b[38;5;241m=\u001b[39mkey, shape\u001b[38;5;241m=\u001b[39m(sim_batch_size,) \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39munifs_shape)\n\u001b[0;32m---> 44\u001b[0m rejs_scores, n_padded \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbatch_all\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mp_tiles\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnull_truths\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munifs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munifs_order\u001b[49m\n\u001b[1;32m 46\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m rejs, scores \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(\n\u001b[1;32m 48\u001b[0m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28mtuple\u001b[39m(x[i] \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m rejs_scores),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m 53\u001b[0m )\n\u001b[1;32m 54\u001b[0m rejs, scores \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 55\u001b[0m (rejs[:, :\u001b[38;5;241m-\u001b[39mn_padded], scores[:, :\u001b[38;5;241m-\u001b[39mn_padded, :])\n\u001b[1;32m 56\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m n_padded\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m (rejs, scores)\n\u001b[1;32m 58\u001b[0m )\n", + "File \u001b[0;32m/workspaces/confirmasaurus/research/lei/lewis/batch.py:81\u001b[0m, in \u001b[0;36mbatch_all..internal\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minternal\u001b[39m(\u001b[39m*\u001b[39margs):\n\u001b[0;32m---> 81\u001b[0m outs \u001b[39m=\u001b[39m \u001b[39mtuple\u001b[39;49m(out \u001b[39mfor\u001b[39;49;00m out \u001b[39min\u001b[39;49;00m f_batch(\u001b[39m*\u001b[39;49margs))\n\u001b[1;32m 82\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mtuple\u001b[39m(out[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m out \u001b[39min\u001b[39;00m outs), outs[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m][\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n", + "File \u001b[0;32m/workspaces/confirmasaurus/research/lei/lewis/batch.py:81\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minternal\u001b[39m(\u001b[39m*\u001b[39margs):\n\u001b[0;32m---> 81\u001b[0m outs \u001b[39m=\u001b[39m \u001b[39mtuple\u001b[39m(out \u001b[39mfor\u001b[39;00m out \u001b[39min\u001b[39;00m f_batch(\u001b[39m*\u001b[39margs))\n\u001b[1;32m 82\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mtuple\u001b[39m(out[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m out \u001b[39min\u001b[39;00m outs), outs[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m][\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n", + "File \u001b[0;32m/workspaces/confirmasaurus/research/lei/lewis/batch.py:60\u001b[0m, in \u001b[0;36mbatch..internal\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(n_full_batches):\n\u001b[1;32m 54\u001b[0m batched_args \u001b[39m=\u001b[39m create_batched_args__(\n\u001b[1;32m 55\u001b[0m args\u001b[39m=\u001b[39margs,\n\u001b[1;32m 56\u001b[0m in_axes\u001b[39m=\u001b[39min_axes,\n\u001b[1;32m 57\u001b[0m start\u001b[39m=\u001b[39mstart,\n\u001b[1;32m 58\u001b[0m end\u001b[39m=\u001b[39mend,\n\u001b[1;32m 59\u001b[0m )\n\u001b[0;32m---> 60\u001b[0m \u001b[39myield\u001b[39;00m (f(\u001b[39m*\u001b[39;49mbatched_args), \u001b[39m0\u001b[39m)\n\u001b[1;32m 61\u001b[0m start \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m batch_size_new\n\u001b[1;32m 62\u001b[0m end \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m batch_size_new\n", + "\u001b[0;31mValueError\u001b[0m: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 1173514448 bytes.\nBufferAssignment OOM Debugging.\nBufferAssignment stats:\n parameter allocation: 1.21MiB\n constant allocation: 128.20MiB\n maybe_live_out allocation: 12.89MiB\n preallocated temp allocation: 1.09GiB\n preallocated temp fragmentation: 0B (0.00%)\n total allocation: 1.23GiB\n total fragmentation: 128.21MiB (10.16%)\nPeak buffers:\n\tBuffer 1:\n\t\tSize: 546.88MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/reduce_sum[axes=(2,)]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=704\n\t\tXLA Label: fusion\n\t\tShape: pred[100,4096,350,4]\n\t\t==========================\n\n\tBuffer 2:\n\t\tSize: 250.00MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/gather[dimension_numbers=GatherDimensionNumbers(offset_dims=(2, 3), collapsed_slice_dims=(0,), start_index_map=(0,)) slice_sizes=(1, 4, 20) unique_indices=False indices_are_sorted=False mode=GatherScatterMode.PROMISE_IN_BOUNDS fill_value=None]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/table.py\" source_line=66\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,4,20]\n\t\t==========================\n\n\tBuffer 3:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 4:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 5:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 6:\n\t\tSize: 62.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=41\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,20]\n\t\t==========================\n\n\tBuffer 7:\n\t\tSize: 36.62MiB\n\t\tXLA Label: constant\n\t\tShape: f64[10,160000,3]\n\t\t==========================\n\n\tBuffer 8:\n\t\tSize: 25.00MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/concatenate[dimension=3]\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=703\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096,4,2]\n\t\t==========================\n\n\tBuffer 9:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 10:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 11:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 12:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 13:\n\t\tSize: 18.31MiB\n\t\tXLA Label: constant\n\t\tShape: f32[10,160000,3]\n\t\t==========================\n\n\tBuffer 14:\n\t\tSize: 12.50MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/select_n\" source_file=\"/workspaces/confirmasaurus/research/lei/lewis/lewis.py\" source_line=963\n\t\tXLA Label: fusion\n\t\tShape: f64[100,4096,4]\n\t\t==========================\n\n\tBuffer 15:\n\t\tSize: 3.12MiB\n\t\tOperator: op_name=\"jit(f_batch_sim_batch_grid)/jit(main)/squeeze[dimensions=(2,)]\" source_file=\"/workspaces/confirmasaurus/outlaw/outlaw/interp.py\" source_line=54\n\t\tXLA Label: fusion\n\t\tShape: s64[100,4096]\n\t\t==========================\n\n" + ] + } + ], + "source": [ + "%%time\n", + "key = jax.random.PRNGKey(3)\n", + "typeI_sum, typeI_score = simulator.simulate(\n", + " key=key,\n", + " n_sim_batches=n_sim_batches,\n", + " sim_batch_size=sim_batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(\"output_lei4d2\", exist_ok=True)\n", + "np.savetxt(\"output_lei4d2/typeI_sum.csv\", typeI_sum, fmt=\"%s\", delimiter=\",\")\n", + "np.savetxt(\"output_lei4d2/typeI_score.csv\", typeI_score, fmt=\"%s\", delimiter=\",\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAysAAAGbCAYAAADEAg8AAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOydd3wc1bmwnynb1C1bliX3BrYxxZhmjCGE7hACKUAIEJr5HLiXdhMCIYEACYR7UwwJNZg4CfcSkgCBBFOcgmkGgrGpBgy4W7Isy6pbZ+Z8f+yutGVmtLteNes8v99iNDNn5uxKOo9Oe19FCCGQSCQSiUQikUgkkkGGOtAVkEgkEolEIpFIJBI7ZGdFIpFIJBKJRCKRDEpkZ0UikUgkEolEIpEMSmRnRSKRSCQSiUQikQxKZGdFIpFIJBKJRCKRDEpkZ0UikUgkEolEIpEMSmRnRSKRSCQSiUQikQxKZGdFIpFIJBKJRCKRDEpkZ0UikUgkEolEIpEMSmRnRSKRSCQSiUQikQxKZGdFIpFIJBKJRCKRDEpkZ0WyxyxbtgxFUdi4caPrsYGmvb2db3/724wfPx6/389hhx3GqlWrBrpaedHZ2clVV11FfX09fr+fgw46iD/84Q+uZS644AIURXF8vfbaa/1Ue4lEIklH+qP/KMQfSV544QXpEMmAoQ90BSR7J1/4whdYtWoVdXV1A10VAJqbmzn22GPx+/0sWbKEkpISvv/973PqqafyySefMGLEiIGuYk58+ctf5t///jc/+clP2Gefffi///s/vv71r2NZFuecc45tmR/84AcsXrw46/gXv/hFfD4fhx56aF9XWyKRSHJG+qNvKMQfmdx2220ce+yxacdmz57dF9WVSLqRnRVJn1BTU0NNTc1AV6ObCy64ACEEK1eupKSkBIBRo0Zx2GGH8eSTT3LBBRcMbAVzYPny5axYsaJbMADHHnssmzZt4jvf+Q5nnXUWmqZllZs6dSpTp05NO7Zy5Uqam5v5/ve/b1tGIpFIBgrpj+JTqD8ymT59OkcccURfV1ciSUMuA5P0ys6dO7n00ksZP348Pp+Pmpoa5s+fz9///nfHMnbT+B9++CFf//rXqa2txefzMWHCBM4//3wikUha2fXr13POOecwevRofD4fM2fO5O677y64/v/85z95+umn+fnPf94tGoApU6YA8NlnnxV8b4gvD4jFYrbnTNOkq6trj+6f5IknnqCsrIyvfe1raccvvPBCtm/fzuuvv57zvZYuXYqiKFx00UVFqZtEIpHYIf3hzlD0h0TS38jOiqRXzjvvPP7yl79w44038vzzz/Pggw9y/PHHs2vXrpzv8fbbb3PooYfy2muvccstt/DMM89w++23E4lEiEaj3dd98MEHHHroobz33nv87Gc/429/+xtf+MIXuOKKK7j55pvT7qkoCp/73Od6ffYDDzzApEmTOPbYYzEMo/vV3t4OgMfjyfl9pPLb3/6WGTNmUFlZSSAQ4JhjjuHuu+9m06ZNhMNhXnjhBY466ijWrVtX0P0zee+995g5cya6nj4hesABB3Sfz4W2tjb+/Oc/c9xxxzF58uSi1E0ikUjskP6wZ6j64/LLL0fXdSoqKjjppJN4+eWXi1I/icQVIZH0QllZmbjqqqscz//mN78RgNiwYYPjsc9//vOiqqpKNDU1uT7rpJNOEuPGjRNtbW1px//jP/5D+P1+0dLS0n1M0zTx+c9/3vV+pmmKqqoqATi+fv/734twOCwuuOACMW7cOFFeXi4OP/xw8corrzje95VXXhEjR44UN998s3jmmWfE0qVLxemnny68Xm/3fTVNE5dccono6upyrWOuTJ8+XZx00klZx7dv3y4Acdttt+V0n3vvvVcA4pFHHilKvSQSicQJ6Y9shqI/3nrrLXHllVeKJ554Qrz44ovioYceEjNnzhSapolnn322KHWUSJyQe1YkvXLYYYexbNkyRo4cyfHHH8/cuXPzGk0KBoOsXLmSiy++2HUdcjgc5h//+Aff+ta3KCkpwTCM7nMLFy7kV7/6Fa+99hqnnHIKQNp5Jz766CNaW1u59dZbOfnkk9PO/epXv+K3v/0thx12GIZhMHnyZF555RXGjRvH73//e0477TQ2b96cNvWfZNKkSXzwwQeMHj26+9hFF11EW1sbq1atIhQKcdhhhzF27Ni0kb9cyHxfmqahKApA9792uJ1LZenSpYwcOZIzzjgjr3pJJBJJvkh/7B3+mDNnDnPmzOn+esGCBZxxxhnsv//+XHvttZx00kl51VMiyQe5DEzSK48++ijf/OY3efDBB5k3bx7V1dWcf/75NDY25lR+9+7dmKbJuHHjXK/btWsXhmHwy1/+Eo/Hk/ZauHAhEI/Kkg/JNc+HH344hxxySNrrvffeY+rUqeyzzz6UlpZy4403MmHCBFRV5Zvf/CaWZbF+/Xrb+9bX17Ny5UoWLFhAIBBg5MiRfOlLX+Lxxx9n9uzZLFy4kE2bNnHKKafwzjvv5FXfzPe+cuVKAEaOHGm7dKKlpQWA6urqXu//zjvv8Oabb3Luuefi8/lyrpdEIpEUgvRHNkPVH5lUVVVx6qmn8s477xAKhfIuL5HkipxZkfTKqFGjWLJkCUuWLGHz5s089dRTXHfddTQ1NfHss8/2Wr66uhpN09i6davrdSNGjEDTNM477zwuv/xy22vy3WOR3LiYGeVk7dq1rF69mjvuuMO23IcffkgoFMqKopXk1Vdf5Vvf+hZXXHEFN9xwA9u3b+evf/0rixcv7h4J03WdCy+8kJkzZ+Zc3/r6ev7973+nHdt3330B2H///XnkkUcwDCNt3fG7774L5BY+cunSpQBccsklOddJIpFICkX6I5uh6g87hBBA7jP7EklBDPQ6NMnQ5PTTTxc1NTVCiNzXHI8YMULs3LnT9b7HH3+8OPDAA0UkEilKPd99910BiCVLlnQfi8ViYsGCBWLy5Mm264G7urrEIYccIn70ox853nfr1q2isbEx63hbW5t4/vnnxZNPPikaGhqK8h6SLF++XADiD3/4Q9rxk08+WdTX1wvDMFzLh8NhUV1dLQ477LCi1ksikUjyQfpj6PnDjpaWFjF27Fhx0EEHFauaEoktcmZF4kpbWxvHHnss55xzDjNmzKC8vJx///vfPPvss3z5y1/O+T4///nPOeqoozj88MO57rrrmDZtGjt27OCpp57i/vvvp7y8HIA777yTo446igULFvCtb32LSZMm0dHRwSeffMJf//pX/vnPf3bfU9d1jjnmGP7xj384Pnf27NnMnTuXH/3oR9TW1lJZWcnPfvYzPvjgA/7xj39krSeOxWKceeaZzJo1i+9973uO9x07dqzt8YqKCk444YScP5d8OOWUUzjhhBP41re+RXt7O9OmTeORRx7h2Wef5eGHH+4e/Vu5ciXHHXccN954IzfeeGN3+b/85S+0tLTIWRWJRNIvSH/YMxT9cc455zBhwgQOOeQQRo0axfr16/nZz37Gjh07WLZsWZ/UWSLpZqB7S5LBTTgcFosXLxYHHHCAqKioEIFAQOy7777ipptu6h5VymVkTAghPvjgA/G1r31NjBw5Uni9XjFhwgRxwQUXiHA4nPbMDRs2iIsuukiMHTtWeDweUVNTI4488siskSpAHHPMMb2+h40bN4qTTjpJlJSUiOrqavGNb3wjrV5JTNMUZ599tjjttNNELBbL+TPqTzo6OsQVV1whxowZI7xerzjggAOyonr961//EoC46aab0o6fcMIJorS0VLS3t/djjSUSyXBF+mNwsSf+uP3228VBBx0kKisrhaZpoqamRpxxxhnijTfe6Md3IBmuKEIkFhxKJMOcRYsWsX79ep599ln8fv9AV0cikUgkQwTpD4mk75CdFYkE2LRpE5MmTcLv96dtpnzmmWdYsGDBANZMIpFIJIMZ6Q+JpG+RnRWJRCKRSCQSiUQyKJF5ViQSiUQikUgkEsmgRHZWJBKJZAC55557mDx5Mn6/n7lz5/LSSy+5Xr9y5Urmzp2L3+9nypQp3HfffWnnP/e5z6EoStbrC1/4Ql++DYlEIpH0M8PFH7KzIpFIJAPEo48+ylVXXcUNN9zAmjVrWLBgAaeccgqbN2+2vX7Dhg0sXLiQBQsWsGbNGr73ve9xxRVX8Nhjj3Vf8/jjj9PQ0ND9eu+999A0ja997Wv99bYkEolE0scMJ3/IPSsSiUQyQBx++OEcfPDB3Hvvvd3HZs6cyemnn87tt9+edf13v/tdnnrqKdatW9d9bPHixbz99tusWrXK9hlLlizhxhtvpKGhgdLS0uK/CYlEIpH0O8PJHzIpZJGxLIvt27dTXl6OoigDXR2JZNghhKCjo4P6+npUNf/J43A4TDQa3aPnZ/7u+3w+fD5f2rFoNMrq1au57rrr0o6feOKJvPrqq7b3XrVqFSeeeGLasZNOOomlS5cSi8XweDxZZZYuXcrZZ58tOypDAOkPiWRgkf5IZ7D4Q3ZWisz27dsZP378QFdDIhn2bNmyhXHjxuVVJhwOM3liGY1NZsHPLSsro7OzM+3YTTfdxA9/+MO0Y83NzZimSW1tbdrx2tpaGhsbbe/d2Nhoe71hGDQ3N1NXV5d27o033uC9995j6dKlBb4bSX8i/SGRDA6kPwaXP2RnpciUl5cD8R/0ioqKAa6NRDL8aG9vZ/z48d2/i/kQjUZpbDLZsHoiFeX5j6q1d1hMnrsp6/c/c1QslcxRNLuRtd6utzsO8VGx2bNnc9hhh+VUf8nAIv0hkQws0h89DCZ/yM5KkUl+wysqKqRsJJIBZE+W0ZSWxV/5YiZ2AOby+z9q1Cg0TcsaBWtqasoa/UoyZswY2+t1XWfkyJFpx4PBIH/4wx+45ZZb8nwXkoFC+kMiGRxIfwwuf8hoYBKJRJKBhSj4lSter5e5c+eyYsWKtOMrVqzgyCOPtC0zb968rOuff/55DjnkkKz1xn/84x+JRCKce+65OddJIpFIJHuG9EfxkZ0ViUQiGSCuueYaHnzwQR566CHWrVvH1VdfzebNm1m8eDEA119/Peeff3739YsXL2bTpk1cc801rFu3joceeoilS5fy7W9/O+veS5cu5fTTT88aMZNIJBLJ0Gc4+UMuA5NIJJIMLCysAsvlw1lnncWuXbu45ZZbaGhoYPbs2SxfvpyJEycC0NDQkBYzf/LkySxfvpyrr76au+++m/r6eu666y6+8pWvpN33448/5uWXX+b5558v4F1IJBKJpFCkP4rPkJ1ZefHFF/niF79IfX09iqLwl7/8pdcyvWXuBHjssceYNWsWPp+PWbNm8cQTT/RB7SUSyWDGFKLgV75cdtllbNy4kUgkwurVqzn66KO7zy1btowXXngh7fpjjjmGt956i0gkwoYNG7pH0VLZZ599EEJwwgkn5F2f4YD0h0Qi6SukP4rPkJ1Z6erq4sADD+TCCy/M6hXakczcuWjRIh5++GFeeeUVLrvsMmpqarrLr1q1irPOOotbb72VM844gyeeeIIzzzyTl19+mcMPP3yP6yyEYN3r63nqnmd56+/vEuoMEyjzM/eEAzj27KP41x9e5qXHXicSioAA3atz8PH7M+Ow6fzlV8/Q3tzRfS/dq3PElw5mx6c7Wf/WhrTnBMr8hDrDe1xfyfBG6BoY2SEYTUDxa6hhM+N6FbPUgxo2USNGzwkFYgKUKWVojUGUkAUCUMGs8WGaCkwpQdvcidpmJI57iXWFYf9xaC3t6Bu6UCIWwq8Rm1aGVVLGKy/8os/ee77rh1PLSQY/0h/u/ph78kG89tSbGFEj7VzF6DJKy0po+Kwpq36TDhzP5ve2YZnpo8O6R6OytoJdW3fv8WcgGToU2x9iYjl6UxA1ZHYfN0b5MS2BOimAurULtd0EFawaL7GOMMwai9rWjr4pmPCHijGlDMtfxssvL+mz9y79UXz2igz2iqLwxBNPcPrppztek0vmzrPOOov29naeeeaZ7mtOPvlkRowYwSOPPJJTXdrb26msrKStrS0tmsPuHa3c/JWf8v6rH6HpKqbR06ArCgz974JkbyH5o6gk/j81JopRoqMFjaxzJqDoCqohssrEJpSib+1CsdLLWEDssBF439wdPyF6zoUOGoO+uxXPpjBCBSXl75/k19F9SxFqCS++f09a/Z1+B3MhWXbDh3WUFxB6sqPDYvKMhoKeLRkYpD8kkuJRbH9ExpbhbejM8ocAYnOr8KxpzfbH/rXobW14Nrv4Y58ShFLCi+vuJRXpj8HJkF0Gli9OmTvffPNNYrGY6zVO2UABIpEI7e3taa9M2prbuXL+9/nwjfUAaaIBKRrJ4CHzR9FONJnnLEBVQTFEVpnY+FL0zV0kl+KmnTukCt8bu1EsUFJFc2At3k3N6Fvis4NKxjLe5Nee9V1obW0cPeuy/N6kRJIn0h8SSe8U2x/RsWV4t3Xa++PgSryrW7P8EZ49Gu/WXehbe/HHJ0G09naO3lf6YygwbDorvWXudLvGKRsowO23305lZWX3yy778E8vuocdm3ZmSUYiGaxkRpg3/Fq3aDLPiVIPWNnHLRX0rV22ZaIHV+N7s9X22VqwA7XNyJJMVh0t0BqjiJKo+4UF0B+hJyVDB+kPiSR3iuUPz/ZO2zKxA6vwvtVm+2w10pm7P3ZEoSTifmEBSH8Un2HTWYHcMnfmmw30+uuvp62trfu1ZcuWtPMNn+3gtadXZ63jlUgGG0KL/5zb/bSrZvaoFyRGxbpitmXMSWVpI16pKG0hhE3rEzxqLN71wV5F030fC3xr2zjqoCtyK5Aj/blBUjI0kP6QSJwptj9i48sd/UFHxNYfoSPq8/aH9912Fsz+z9wK5Ij0R/EZshvs8yWXzJ1O1zhlAwXw+Xz4fD7H808/sAJVVaVsJIMeYQlbMQhAidn//JpVfjyt9sEctO3BrPXHALHJZXg+7bQtoxqhrDXGvSKA6uyNnHuCRffKg7zLSfY+pD8kEneK7Q99R5etP4yJpXg+67Ito5rSH3srw2ZmJZfMnU7XOGUDzYUP//2JFI1kSKA4DOqYfg0FhxGzsOE4ca2ELdsy1kjnP860htxHxboRoLbI6HeSvkP6QyJxp9j+UJ38Ue11rIO6I1SYPxw6TJLBw5CdWens7OSTTz7p/nrDhg2sXbuW6upqJkyYwPXXX8+2bdv43e9+B8Qjt/zqV7/immuuYdGiRaxatYqlS5emRWm58sorOfroo7njjjv40pe+xJNPPsnf//53Xn755YLrGe4q/npIiaSvsB0Z05yXsSiFTFurzmWUaP5/mCmAEivuyJiJwCxg/XAhZST9j/SHRFJ8+scfLs93mMFxQwEo8n4w6Y/iM2RnVt58803mzJnDnDlzALjmmmuYM2cON954I+CcufOFF17goIMO4tZbb83K3HnkkUfyhz/8gd/85jcccMABLFu2jEcffXSPYuRX1VS4rlmWSAYTdk2lGjNsjiau1wpoQlz6FVapnndzLVQQ/uKOu5ii8Jdk8CP9IZEUn/7wh+LiD1GiFeQPfFre9XBD+qP4DNmZlc997nO4pYhZtmxZ1rFk5k43vvrVr/LVr351T6vXzZGnHcrry92fKZEMBoSqoFrZv1NaND12fiqWT0MLxmzvZ430oO7K3jypr+9A6Ep3qMpUjPHleD4N2lvPAcUCY2Rp7gVyQK453ruR/pBIikux/WGO8KLtjmaV0T5z9oc5thz9s1De/jBHSH8MdobszMpQ4dhzjsJf4rxGXyIZLCiWcGzjrYBmO8Xv2R1O5uPKLlPitS2jtceIzi63jeaibTXAZdmAbd3KVKZS3ARaFgpmAS/7VdYSSWFIf0iGCv3lD7XdIDbL3h/KtsL8MVmtzKtMr/eU/ig6srPSxwRK/Zx93RkDXQ2JpFcUQDgsOVFDpqNUjBF+2zKeLV1YZfbT8mqrCbqCyHicb/0uwkeMzGsqP3zYaH7//K15lJBIhgbSH5KhQrH94d3WiemwLFhtc/DHpy1EDq3Oyx/Rg2v4/d+lPwY7srPSD4yoLW6vXSLpKxTRMzqW2uArgBnQs45DfHTMLPfZykhYAuFTssts7CI2swo0JWuETFvVTGROhatwkudCR1YT3m4fxnJPsEThL4mkmEh/SIYKxfYHlkB41azj+qYujH0q7P3xxi6iB+bmj/DhIwg3Bl3fUyFIfxQf2VnpY2LRGA9e978DXQ3JMMNpFKu3c8mBKkvtGbJKXq+FDCy/CkrPse5zHRGMEYGsMmrQQgiBWevLKuN5ezfGlHJiU0viZVQQGngAdU07oWNrMSvjGx+F1vMCsEZ5CB5fx9Z9LXZfuQ8rHi884pIdhUzhJ18SSbGQ/pAMBIPGHyETS1jERvuzyujvtWFMKsWYku4PHVDebieyYLSrP0LHjGHLLEHLlfuw4jHpj8HOkN1gP1R4+fE36GixT4AnkfQ3uQzcKPSsPxaaCil5HtRY/CaWTwGhoCZCPgpVRUTjm+ljI/yoIQPFtEBREH4dpTmK8CmYdSUowRiKCcKrQsTEuylIbEIp1hhvPISkooCmYn7Ywmf7l1PvL0ULhsGwEB4VKxBg+6hOOLkeiG9KvOGPf+eELx9VtM+pUHFI2UiKifSHZDAxEP7Ar6PtimD5FIzaMtRgDCwLdA0rAr4tQYxxJVi13vizFAVUFWP9brbsV844bwlKKIxiWOBRsXx+ttYF4eSxQDw45Q/+bwUnfEX6YzAjOyt9zMo/voKqKlhyfk8yCBCaiuqQZC4zYosCYFpYOihGz9cAWiQ+vmVUeNDaY6iWhTchHs/ucM8NBKjRRKxJE9SNXQSOryT097a0Z3s2d8HmLjb9z1yio0gLG7PdtrYpWcEt6NjPft1zoVhCwcpcEJ1jOYmkWEh/SAYTA+0PbXMHgWMrCf0r3R/61iBsDbLl9oMJj1HS/LG1tzcl/TEkkMvA+phdDa1SNJIBwa7Z6y0Jl10Z06c7jve4JuFyeJQVci5i+h0q4YaaKCeR7GVIf0gGikHrD5dk82ZAKcwfgd4vkwwscmalj/H6PANdBYkkZwTZbb3i9sdSAQNBilsG4kICzYsCy7kgp/ElgwHpD8lQol/84ZK/sWB/uCSaLATpj+IjZ1b6mMn7T0DT5ccs6X9sFaHklS8LAC0RdtIOM+A8aobXoV7lzrbxtJJ/ZiwB3t3FHX02UQt+SSTFQvpDMlAMWn+UuPijRUh/7KXIT6aP+cKlx2MaMi+ppH+xG+ECUEzRfT4Ty2svAQUQWnb4YQCtPeIcNWaC/dz69uW7MEf4bctVvtaZf6ukQPWLbb1flwciseY435eQa44lRUT6QzIQDGp/PO/ij9cL88eIV6Q/Bjuys9LHTN5/IvvN3xdVkx+1pP9I7E20F4Gq2GeWT2xktCtj+u1FpMXAKrFP/Mg2+80pI1Eo3c8+K3fJMx+ht5P76JgFviZY+epPciyQGzL0pGQwIP0hGQgGvT9m2vvDv+JjPLvJyx/+7YIXXrsjxwK5If1RfGQL2A9c8pNzEXKTpKQfUbr/45Boy+44YHlV2zKeLgPLZ3+OZPjJzDqEwNqnxLZ+219pRUwozSqjAbWPtsTXHvcmHCseCrP+0ZZeLpRIhi7SH5L+ZtD7Y1UrOPhjzKMt8T0oOfhDi0Ddn6U/hgKys9IP/PamR103FUskfYEiQKQIJzO0ZJLUBl+LWmnT+WnnIhamjXC0qMAs1Wzlpn4ctBVOtVDoCHahTC2Ll0mpVMkrG6h/uA01alMJ6JaQFoIxD7XSekFp1v33FFOoBb8kkmIi/SEZCAa7P9qDXShTsv3hf20DY3/bipaMGpbZaUl8rXfCmKW7aT9X+mMoID+ZPmbzh9tY+8/3sEw5Mibpf9TEj53jBkctWx5qNL4h0vTYnItYxPwaQlfThdNlYgmI1PrjmYJTzikfB4lVaFhT/PGhrwSVzUBzEP/CkShTyxBaj3H8r29kyh2fMfLFGHrqcmIB3haoWRGlZO12Or/nxZxgctz87+T0eeSKhYKFWsBLTuNLiof0h2QgGfT+2BXEd1K2P3yrNzH5vz+j5h9RvKkTJwJ8TYLa5REC726n4wc+YlMtjj/i2zl9Hrki/VF8ZOjiPmb5AytQdRVLbpKU9COpo2DJF8Sz9aYe08yeKXgrpYwKkIiBbwGWCqoVP+cN98R5DCvg1eJJvzRA2xHuLhMt1dDDJpoJnnYT2uPlohMDeHZHwVeCNm480QZQy2Fn1SZK9vURUn3EvlOVeEKMEmKIv7VSslYneISBcnwVkemgUNX9ZtuOLyvSJ5f8nGToScnAI/0hGQgGsz9iEwPoKf6INfX4IzDNR0jzYnx3ROIJBn4MfH9rpfQtna4j4/4IzUz3R8fnpT8GO7Kz0sesX7NBikYyINg2ew4ZiBWH/wewyjzonTHbZ3hKdJQuI6uMCnjDwjZ+vXLsRLS3syd1ayZMpPESBTHSpn6nVhE61eE9KWDW2FavYAqdkjd7SZomkeSD9IdkoBi0/jhmItq79v5ougTM0dltsHJqFUEXfxjSH4MeuQysj4mG7X9JJZLBiF1jLlSX0R63xtVpU7Db4JFLwi9HFBBy2EWyFyL9IRlK9Ic/3PRRkAcUEDL36qBHKr6PGVFbiaIqMpqLpN+xjZWvKvG5/DzKqFHnP5iER8XphsKroUSM7BMuz1fCStxf+cyGW6DaR7ksmPia4/yn5OWaY0kxkf6QDBSD1R/CZaJRDSpYI0T+/gjmcX1Ot5T+KDZyZqWPWfCVI6RoJP2O00+cGjOdw056VPv4+WHnUJVmicexebVqyrDLcaW+1QYe+xr61jnczA0V/O+4GLQArAKzD1uySZUUEekPyUAwGPxhOvhDWePij/cL+F1Rwf9ucZdaSn8UH/nJ9DHHfG0epZX2scIlkr7CSQDJZF92qDHLWSrl9lLxNQQRisM9J3lQbE5o7zVg7Gv1hJpJoeyZ3a4jd1kIIKRwzfyj8yjUOzL0pGQwIP0hGQgGgz+s8QF7f7zfgDnd3h8l/2gDm8l8RwQoXXDVMdIfgx35yfQxXr+XC249e6CrIRlmJMVgK5ZEiMesJFyA0B2yE3fEHEUUrSu1vZ/+8k6MMRX28mq0wAuZNtJLKvH9O4+NKwqUrFT46iXH514mBwoLOylHxiTFRfpDMhAMBn/4VjUQc/JHk7M/Slbl0QYrUPpP6Y+hgPxk+gEjmk9XXyIpPmmx7k2B5dGyjgOohsBSs2WkAGaJblvGt72LWG3A9lyXrxNzVGmWqDwr1hMZL+LCyRghq/zdLjzva9kVTyUxa+97TcdsjjpcJJEMfaQ/JAPNgPlDs/eHvmI9kXp7f5Q90oJvjZpd8VQS/gi8qGLujDhcJBlMyM5KHxMJRfj9LX8a6GpIhiHJEa7U1bjJRl+NmYgU4aS+FEtgerSsMlrQQPgUhK50OyB5zrMjRHRsaXeLkjxescmis70Tc2o1wh8PuSIUEIqC/scPiYwyMadZ8REyRYAm0OqqGPGrXZQ8q6K0qz1vwux5M8puldLHYee+XnZ9vpwHb32kWB8bAKZQCn5JJMVC+kMyUAwGf1Rus+hs78KYOjLbH499SLTaxJqa7Y/KB1oo+6uC2qr0vIkUf2jNCuV/hB37+2k+sUL6Ywggo4H1MS88+irB9iKHKpJIciSZvCszaZcAFCOegkqo8QiSyRl1ocRlpACGX0eNmihCIBQFCw3NMBAqWAEdJWqCiE//a21RFAsiY0pQw0Z8DbOmECj1oH7WQvtYQXlNDaLVjFfGo8DuNjxvqUSmBhDTvKhWQkb7V9HRZNG1oZWxb1cQ3deL8IISEfjeibD1zArajo3X1wooPLhqDZfw9aJ9bskNj/mXk5uhJcVD+kMykAwWf2if7aJ9JJTMqEFpM+NhjT0q5u52/GsVIlMCMNXbXQdrvyraWwSdm1oZ/3g50X08CC+oEfC8F2XL1yvYfUL8WrNUZenjq6U/Bjmys9LHvPzEGzL0pKRPsHCeGk3NQJz8VwEMj4oeSwRITFykWOkbJ1Nn1fWwgVGiogcFihCoiVCSigV0GZi1Vag7WlGNHp35GuNxILX6up71y3WVjLKAHdA6dxRVq9sgBmyIn/Z9GoJPQ2z4jzGE69SUHZ4lbJuY8eZm+LI+iM45FQ6fRGFYQsUqYLOjJZN6SYqI9Iekrxhy/gBograDRlK5NuGPTfHTvs9C8FmIjYtHExqvgZIsWcKWTH/s78/4IASdc8odPonCkP4oPnIZWB/T2tQmRSPpM9x+suwmlO2iq/RWxvI5Z8xSLJEihj3HDCj5xcgHUBPlhij33HMPkydPxu/3M3fuXF566SXX61euXMncuXPx+/1MmTKF++67L+ua1tZWLr/8curq6vD7/cycOZPly5f31VuQ9BHSH5K+ZCj6w22llFmi5O8jVcEsGbp/Cg8Xf8iZlT7G65epUSWDiF7acWFzieL2x1LqEFsR6qCYdjXoBQFKkfcg99c0/qOPPspVV13FPffcw/z587n//vs55ZRT+OCDD5gwYULW9Rs2bGDhwoUsWrSIhx9+mFdeeYXLLruMmpoavvKVrwAQjUY54YQTGD16NH/+858ZN24cW7Zsoby8uKOHkr5H+kMyqBgE/nCrglJIui0hEjM7xUP6o/gM3e4k+fUoL7jgAhRFyXrtt99+3dcsW7bM9ppwOFxwHfeZOxVVG9Ifs2SQYvUyCWHX7CmKcGwOhWp/N60t6lzGpzuOtgmffR00lyX4vgYzfUdnLgjwNTpnSS4Ei8I2SeZb9Z///OdcfPHFXHLJJcycOZMlS5Ywfvx47r33Xtvr77vvPiZMmMCSJUuYOXMml1xyCRdddBE//elPu6956KGHaGlp4S9/+Qvz589n4sSJHHXUURx44IGFfyB7IdIfkmHNkPWH8x/0/m0xMPPseAjwNUh/JBms/hiyrWCyR3nDDTewZs0aFixYwCmnnMLmzZttr7/zzjtpaGjofm3ZsoXq6mq+9rWvpV1XUVGRdl1DQwN+v9/2nrnwhUuPxzKLmx1VIgHQRE/UlEyEg4jUaPxq29j1fvuJVhUQHsW2TKylFaHb50UJTfbb1qH8/TZCdR7b6fzKvzfm3yopMOLZrXkWcmdP4+S3t7envSKR7PCY0WiU1atXc+KJJ6YdP/HEE3n11Vdt67Vq1aqs60866STefPNNYrG4cJ966inmzZvH5ZdfTm1tLbNnz+a2227DNHMbdtyyZQtbt/Z8nm+88QZXXXUVDzzwQE7lhwLSH5LhjjIo/LHb0R/hiT7bOpR92O7oj4p/NnXngcmHyr9vy7uMG9IfxffHkO2s5NujrKysZMyYMd2vN998k927d3PhhRemXacoStp1Y8aMca1HJBLJ+sFKZdw+9cw94QA5OiYpOm7ZhN1EZOqarQQ8wXhjZVum1Gt7zt9lIqrL7UfA3m90zE4cGVtiO6JWvlPDt8PKfXbFgtJPDf658cEcC+TGnmYgHj9+PJWVld2v22+/PesZzc3NmKZJbW1t2vHa2loaGxtt69XY2Gh7vWEYNDc3A/DZZ5/x5z//GdM0Wb58Od///vf52c9+xo9//OOc3vs555zDv/71r+7nnXDCCbzxxht873vf45ZbbsnpHoMd6Q/JcGdw+MNy9scHzv6IjrH3R1mLjn+7EY8WlguWoOzjGP/c9FBu1+eI9Efx/TEkW8BCepSZLF26lOOPP56JE9NDRXR2djJx4kTGjRvHqaeeypo1a1zvc/vtt6f9UI0fPz7rmnNv/BrCkqNjkuLjlGlYSTmZeU43nFfGGn77ZF+e1ghmucdWcNGuDkR5SfZzdoTomlVie7/SN9tonxWwrceYez6Lj+D19itjgd4lqL73o14u7H+2bNlCW1tb9+v66693vFbJ2BAqhMg61tv1qccty2L06NE88MADzJ07l7PPPpsbbrjB8Q/xTN577z0OO+wwAP74xz8ye/ZsXn31Vf7v//6PZcuW5XSPwYz0h0QSZ3D4o93WH1pTmK6Z9v4oWdNGxwwnf2xAC9N7h8USeNoFIx9Y737dACD9kc2Q7KwU0qNMpaGhgWeeeYZLLrkk7fiMGTNYtmwZTz31FI888gh+v5/58+ezfr3zD/P111+f9kO1ZcuWrGse+PbvihoxSSJJkvlTldo8KyI9ckpm020p6cm8ADxhk1hAty2jd8SIVfSMkCXP+btMIiMMRFVZ/FzKM/0rPqVrv5Is8XmAwAch2g9IZCdOKVMSK2H8f29Ab08Gzc+sePwfb7NFzU8+ovnKGRQbC6XgF8SXA6W+fD5f1jNGjRqFpmlZbVZTU1NW25ZkzJgxttfrus7IkSMBqKurY5999kHTepZXzJw5k8bGRqLRaK/vPRaLddf373//O6eddhoQbx8bGhp6LT/Ykf6QSOIMDn9YRKoc/PH3TwnOsveH/8MQHbOz/RGwShl3x6d4WpP+yKh5Yk+Lf4fJqDs+Yud/7EOxkf4ovj+GZGclSb49yiTLli2jqqqK008/Pe34EUccwbnnnsuBBx7IggUL+OMf/8g+++zDL3/5S8d7+Xy+rB+sVD5Zu4F1r6+X4SclfUYyBr6Zkf1XEBdOMnlv6jkS50yP2t3QJ895QgYmYJToWffT26NES1SsQHr2Yv/mKHR2Edt3NKK6AqFr8XOqgnfVdoK+GF37+hHelHsBJdvDtM8op/2AMoyyeF0sDbSyMkYu+5gxv2smsNVEicXXJSgxQelnBnX3NxINbafhxzMITdA4ZtqlRf1M93QaPxe8Xi9z585lxYoVacdXrFjBkUceaVtm3rx5Wdc///zzHHLIIXg88chR8+fP55NPPsFKGY3/+OOPqaurw+v19lqv/fbbj/vuu4+XXnqJFStWcPLJJwOwffv2bqHtDUh/SCQpiR8zPNCv/tji7A/Pa9sJeWMEbfwRaAzTsW85HbMz/FFZwchl66l/aAclGwyUqAAhUKLxZV/1d28nHG2k4faZBKd4OGZy+sDDniL9UXx/DMnQxYX0KJMIIXjooYc477zzev3gVVXl0EMPdR0Z643lv/4Hmq5iGnIaX1JcTCW+yT6JlvIj1i0UIHX7YsynoUfM7hE1PdZTSCTm6BUSDUMwHg847Ad/VIVE4+ULppQ5ZBye7Skb7zoAXymMLiW6sxElJlA7Q/g6Q/BZMxYQmjmSwLpd8fpth9KU+m297gA69/EkhlFGZ71n4VHomqbTNS1lL4AFbSfXO3xKhVF46Mn8ylxzzTWcd955HHLIIcybN48HHniAzZs3s3jxYiA+8r5t2zZ+97vfAbB48WJ+9atfcc0117Bo0SJWrVrF0qVLeeSRR7rv+a1vfYtf/vKXXHnllfznf/4n69ev57bbbuOKK67IqU533HEHZ5xxBv/zP//DN7/5ze4oME899VT39P5QRvpDMpxIdUHm8dRjqXtAnPwRDWh4QsXzB3PHoTf07g9vZwg2xP0R3nck/o96/FGSUr/t/zWb9lnexCZ7G394oXOml86ZKb6wBB0nSn8kGaz+GJKdldQe5RlnnNF9fMWKFXzpS19yLbty5Uo++eQTLr744l6fI4Rg7dq17L///gXXdeP7m6VoJH1CchOk01iwbTQXSziHq3QYvNUnj4RE5yITrcM5QogSs3+WWe7c7ERrtIKigUVripuPwhIKllv2MZdy+XDWWWexa9cubrnlFhoaGpg9ezbLly/v3gvR0NCQFqFq8uTJLF++nKuvvpq7776b+vp67rrrru4Y+RDfnPn8889z9dVXc8ABBzB27FiuvPJKvvvd7/ZaHyEEkydPZtOmTZimyYgRI7rPXXrppZSUlLiUHhpIf0iGC04dld7OOR3XzOL6Qy3AH1aZffQwgMhoPf9oYApEpD+6rxms/hiSnRXIv0eZZOnSpRx++OHMnj07654333wzRxxxBNOnT6e9vZ277rqLtWvXcvfddxdcTyNaSJYiiWRgsM1a7BaNv5C/o1xu5xSrv7f7CWd/DXouu+wyLrvsMttzdhsSjznmGN566y3Xe86bN4/XXnst77oIIZg+fTrvv/8+06dPTzs3adKkvO83WJH+kAwX+nO3U97+KGh1o/P9CvKAokh/ZDAY/TFkOyv59igB2traeOyxx7jzzjtt79na2sqll15KY2MjlZWVzJkzhxdffHGPpq6q66pQVQVLrjmW9AG9JfXKPG9pKlrMuYdhV8aMmDi15cKnQpfD/RxiYyou2YK1LkGsUslvdsUCzakOBWIVOI1vDe1tgKiqyvTp09m1a1eWbPYmpD8kw4FkE5zP7EkSR3+4jFDl7Q+vAl0OP9tOsZVdkj7qnRbRahXyGfSyBLr0R1HoS38oIhm3TFIU2tvbqayspK2tjYqKClb+aRU/OuvnA10tyV6I2zS+Rc/GSbvjduXMEg9a0CGTb0kAgtmp52NHTSLwWXbCKoCg1YqnMZT9HH+8QdbC2YJo+8p0mk6tyHs4sO63O1n1j58B2b+D+ZAse9sbx+Ivy38sJ9xp8L3D/lXQswcLTz/9ND/5yU+49957bWcQJH2H9Iek2LjtWbE7Dv3nD+PIifg32keYcvSHTwXF3h8dp02h8YyqvKPnjX1wB6+8+AtA+mNP6St/DO1u3BBg/umHUjmqfKCrIdkLcYqRn3ouExVsM/8CKMGYc0bjSvss3J6XN+IUwER4dPt1z2GL8H7VtuXKln/mOvOS/RDQOgV33HRW7mVywEQp+DXUOffcc3njjTc48MADCQQCVFdXp70k/Yf0h6QY2C/PijOQ/tBf3eTsD4fkk1rEIjLT3h+lz29Ecegv2T9EoLdb/OTWr+dRqHekP4rvjyG7DGyooHt0Fv3Pefz0wnsGuiqSvRChgmLZRHYhfRQsq4yZXUYFTL+OFjayyoQbWggEAhAKZ52LTPHh/yR7dqXULCNYE8WzM5JVD8+2EGapjtZloKQMkGkhk1H/6GTnyTn+gaZAzd92c/RlxY1SZQkVK48wkqnlhjpLliwZ6CpIEkh/SPaU1NVUWT5InMzLHxooRhH9MdGLf0P27EqpVU6wJubgj6CtP9SwRc1z7TR9sdKm5jYoCjVP7WLBf0h/FIu+8ofsrPQDTRubB7oKkr0U1YqHMFYdhCMy/gXQTIj5dfSwkVVGCxtYHhUlsa8leS6AQrQugHe7gHB6x0R/cSPR+RPwbsge0tKbI5hVHvTWWNqzvI1dhKePgMaOLOFU/eFjjPJZ7J4fiBvTrv1OmHTUcx3EarITZkkK55vf/OZAV0GSgvSHpFhkdloUEU/uaBdZ0tEfBkQDOp5Qkfzxyiai8ybg3ZS7PzxNQSJTq2BHJ1ow3R+Vj32CWb4vuz5XFk8Iabd/xYrHWB7913aMUdIfxaSv/DH0u3GDnGBHiEf/58mBroZkL0YTdE+JJ2WUKqXUeELdibvCBoZPySojADVmYZamZxoWgOez3UQrYzCiAvT0LZPqK5sJTfMhMiJAeuvqUFtjhKdWIjQl7Z7+9bsxy310HTQSyxt/A0KJv0b9+gPGPLwLX5PVUzmrp3Bgq0ndrxtpPrmc3UeVcPulzon3CsGk0Kn8vQPTNHnsscf40Y9+xI9//GOeeOIJTHNveXdDB+kPSTGwW/LV3d6L9ISQqf86+cMbMjB8alaZgv2xajPhKV5nf0ypyPKH79NWrDIfoQOy/VG97CPqlu0kkMwBJkS8gyLir5KNBvX3NdJ0WiW7PlfGTxZJfxSTvvCHnFnpY/7xvy8RCdlvQJZIioWa8Td9Eisx66KQaMhFz3V6VHRP91u6imLGS1qaitoVRQGi5V60sIGSGKFSQh70slI629rx14+AiBW/qa4iGtpRYj7C47zolokSsRCagpg2Ec+LmwAITR+BYppxcegqaluUsrd2YZZ56DxwVPcomBK1aJwUhDEjGXvHJiL7VyP8KmrIxLu6mW03TiF0aTwxpOVT+MuOrVxfxM9zOE/jf/LJJyxcuJBt27ax7777IoTg448/Zvz48Tz99NNMnTp1oKs4bJD+kBSL5Ib5TEcI4jMrducgPqJt7w+rxx8eFSWRD8jSC/OHtaMj7o96D7qwUKIWaApiykQ8Lyf8Ma0y7ilLIHQVrS1KydpdmKUegrNHdQ/aKVFBw7QwjNMZd9sGovtVYyX84VnTzLabpxH8Vl28vn6FJ7dv5roiftbSH8X3h+ys9DGv/e1NFBREYQHFJcOc3iK2ZDZtqUJSUm9AXEimX0MPJ0eb4v+oAIYFZSXQGYz/fwJvR5TY2GoCIn2qvGxMvKE3RqloHXFp6V3xc/6t8fXHse0NKKRnQA6s303XoaMp/XdT2v20zhhl/25i80/mEKlV097wtu9OTH+TX8iIkmJB5/7F3YRsChWzAHEUUmawccUVVzB16lRee+217g2Ru3bt4txzz+WKK67g6aefHuAaDh+kPyR7gp0/ko7IyR+p5wSYPg09YuOPWIo/Ynn4Y6SK1pnhj+3x5WC2/vikjeCc0ZSsyfBHV4ySt5rY+qMDCY3V0qKBbf3e5PQ3cnpV+teWkP4oIn3lD9lZ6WPamjuQ0aEle0K+sfBd7+WWr8FyiDVfHoB2+3N5Re5K4Jb40fIr+WcxUxPliohAwSogMotLfuchw8qVK9NEAzBy5Eh+8pOfMH/+/AGs2fBD+kOyp+xt/nBM2gKYASXvsMWoCqb0R9HoK38M/W7cICdQZh+yTyLJlaL+qeLWkCv2zYFwSSJZSAuiuLwhJVbAuxXkF65S4orP56OjoyPreGdnJ16vdwBqNHyR/pDsKUX1h1uyxX7yh0tOysI8IARqId6R2NJX/pCdlT5m5uHTUTX5MUsKwy0WvqX2PmqWVUZxWVDisZ9o1T5rcIyfb5QojnWwAqptGW1ndvjKJIGthquMnPBvLe66/uQ0fiGvoc6pp57KpZdeyuuvv44QAiEEr732GosXL+a0004b6OoNK6Q/JHuCmz9EL5PYtmXcFiQW5A/nOjj5Q29xbusDm6Px/ZD5IMC/RfqjWPSVP4b+JzPI+cKlJ2A5TY9KJDngNFWvWzhKwNTtOxGeRNZfuzKhCgfZCLC8Dkr5YJtjHaJjy2zr4N/QRnhKmW1Sr8rnGvJvlQSM+mdDnoXcsYRS8Guoc9dddzF16lTmzZuH3+/H7/czf/58pk2bxp133jnQ1RtWSH9I9hQnf2gif3/oERd/lOfvD+WD7Xn7w7uxjfBke39UrGh0n/1xYOTKHXmXcUP6o/j+kHtW+pjaiTXMP/0wVj35b6x8e/wSCe5JvQT2f9trhnAs45S4K7C5BcpKobMr61xYsyhByRKfr6YOo0JFs1mT7Pu0PR4SU9jUYUQJymed2WU+bMa/dTzhejW3TosFZR/FeH7Dr3O4OHdMVMwCxnIKKTOYEELQ1tbGI488wvbt21m3bh1CCGbNmsW0adMGunrDDukPyZ7Sb/7Ykr8/vKPrMMrjQVoycfOHVVmCYtn4Y30LJRvHEZyg59ZpsQQV70V5btODvV+bB9IfxffH0P5khghfvvILWHKTpKRAUmWT+VOk2BxLHjeTseczzulhIyuufpJwuQq+7HWlvk8biVVotmXMT7aBln3cW1dHZHyZbZnS1U10HVJjU3MYc9fHaEHR+3IwCzytgqq7P+zlwvwZriNjQgimT5/Otm3bmDZtGl/84hc57bTTZEdlAJH+kOwJ/emPSAH+sD7N3x8la5voOniUTc2h9u5P0DtF78vBLIGv2aLy/o/drysA6Y/i+0N2VvoYIQR3X/EQSr4RKiSSFFSyE3pB+prk1BfE4+AbXjWrDMRzr1gZSbYA/A0dhEaXQEkguw4fbiVa1ROKJVnGN7qOaNMOLF/2soIyo5zIhLLulib1XODNnXQePjqei0XpOedp6mLcjz7GtzPRW8nstCS+DmwxqbxrHTtv2C+rrpLCUFWV6dOns2vXroGuigTpD0lx6C9/+Bo6CNfk5w/v6DpiOxrz98dbzXQdUpPlD31nkHE/+hh/QyLEcmanxYx/XbrBoPzuD2m+bmZWXSWF0Zf+kJ2VPubDNz7hs3c2IeQUvmQPSf6ypo6Spf6/pWWf06MWJmDoatpxASimIOLpCSWcPO7f0ooVCWOMGwkVpfGNk5oKuo62uYUQEWJlWloZ78haQCEYCGOVKmnPKRXlxKohNLUCs0zvFouigndzB8FKP12H1RKt82OWahgVOlYJjFiyhvq7tlH2UQytU6BGBFpnfNp+7B2bCE3UaP7hLCJ1KkfP+lZRP2sLteDXUOe///u/+c53vsN777030FUZ9kh/SIpFr/5Q8vdHVM/2h29rwh9jq3P2h2fUGEAh5A1jlWT4wyrHrLT3h2dbJ6FyH8G5o4ml+MMsVai+823G/mILFe9F0dst1LBAb7OoWhth3O0b6Zrqofnm/QiN1zl63/9X1M9a+qP4/pB7VvqYZx78B5quYRrmQFdFMkRJHQVLHV81VdCsnnOY9mV06E70aFaXobX0rPX1x3qu1sbWpa8PtoAyL5GuBlQTFCzoNPB2BoEWTCBWE0DfGeou4wMEuzEBS1dQjHjkeC9AczsAwf1HUfJuM1jgbQjFz70eAmD7DQfSNVV3HkbxQfsBXtoPmNhzzIK2E8Y4FCgMUyiYBUzJF1JmsHHuuecSDAY58MAD8Xq9BALpo6QtLS0DVLPhh/SHZE9x9IcS3/yePJc6beHoj6oytNYef/iMnqu1+rqeeyUPl/mIdjWgOPljlB+9Odxdxou9P3RA3x33R2jWSAIf7Er3x5vxCJON1+5PxwyP434Vy6/QOtdP69xJKQcFHcdJfxSLvvKH7Kz0MVs+2iZFIykYp0guAIrVy3mbY1rYZSOIw+CtWR5Aaw3ZnlODzoHtk6LJxAo4Z/WKVue4sT7tQRAbWdymrND1w0N9zTHAkiVLBroKkgTSH5I9wdUPogB/RPOPTGeW+fG02YerV0PZG/W7n+/gD+Hmj1Fa/tHAFIhKfxSNvvKH7Kz0MaYhw05KBhP5LydxbT8LWZ3idr8Cwk5CL3WU5EwsFuOFF17gBz/4AVOmTBno6gx7pD8kg4sC/OE2+FTQ6kbnxr7gNCVDf/XVoKAv/SG/RX3M6PEjZVIvScE4RWtJPe+EXTlLz/9nUeuKOtZB+JxHuZwq55alXu+w8k8KKUDvLO4fdUKoWAW8xBBP6uXxeHjiiScGuhqSBNIfkj2h2P4QBfwsuvrD63I/p8oZLv5ot/JPCmmB3lHc2Uvpj+IztD+ZIcBx5x6NZcrRMcme4djYOx13aOg727tAs+9gxHz2fvCPGu34LNPjMiVfV2JbJvDuTsxS+3IV/27Pv1VSofyV4kYfMVEKfg11zjjjDP7yl78MdDUkSH9IikOx/BHucPGH194fvlG1js+yCvCH/31nf1T+uz3/2XlNoXzV7vzK9IL0x1+Kfl+5DKyPOWzhHEbWj2BXw+4Cpzwlwx0rZSNkKslwlLbrjh2G1CoAPB4wbUaSNjQgMjdJJjCq/Xhastcd+5qCWBpg2tTBYQmLGhN0HlRN6eqdKBmXlD27kZ2nVsUzHufSbgvQ2wX/9/BVOVycO5YobP3w3hC0adq0adx66628+uqrzJ07l9LS0rTzV1xxxQDVbPgh/SHZU4rpjwA4+2Ojsz/MEX703dn+8O4M5e8PQ9B1QDUla7P9UfL3zahfqsbyAbmE+7YEnlbBw49c3fu1eSD9UXx/yM5KH6NpGpctuZBbz/z5QFdFMkTRRHZc/CSmR0GLiSzhqFZcUopN49deGqYipmUJRyU+u+KJZAtMbQk71iFWW4Z3e2dWGV9TmEh9AN/27M35vk87Maq86K3RNOGoMYua5W3sOKMqu+KZJCo0+vGdTPrPSb1fnwfJaflCyg11HnzwQaqqqli9ejWrV69OO6coiuys9CPSH5I9xc0flq6gGvn5I+QPE3Dwh+EFPWrjj93O/jBqS/Fs78rPH591OPpj9FO7aTyzOrvimSQSrY5+rIlJV03q/fo8kP4ovj9kZ6UfWPda8TOkSoYvqY26HhNEPQoeuw6LiG//UDPKVOyCSE0AX0soWzgbGjAn1qHF0sv46usINzegRbPF59veSWRcOd6tHVl18GwPEa3x4d0ZSTvnaQkRmViBAei7o2lSrHjyU8ySfWk+sSz+MLv2OyGo2sd3E9qnvLePTJIHGzZsGOgqSFKQ/pAUk9R2WDMEMY+Cnoc/Aq0QGRnA15rtD2VjA9bEOtQMf3jr64jsbOg+Tuq57V1Expbh3ZY94OXkD701THR8ua0/ypdvwCrRaPpCRfwNaDYzHFa8B1f36C7C08ty+dgkOdJX/hj63bhBTsfuTp68+9mBroZkiJNsbu0mxr0xgeGJ/3/mQJgKmJqadc63s5O2ag/4/aBmNAObGoh6rawy/lF1WCqYJZ6sOvi2dhAdU4oV0NIzHQP6zgihqRVY/vTn+Da1Axpdh4zGKEsfNxnxyEfU39dAyUYz+01ZUPZxjPpfbGbHV0fQepifK75wS1ad9gQLpeDX3kI0GuWjjz7CMJzDi0r6FukPSTFw84cnJjDz9ceuToKVqq0/xKYGop5sf/hq6hCKgz+2dRIdXZKXP7xbOsBSCR5ck+WPyj9/Qv3d2yn7JGabwb78/Shjf7aZhnNG0XJkKVcuvDn7g9kDpD+K7w85s9LHrPjtSoyojJMv2XNSk3qZKVP0AtBjPcuMTZXuqXFTV/Ak1v4aPg0lZqEIgVAUylvDEIN2oKIsAGZi7EqDro070IBYTTme0jKw4mEhrTFVeLa3xpN2jSxBiRogQHhU9MYuNMAIaFhV/viaY01BqCqBT+MJvcLjSsGnxmfgFQVPYxdl/27C0iA4pwbLExeS0hFj+0IDJmrU/+BjzNmjsPwaashEeXcHjT+eSeeMCfH3ryu8Vmofx79QhnNSr2AwyH/+53/y29/+FoCPP/6YKVOmcMUVV1BfX8911103wDUcPkh/SIqFmz+0Avzh74hBLEYICJQGEp2CuD+im1L8UVIWz+migFkX94cFGNUBlJgJlkB4NDxNwXjnKKBhJv2hApq7P0pX78TSILR/DZZXQRGgdBls/6IFE73Uf+8jrFkjEQl/iHVNNN42i47944mFhQde89nnESsU6Y/i+0N2VvqYN59fixB7wa4pSZ/itJ7X6biWmKJXSJ8eVejJag/xzYhJ9IiJUaKhBc34et3EdRUAnaG0Z1Uk/t+zswMjFEHvjKIAyTExDdB2BTHLdNROI61+esiEUPYaZAD/1i66Dq2l9N870o6rJpSs2cnmO+YQGa2mFdx+6z7pNzk7Yz2ygK59Sygmw3nN8fXXX8/bb7/NCy+8wMknn9x9/Pjjj+emm26SnZV+RPpDkgv95o+AhhZK90cAoCvdH8mc5Z6dHRilEfSuaPzr5H0BtSUU90eXgUJPcmEtZKK5+CN48GhK3mpKO66a8SiTW358IOF6LW1z/fbb9s24y6j0LwUEpT+KRl/5Q3ZW+pj2ls6BroJkCOO0ZaNQFNP9Dx/bjPMu4SXp5X52uLXHli/HKGCpqGD6izsiZVFgBuK9YBr/L3/5C48++ihHHHEESor0Z82axaeffjqANRt+SH9I9oTB4A9RZH+4vSHLr+QWBSztfgpmQPqjWPSVP4Z0N+6ee+5h8uTJ+P1+5s6dy0svveR47QsvvICiKFmvDz/8MO26xx57jFmzZuHz+Zg1a9YeJ7gpKQ/0fpFE4kBvSb3yRdhtNkw9b1cHN6H0cr+cH5J8VrSAdytAi+ZfTGLPzp07GT16dNbxrq6uNPkMdaQ/JHs7xfZHbx2BgfaHWog/LIEakbOXxaKv/DFkOyuPPvooV111FTfccANr1qxhwYIFnHLKKWzevNm13EcffURDQ0P3a/r06d3nVq1axVlnncV5553H22+/zXnnnceZZ57J66+/XnA9918wEzXfJEWSYYdtrPuUY/YJGdW8x2GEMJ0Tgen2d9Paw93x+LPqUO5zrINVqtmW8TQEHesQ2BzLP4M94N9Y3D0rosDNkWIvGBk79NBDefrpp7u/Tgrm17/+NfPmzRuoahUV6Q/J3kR/+QNc/OHQ8VA73PzhzdsfemPI2R8bovknK1EgsDGSX5lekP4ovj+GbGfl5z//ORdffDGXXHIJM2fOZMmSJYwfP557773Xtdzo0aMZM2ZM90tLyca6ZMkSTjjhBK6//npmzJjB9ddfz3HHHceSJUsKrucplxwnc3lJesVtBMxJRJ6Y5SwBh99sT6jnnlllfPaFVMDy2U/l6w1djnWIjSm1rbdvSwfhfSpsl4NVLd+Wf6tkwYQ3WvIs1MsthVLwa6hz++23c8MNN/Ctb30LwzC48847OeGEE1i2bBk//vGPB7p6RUH6Q7I30V/+0CI998x6jtfZH8LRH8G8/eHd1kF4ermtPyqe2553BnvFhHFvteZVpjekP4rvjyHZWYlGo6xevZoTTzwx7fiJJ57Iq6++6lp2zpw51NXVcdxxx/Gvf/0r7dyqVauy7nnSSSe53jMSidDe3p72SmVUfTWf//pRKHJ0TOJCUja20+gu5yyHLR6ay+xEMsxj5v08XabtcQChZ4evhEQisGr72RXvp+1YmmJ/v4A/K/swgPez3ZRsMHOfXRFQ/l6UJ9bdnWOB3EhukCzkNdQ58sgjeeWVVwgGg0ydOpXnn3+e2tpaVq1axdy5cwe6enuM9Idkb6Nf/RGw94cW6j9/4PPZ+2NTG6Uf5zG7Ygkq1kZ44kPpj2LRV/4Ykp9Mc3MzpmlSW1ubdry2tpbGxkbbMnV1dTzwwAM89thjPP744+y7774cd9xxvPjii93XNDY25nVPiPciKysru1/jx4/PuuaE849B5Ds1KRl2OE3ZK4Ch2Z9LzU6cSSRgHz/D02kgPIrt/SzV/jl6VwxjRMC+Di0RLG+2VFQgVmtfpuTtJjoPG217rnbJOjxtovcOiwDfDouK37zfy4X5M5xHxgD2339/fvvb3/Lee+/xwQcf8PDDD7P//vsPdLWKgvSHZG+k+P6wnw3xhIzuJV+ZZUXGv93P6YphVhbPH4H3muk6xMEfv/wI726r9w6LJQhsNyn/3Tr36wpA+qP4/hiSnZUkmZt1hBCOG3j23XdfFi1axMEHH8y8efO45557+MIXvsBPf/rTgu8J8TBtbW1t3a8tW7Zklf/Vfy7NO0CFZPiR+iMiSB8N85gQ09PPp/6/ZVPGFzIIl9h3WNSYwEqZtk+W0SwwU8SW+hzP7hBGdSCrjAqIqMAq07PrsD1ItK60e4lA6v1K32ii8/BaLL8aL5P4APS2CGNv/oDA5kR+icxOS+LNln0YQ/vjJ+y84UDb9yiRuCH9IdmbKL4/TMIOA16qKbpzYqXeTyV9wCv1OXpbCLOqeP4oebOJ4NzRWf7QOqKMvWUdpZ8mkhFmdlosAUJQ8V4U5fHP2Pm9A2zfo2RwMSRDF48aNQpN07JGrJqamrJGttw44ogjePjhh7u/HjNmTN739Pl8+Hw+x/PvvPgBWz9uyLlOkr2bZLPp9LdH6uhYcq1xt3Di+Rcx1XinIvVeAjCU+EhZahlf0MDQ4nP9upHeaGtRizDg8aqoUaunw2KC5VWJBXR8HVFESmdB3x0iCqhVfvTWcJpwTKEQrfCiaaC1Rkmmh/DsChIbFReO1h5Ga4/FK6gr6A2dxADjsFq0XV3xc5qCUe1nxJPbqO4w6DxlLKGJHiyvghoRlH4SpeTpzWz70T4wM56D5ag5l/PymuJN5ReaTXhvCD25tyP9IdlbKbo/Qi7+iFlEAN2josZ6/KFaYHlUDJ+Ot6vHAwBaW9wfWqUfrS3dH5apEC33oGkKWluGP0aWIHxalj/UnV0YAsy5o1F3B9E6YghNwRjho+qpBkZ0xug6qZ7gZG+3P8o+DuN/bivbfrwvHBD3x4IDLuOld+7Zw0+/B+mP4jMkZ1a8Xi9z585lxYoVacdXrFjBkUcemfN91qxZQ11dXffX8+bNy7rn888/n9c9M3lu2b/Q9CH5MUuKjFtHxSJ91CiZrEtJ+Tp5TLd6vk497hHpZZLldFN0iyZalj4+4SfeaVFSKqYkjvnboigWWFpKHQT4AE9rOL68oNrf85yuGL72KPruaDxjcWKpgBoV+LZ34t/YjqclSni/kaimQI1Y+Dd34Q9blL2xg8Cnnez81kw23bYf266dyvYrJ7Dt+1Nom+sjOkrFqFCI1qjsnuePd1RSPrz2z+f+R2Yu9Oc0fj4hdAFWrlzJ3Llz8fv9TJkyhfvuuy/t/LJly2zD7IbDxY2YNlSR/pDsbfSbP0o8pOIj3mnJ8kfMwteZ8ICaUoeEP/S2pD98PWVCMXwdMfRWG380dHX7IzIj3R++iEXJ6ib8n3XSfMk+bLptFtu+O41t/zWBrTdNZfcRASK1GrERKpExGruOLo13VJKYgo7PZYfa3ROkP4rPkJxZAbjmmms477zzOOSQQ5g3bx4PPPAAmzdvZvHixUB8en3btm387ne/A+KRWiZNmsR+++1HNBrl4Ycf5rHHHuOxxx7rvueVV17J0UcfzR133MGXvvQlnnzySf7+97/z8ssvF1zP7Z80YhoFxGKV7JX0NqOSiXA5V8iztJjLOl6HU1a5H63VvqFSw4bz8037QIyWQ9QxgFiVmv8QigKxSpfEYwVQqDjyLZMMoXvPPfcwf/587r//fk455RQ++OADJkyYkHX9hg0bWLhwIYsWLeLhhx/mlVde4bLLLqOmpoavfOUr3ddVVFTw0UcfpZX1+/15v5+9FekPyd5Ev/nDdPlZdPCHWe7H0+bkD9P5+Q7+EC7+iFZreUcDQ4VYlfTHYPfHkO2snHXWWezatYtbbrmFhoYGZs+ezfLly5k4cSIADQ0NaTHzo9Eo3/72t9m2bRuBQID99tuPp59+moULF3Zfc+SRR/KHP/yB73//+/zgBz9g6tSpPProoxx++OEF11NujJQMdYr/E9wHU91FvmV/ySY1hC7E/yh+7rnnuPfee7n99tuzrr/vvvuYMGFCdzjcmTNn8uabb/LTn/40TTaKojBmzJi86z9ckP6QSCTdSH8Men8M2c4KwGWXXcZll11me27ZsmVpX1977bVce+21vd7zq1/9Kl/96leLUT0A6qbU8uG/P8GSo2MSnEe6nI4rLucKeZbpUdEizqNZdmiJpF72o1waBB1mV1QQVnY5Neb8u+Bpt+KjXPnMrgjwtOX3nvqazBC0dnsTkiF0r7vuurTjbiF0ncLjLl26lFgshscTX6bR2dnJxIkTMU2Tgw46iFtvvZU5c+a41vnLX/5yTu/t8ccfz+m6wY70h2Rvod/8oalo5OmPTjd/qBB0KOjgD8UlS72n1SI6Us1vdsUCvVX6Y7D7Qy6G7WNOOP8YKRoJ4J5NuDfsyrj9VAnVXg7ezphzEq5K+2zCmgkoTnH6naUQrS+xPe5/eydGhW57v4rX2vJvlVQof7E5z0Lu7Oma4/Hjx6eFpLUb5SokhK5TeFzDMGhujn8GM2bMYNmyZTz11FM88sgj+P1+5s+fz/r1613fc2p93V6S/kP6Q1IM8vaHQ/4Vb9DZH0aFfb4U1XSrQ/7+8L3fmz/y7JppCuUvSX8Mdn8M6ZmVocCc4/ZnzOTR7Ni0U07pSxDY/y2ukh7BJRXDA3rMvozjc1Li52feT2gKipn9s6i3RR3LGCMCeFpCWWW8LWEsXQEje32xErLXoWpBcEY1pf9uyjJY6YoNqF8ageV3sGUmFnhaBf/7x2tyuDh34uE88x+PTL6dLVu2UFFR0X3cLeJTvuFu7a5PPX7EEUdwxBFHdJ+fP38+Bx98ML/85S+56667HO/7m9/8xvGcZGCQ/pDkQrH9QQH+0NojjmXMqhL01uwpFO/uSEH+CE0fQclbO7P98a9NaF+uxixRyCnetyXw7rJ4+PFv935tHkh/FB85s9LHqKrKf/zyYikaCdBLNuHEv5nnPDai6S7jtT+uCRBaehjKbhKiyTyuAkal1/ac1hJCOMyuxKrtE3d5d4WJTCi1rV9gXSuxWj+ZCXtVE2ofb8H2hpkkPsjRjzQwadKkXi7Ojz0dGauoqEh72cmmkBC6TuFxdV1n5MiRtmVUVeXQQw/tdWRMMviQ/pDkSjH9odIzu5KPP8xye3+orUFnf1T5bcu4+cP/USux0dn+UCyo/dOu+Beil98ZS0h/MHT8ITsr/cCrT/27T/YUS4YeWaNUKf+v4Syc1GOp59RoTxKuTFQTLMWmDGAGeo6nnvO0RTHKPLZlRMqwXeo5X1OQaH2pbb29m7uIjC3JOqd1RAEt3mHJkFjZio3UPtHak63MDiseLWbMwzvpPNy+kd0T+iP0ZCEhdJ3C4x5yyCHd640zEUKwdu3atDC7kqGD9IckF4ruD+HsD8undB9PPad3RDFL8vRHc4jomDz90RlDEWq8w5Lhj9IXNlP3p5aEP5xCXAoUC+p+00TXIdX21+wB0h/FR3ZW+pjdTW0899C/+iKkkmSIkmyO7P4OdxJOsoypZp9TLTAdZolVAUZptjz0UCI2v5bdOHo6Y0SrvFn7VFTibb9RkT0c59veRaw6gFnhySrj3RYkOLUCM2OdsbexC7XNoOuwWmI16W+g4qlPGfs/myh/P0bmfk4lJqhcE6H+tg00frOG9oO8fGPu1fYfwCDnmmuu4cEHH+Shhx5i3bp1XH311VkhdM8///zu6xcvXsymTZu45pprWLduHQ899BBLly7l29/uWcZw880389xzz/HZZ5+xdu1aLr74YtauXdt9T8nQQfpDkg8a9p2SPfKHV8sqo0WEoz/0YIxohc/ZH+U2/mjsIlblwyzP3R+epiBqm0Fw7miMDH+ULd/IuDs2Ubk2gpKRzFKJCka8Eab+1k9puKSW1kMDfOPgq+w/gEHOcPKH3LPSxzz30D+xLLlBcjhh0ZNsK5VMeaRuuE8tk0zMJQAj5TqFeOZhiK9DVsz4RUIFLZL9bEF8KZinK74OwCj3o0RjYAqER0EJmaiJKf1YmZ6I0qVg6gq+1mj3c0TAi2JacTGZFp72+LnoiACKEN0JvPSWUFxIQHRsGYol4hUJm5R8Go9uEhnpx6pKyEoIfBs6KHt9BxYQmlGNVaLHh+Bawmz/7kQARv3wAzz11YgSD0pXjM5gG21X70vb3Cnx22gKH82wXy5QKP0VejLfELqTJ09m+fLlXH311dx9993U19dz1113pYWdbG1t5dJLL6WxsZHKykrmzJnDiy++yGGHHZb3+5EMLNIfkiR2nY8kFj0jz8l/i+aPaHy0yPQSXxtmxttcNWb1+COgo5oWKAqmpuJrj9/Q1MHyx/2BpoBloXfE/RGrCkDSH6qC3prijzGlKIj4vpNe/FH6ZhMWEJk+Ij6rYwnYHWbb9ZMAGHXTB3jHjEAEdJSuGB2RDnZfsy+7j5gWv40O66cG8vtm9IL0R/FRhOhtYZ8kH9rb26msrKStrY2KigquO+lWVq94Z6CrJelHHDep4zw64FQmTDzLvB2WEh/5crqfXbNnBDT0kH2YRsd6l/vQOyK2ZcwyHbXTyCscc9fBYyh9yz5ayeY75hCpzXPCV0Dppybv33IDkP07mA/Jskc9dTl6qfOmRieMrggvn3Z3Qc+WSKQ/JE6kdkgycWq7o4DDtpTC/OHX0R0SATv6o9SP3mWfFLIQfwQPGEPJO/b+2HLbgYTr8xyDF4Kyj2K8d9uNgPTHYEUuA+tjOtucgohL9macGt+8cQ35lf/tlF426trV29JdKmETFab3SjifSq6Fzvd+TssYCkUIpeDX3sDvf/975s+fT319PZs2bQLiCceefPLJAa7Z8EL6Q5Kk6KPKBfnDfZbP1nselzaxEH+46KgwfyiFlXNB+qP4/pCdlT6mrNI+Vrhk78auCS6oGXJzQwE3FL3EoLert+qW58FmzXKvuNxOjRQgL9GzjKFYWCgFv4Y69957L9dccw0LFy6ktbUV04zPxFVVVXVnPpb0D9IfkiRFb1kK8of7n4y23ou5tOkF+cP5fgX5wxKFlXO7pfRH0f0hOyt9zJzjDkDJN0mRZEjj1OwlNz86hS22+ylxWgIGYPm1vOuhxEzH+gmPfXOgdkQc621W+B2bV7PcPnGXp6EDpwGkks+i7h00B0o+kSPQxeKXv/wlv/71r7nhhhvQtJ6fsUMOOYR33313AGs2/JD+kCTpLey9bRJHl/sV5g8jf390hV38YZ+IGFz80djp6I/STyKunRlbFChZn51DTFIYfeUP2VnpY06+6Fg0TX7MEoeY9QlSk3pl4lRGDzl3PGI+1T4bveF8T0uzv5sKWAH7dcD69k7HesdGl9gLtLGL0PSqrBj5ABV/25z3iJ9iCmZ+Fs2vUC/0R+jJwcqGDRuYM2dO1nGfz0dXV9cA1Gj4Iv0hSTIo/OGWjd7NH34nfwTz9oenqYvwVHt/lD+7NX9/GDBjs/0+nEKR/ii+P2Qr2MdUjqrglEuOc80oKtm7SI3G4kTmuaSI7H5K3FY4WQ7rgb0Ry1kCpbptHfSwyAo32YOwLaMCRk3Att6+T9uxPIr9/XQ9Hokms97bOij7KJb77IqAyrci/O61/8mxQI63HcZrjidPnszatWuzjj/zzDPMmjWr/ys0jJH+kNiRjz9cckIW5o+SIvtjlP3svJs/FM3eH57tnZS/H819dkUIRrwR4nevS38Ui77yh+ys9AOHnjwHGXRteGLXOJsO55xGzvxA1OE3VYsJ57yJuv1zvF0Glle1PZf8Mc08roVMDIdM9Z6dIcyS7JEzFYhVZcfbByj5oJngwWNs3/PoO9/Ht9PqvcMioGSzSeCp93u5MH+G88jYd77zHS6//HIeffRRhBC88cYb/PjHP+Z73/se3/nOdwa6esMO6Q9JkkL84QViDs1SQf4IFuCPsIlR5eCP5nDe/vB/1EzoQAd//OoD/I1m7x0WISj91CDw9Dr36wpA+qP4/pB5VvoYy7K4+4qHBroakn4mtRHNjI+vkR7GODkippBI1Ej6CJkCeC2IaOCziTqsYh+GUjPA1OKZ7DPDSqpRC8uvoYbNtDokZagqPeJJltNbQsSqS/C0BNPKABA0MEb40HZH0kb5fDvDREb60cMx1C4zrQ4lqxvpmjMG36e70NtjCAUUEe8Y1f/wHXb+x350zvT0VCD1Q7WgYk2E9s07if3XwWzcuJFJkyZlfziSvLnwwgsxDINrr72WYDDIOeecw9ixY7nzzjs5++yzB7p6wwrpD0kmOok2OvF1Lv7wCIip4LHpmRTsD5+GGsnDH60hYlUleFod/FHlRWuN5uyPwNpGQvvX4tnYgt7R4w81bFF/y7s0L55J+wG+ngooPZVSTKj6d4jdDbuIXXOQ9EcR6St/yM5KH7N6xTvs2LRzoKshGQCSjaqV8nWy/U5mqhekZx1WSJ9QSC3jNeMisABPxrNUASHAqyqoKSNKqglmQAPLyop4okRMTEBJdFqSZ1XA9GtYCugxCyUmus/pbaF4BmKPhtYa7h69UhUwLTB0YEQArSXUvetTDxsY5X6oFCiWQGuLolgCy6OitnUhOmJ0HVSLEo6idsVAUzDLfVQ+uY3y/wsTOXECock+LJ+CGhaUrA+hvbiJplv2g0PGEQO+cebPeeWNu3L6vuRCoVPye8M0PsCiRYtYtGgRzc3NWJbF6NGjB7pKwxLpD4kdSWcY5O4P3Yr7Q5D9h58q4suN9Xz8EU34I9FpSfOHT8NSQY9aKEaKPzpCGGVe0DX09jAixR+WqWBowAg/2u5wjz9CBkaZDyrJ8gddQURHjNDsWohGUYMxhKZglfmo+GsDpY+GiR03juAUP5ZPQQsLSj4OobyymaYfzQZKiALnfvmnvPzWrwr5Vtgi/VF8f8jOSh+z4ncrUTUVy5RZiPcm8knQlbwuNTuwW5OUGaMldQRKSzlvqj0ZiQEC0N15SK1DahLIWJmOpzO+mVARibqlzK7YlTE9oMYS9TYFakfPZvbu2RQBeltid83OeGQVkUhHrHUZaF09GxiDB9RS8s4O1IiF3tkBgGftDgC23XwQwfGa6wLV0MQyOH6/ngMWdBxT41ygAESBU/J7i2wAmpqa+Oijj1AUBUVRqKkp7mcs6R3pj70TN3+4lUl1h4L9H3C9+SNJpj980Ls/AjqeUIY/Ijb+CKf4QwfVSPFHZ4o/krMpArSOSLx+zfEEkt3+CBpowR5/hGaNJvBBU7o/3ov7Y/v3D6Brih7v/TjQNdUDp8zuOWAJOo4e5Xh9IUh/FN8fcs9KH7NjY5MUzTDErslxCi/phlsZxXLfxG9XTnWLee+AWeocnliJ2KxLS2I5fA4uScJi5Wr+rZICsYriNmWC+IqBvF9FrcXA0N7eznnnnUd9fT3HHHMMRx99NPX19Zx77rm0tbUNdPWGFdIfkiROG+jdKLo/8g0LDFhu/oi6/Gw7+EN4ndv6WKXq2lGxrwQYle5hnPNF+qP4/pCdlT5GlWEnJYMJJf/m0LVIIQNBbvcrdCNxkVv54ZzU65JLLuH111/n6aefprW1lba2Nv72t7/x5ptvsmjRooGu3rBC+kMiycClrS9Ab73esxCkP4rvD7kMrI+pnzaGda99jOmWBVwy5OitSbEbBVMdjrvhVkaooLr8WNmVMzUVzSn+i0M4GbUz7FgHy6ejdtnHqBeaAqbIKqdFnWdjPK2C2AjyG0YR4N3tMsMjyYunn36a5557jqOOOqr72EknncSvf/1rTj755AGs2fBD+mPvpJA/SQeHP5SsZWbdOPhD63Dxh19DDebnD8XNH7tNIqPynF2xwNNS3Dwrw5m+8occtuljTr7w81I0eykC+wEZE4cp9JRymTiFlkx9Vtb9XKbxheqQ0bjLcKy3UWmfTVhLvCG7MorhLI7o2BLb4773dhKr8trer/KV3fm3SipU/LMxz0LuDOc4+SNHjqSysjLreGVlJSNGjBiAGg1fpD/2XpzaYSeSLYtt253DszIpyB9BF3+U2ftDTaxFs/VHzPln29Ef65z9UfHK7vyXgWkKFS805VemF6Q/iu8P2VnpY2YfNYPxM8ai5PsLJBnUOMW0h57oLPl0ZDwu1jIyQ3+l1AGH5yTXI9udEx7Fvg6tUWcRVdknftTbYwivfeIurTVmWz8ViE6ttv0gSv61Ca1T5J4U0gJfk8VLb9+TY4EcbzuM4+R///vf55prrqGhoaH7WGNjI9/5znf4wQ9+MIA1G35If+yduPnDDSd/uC2RKbo/dAd/dBTgj47C/BGbNMLeHy9tRW+3ck8KaQn82w1efO/e3K7PEemP4vtDdlb6GEVR+M9fXSyTeu2FODX2Cs6Ju9w6Mk6Zhj0xlxEwzbkOTkm9SIQitp9d8dmW8ewOYen2UjHKHMq0R4lMLreVlP+9JqL1pfGILymoAmr/sMOh4hlY8VCWo3+3tZcL86egzZGi8C03g4l7772X1157jYkTJzJt2jSmTZvGhAkTePXVV7n//vs5+OCDu1+SvkX6Y+/FrbPghJs/nGZXCvaHwzkMF39UOPtDaA7+KM3fH74PdhKty/aHImDM/yb80VuHxRIoJoz+vfRHMekrf8g9K/3AM0v/iaIoUjh7GZmjY6nrclMTd6UedyvjIT1ZZCbJcJepZVQTDJ+CloiBn3ouM6lXaijlZDjJrDq0RYhVeNHbo1khmDEEQlNQTJFWxtsSJjKmFG9jV1YZ/4YOwhPL8G3qTDuuRSxEl0FkbBn+LZ3dCb0ASl/eRp1Pp/GcUQiF7CGVxHWKIaj7dQOtJ45x+MQKZzjHyT/99NMHugqSFKQ/9k7cXFBImaRznPaU5O0PMx7eOLlcLM0fKftd0vzRHsEo96J1ZPtDmAIlsQ8lzR+7w0RGl+BtCubuj6iF6DSI1pfh25ruj8Dr26n3aTScX4NAZC8LS4TdUqNQ98A22o6X/igmfeUP2VnpY5q37eKFR1/pTn4k2btIysMuRGQiTLxth8VyOKcDMR08Rva9AGK6gm6kN/Z6RHQvD8iqgwmmP54MK6sjA+BVUTPCR+rtUWLlXvRgDMXs+blViQsnNiKAvjuUdj9fYxdGqYbwedBbwmn3823qJDyhDD0YQ2+O9NS7NQLtUToPHoOnqRPf1s7uMqX/3ET9xy10nTqB9oPiCSGTaCFB5eowvmc3sf3HM8CEU6dfyt/WP4Bkz7npppsGugqSBNIfezdu/sgaLMooYzcYppGdPwX2wB8WxHwqnoiV/pxEgkk8KmrGvhOtI0qszIsesveHURlAa8vwR1MQo0RD+HX0lkja/Rz90R6BzijBA8agtXTg29rVXabkpS3Uf9pK6JRxtB4SwPKn+KNLUPV6EO/ft7Dt9plgCr447VL++on0RzHoK3/Izkofs/zX/0BBQSBlszeR2flIndJPlYyWcjyWcq1KerJII+VcsqOS2l9J3tNjxP/P1MBS4muLLU1Bj4keIXlVFBH/iVMVgR7uGTWL+ZT4CJSqoIWt7jj30VIdNfEUSyh4E4kfLSBW5UOxBEJR0NsieHbHkz4apV4snxaXpyXwtoahy4yXGVOKpSmJZJFhAps7E2U0oqMCKKqCGjXxbAtS9lZ8c3y0vpxYRfxZRkeQHbfuA4D6m08Z26wjAh6UrhjbDtJoOWkcLJgRf8MqNMyv7e1blhfDeWTsggsu4KKLLuLoo48e6KoMe6Q/9k5680fqOacySbdYkNbZ0FJmPDKz2RfiD2/E6n6O4VPj2eNVBU/E6t4gHwvoca8kNtJ7O1P8UeFHFRZCUdDaI+htCX+UeBHehD+EwNMWhmDCH7UlCCU+XaK3RVz9UfJO3B+x2jKMCj9YFkZXkMYfJ/zx0CeM26EjAjpKMMbWuR52nTQOjpsZf8MqNB5e/KTC0h/F9YfsrPQx773yIZYlo7nsbVjYT7fbSSb5tYb9JjHV4bhOdtbiJJrZc1zPGHX1RNNHwVLroCoqWsQkc4VwMkpY8rmpdVMsBU97+mgXgN4VxVR01E4j7VkqdC8Jy6yD3mUS2beMkreyo3d5t3fQcNU0oqPTPw3rwqlsy7o6nfBEfy9X5IclFJQCxLE3bJDs6OjgxBNPZPz48Vx44YV885vfZOzYsQNdrWGJ9MfeiZs/nFoQp2ViTv5QMl6pFOKP+CyOghazyPSHJ2SkDdKl1U2A3hElEz0YxVQd/LEj6OiP6LRSAm/vyLqfZ0cnjVdNJVyX/metddE0XHelCAhN9LldkTfSH8X3h9xg38cEO0IDXQVJH1BIk1JImVzWMedFL8tJ7J4lNJcamAWM+LrcTngLeLcKmN78i7kxnDdIPvbYY2zbto3/+I//4E9/+hOTJk3ilFNO4c9//jOxmFMYCElfIP2xdzJk/dFLA9c//nC+n1WIP1QlbZlxMZD+KL4/hnRn5Z577mHy5Mn4/X7mzp3LSy+95Hjt448/zgknnEBNTQ0VFRXMmzeP5557Lu2aZcuWoShK1iscDjvctXcqqsuK3FpIBgO9tSl25wsZH83cULmnuIrD4VmqSxIuoRfQhLh8EGpY5P+GLdDCxW3l4+IoJE5+UasxYIwcOZIrr7ySNWvW8MYbbzBt2jTOO+886uvrufrqq1m/fv1AV3GPkf6QDBSFNBOFlCm6P3oJoW33LLckjgX5w6WRjfsjz3dsCbSQ9Ecx6Qt/DNnOyqOPPspVV13FDTfcwJo1a1iwYAGnnHIKmzdvtr3+xRdf5IQTTmD58uWsXr2aY489li9+8YusWbMm7bqKigoaGhrSXn5/4UtMDj15DvZRySVDmeTGRjsKEpGDBNwSSZq9jCLZiiNkOtbP8tk3B2qXc/x8s8Ln+NNtVnlsy3i3tGeFnExSuj57uVmvKFC6Lph/OUmvNDQ08Pzzz/P888+jaRoLFy7k/fffZ9asWfziF78Y6OoVjPSHZCBx84cTbj8FTuM/xfaHGnbxh9c+DpkadPFHqX0iSXDzR4ejP8o+DBf0wZau6+r9OkneFNMfQ7az8vOf/5yLL76YSy65hJkzZ7JkyRLGjx/PvffaJ/dZsmQJ1157LYceeijTp0/ntttuY/r06fz1r39Nu05RFMaMGZP22hNOOP8YdJ/cGrS3kboh0ulcJk4hJQHX5VmWYn9PT7Rn43wmpt++Fqkb/rNwyCasAmapfWYxz7ZO57wx1X77ejcHCe1TbSuciqc35y0bJSY4IlTcPSvDMQPx5s2bEUIQi8V47LHHOPXUU5k4cSJ/+tOfuPrqq2loaOC3v/0tzz//PL///e+55ZZbBrrKBSP9IRlI3PzRW5l8zxXiD8Np4Crxr633DPsZFBWwAg7+SOxtzMcfekuQ8PQRtv4ofy7/nClqFA6PleRdzg3pj+L7Y0h2VqLRKKtXr+bEE09MO37iiSfy6quv5nQPy7Lo6Oiguro67XhnZycTJ05k3LhxnHrqqVkjZ5lEIhHa29vTXqmUVZVyxn8uzKlOkqFF8k/7zIbWTUROibtcRSRwHpnSFdtnpUYAyySa6HhkntMsHBN3KVGXbMJjSmzr7/usA9Ov2t/PjG/vzDznaeyk/L1o7hYXMOL1MHc9fWOOBXK+bcGvocrkyZPZuXMndXV1LFq0iIkTJ/LGG2/w5ptvsnjxYsrLy7uvPemkk6iqqhq4yu4B0h+SwYCTP9xwutZtuVch/vAkIoDZdiJK7P2huviDmIs/Rttnt3fzhxqz94fe1EXlmkjuS8GEoPqVLu5cflNu1+eI9Efx/TEkOyvNzc2YpkltbXq40traWhobs6MM2fGzn/2Mrq4uzjzzzO5jM2bMYNmyZTz11FM88sgj+P1+5s+f77q+7vbbb6eysrL7NX78+Kxrph88Ocd3JhlKaPQunKzG1OVaJ1TA8Djcz+gJapprHXxdMSy/Zl/GtO/kaDGIVQdsz3kbgxgV3qxzKmD5dVuBBdbvInhgbTzUZYapRt/1Pv6tZk4bg8o+iuFd+VEvF+bPcBwZSyYd/MUvfsH27du5++67Oeigg2yvHTFiBBs2bOjH2hUP6Q/JYCAXf2Ti2ilxOF6oP5yOe4MxLF+e/jAgVuXgj6YQRkV2B8jNH77PWgjtP9rWHzX3fkDJRqP3DosQVLwXxfOS9Ecx6Gt/DMnOShIlIyqEECLrmB2PPPIIP/zhD3n00UcZPXp09/EjjjiCc889lwMPPJAFCxbwxz/+kX322Ydf/vKXjve6/vrraWtr635t2bIl7bxpmNxz9bL83pikT+iLUQuNRHKslGdkNvLC5ly+m+09sXjW+dT7JF8KYKX8JjvVIfWlhk3MEj2rjJKoW6ogkuc8LSGiVX7b+2ntUWI1gaz76a1RzBINY4Q3q0zJmkZC+43uLifURB1Mi/qb36bqzQhKLDEsmIyWmfjg1Iig+sUgneEdNF9xIBs3bszzE+2F4Tg0luC8887bo30WQwXpD8lA4+QPt6Yk2UY7YVeuEH+kPifLHxETM5CnP1pDxCqd/BEjNsqfdT83fwTe3kF4ZrY/MAV1P3qH6lUhlKiId1qslH+Jb8Qf9Y9O2o0mmq+YI/1RRPrKH0NyMeyoUaPQNC1rFKypqSlrtCyTRx99lIsvvpg//elPHH/88a7XqqrKoYce6joy5vP58PmcY3S//vRb7G5sdX2OpP9INqjFJPlLlMwonCTZcCcz2Wc+t7vhtzmevIGS0njpRnykyTQNtFjG/URCErqCZqS3eIZXQRMCjPT7qeH4orRYiY4e7FmgpgIxn44ejM/AqOEenXq6oph+BSvgR+8IxzMUC+IBAoz4Z2DVlaG3BOOjbIqCoqoIRcOo9mGV6mgdMTCteEKwUBi1KURo5igsj4IaNuLSCXgo+csmlBUm+kFjCU0pwfIqqFGLknVdNLfuouXCqUAJBnDO+Xfy6ou/yP7mSPLiwQcfpKyszPWaK664op9q0zdIf0gGE07+SB6z2+uYmlA4s0zq7EuqI3QjftzwkLM/FOL+UC2BYmb4I5KDP3waaqTHH3owiulTEAE/emcETCshQQXFUOP+GFOCvjuc4o/4jIOdP4jE/RHeZxSWF5SIAYoS98fftiJeMPDsV094aiDuj4igZF0XTZ0tNF80DSgnBpx7zi94+dU7bT5pST70pT+GZGfF6/Uyd+5cVqxYwRlnnNF9fMWKFXzpS19yLPfII49w0UUX8cgjj/CFL3yh1+cIIVi7di37779/wXX9x/++iKqpWKZM7DWQ2DXee1LOIPuXx04qyWMKYOkqqmGl3S9zf0taAi/R80/ymBo2ekSlxBNuAWjJGyREk/xSBdRoj2FiJTqehFiUxI9k8uvUOniDiXjoYRPTq6JGrXjdYhZqDAgnshBXedFao6iWQE1ktqchnm1YKKAIgdqWkhCsJULwwFpK3t4BGHh2xaN/6euaAdh6y0GExmkuc74anTOqgKqeQxZ0zKt2uL5ACp2SH8LT+AD33XcfmuYcCkJRlCHfWZH+kAxG3PwB6e19kswN75leSZ5LPe6JZd8vZ38EdDyhPPwRsfEHQCThj0ovWlsUxRLorYmIjo3xf3v8kfKz3xIhNHs0gfeaSPPHx3F/bLvxAIKTdHAJsdy+vw9I8YUp6DhC+qMY9KU/hmRnBeCaa67hvPPO45BDDmHevHk88MADbN68mcWLFwPx6fVt27bxu9/9DoiL5vzzz+fOO+/kiCOO6B5VCwQCVFZWAnDzzTdzxBFHMH36dNrb27nrrrtYu3Ytd999d8H13Lm1RYpmkFBoM+CUNThfRAE/B3YjZz03dJ8psq13AXUwS71oUftcEUrU5X4OU9pCd/5OGOVq/h+uCkZZcVe0Fpqga6jHyX/zzTfTljbtrUh/SIYahbQtg8Eflt/FHw4RKAEXfzi39UaF6tpRsUVT4t4pItIfxWfIdlbOOussdu3axS233EJDQwOzZ89m+fLlTJw4EYjHd06NmX///fdjGAaXX345l19+effxb37zmyxbtgyA1tZWLr30UhobG6msrGTOnDm8+OKLHHbYYQXXU/e4BqyVDFEKalMUZeBboxzW5GcVcatysQeCrAIW6omeUb5iUehmx6G8QTKX/Rp7C9IfkiFHsTM8FlSHwdBGOH8IBXlAiMIyNrveUvqj2AzZzgrAZZddxmWXXWZ7LimQJC+88EKv9/vFL35R9ERn4/et54NVH2EacnRsoCl0v0qx9rmoQjjey+l4avKwrD0vGqjOyYFt72loKhoOhRxkqLWFHetnBXTULvuAzEJXwBBZ5ZSQUwBn8O6yiI3Ic3ZFgKfF+Z4FIWzCzORabogiBroj3c9If0iGEqrLTEh/+cMsxB/tLv4o0VGD+flDDTlX2rvTJDIqz9kVAd5d0h97Sl/7Y0hHAxsKLFx0vBTNIKCQJFzJ6+2aDw3n4B1Oz0jOUNiWcWmjnO6nms51sDT7evu6Yo5lYpX22YTVxAZ6uzKqS8cjOq7U9rj/g2ZiI/2296t8cVdBy8Aqn2/Is5Akk5tuuqnXzZGS/kX6Q5LEzWH95Q9v0MUf5Q7+AHDyR3Jviw1O/vB+1ExspM/eHy/tyn8ZmKpQ8ffcQpZLnOlrf8jOSh+z76HTmHrQJFRNftRDkeRgkVOnxK5ZdGsq7aKCQc8GR7vnmF77c26rAlxF5FXtRdQadSxjVNkn7tK6DCy/ZltGbw7b1lsFIpOqbOsdeHkrelse0/IW+LZbvLjuvhwL5EZyzXEhr6HKTTfdRElJcTM5S/YM6Q9JKoPCHx4Hf7Q7+yNW4eCPoInlkPjRzR+x8VW29fa/th1Pi9UdorhXLEFgs8HKj+/P7fockf4oPrIF7Acuv/NChCVHxwaaQpb8ppZxauxtG3SH+yVnZOywFPvneKI9HSOnsnZ1M72K/bmo5VjvaKXPtoy3JYjptW8uROJ4Zhm90yA8tcL2XMmaHUQmV2SNCKpA7bLt8QK9/cpYoBiC2oc29nJhAYg9eEkkRUT6Y/jh1Iz0qz8cstsrMWd/xMod/NEaxHLYfyU89gkm3fzhf6eJyKTyLH8owJhl2+J7V3rrsFgCNQqjf7PJ/bpCkP4oOrKz0g/86ad/HSQb0yTJnCf5tAup635Ty9jlTbErY4fd/TQBppp9TfJ+dtmOVcBK2XmWek6PCqyUzMUipUzqdan/72uLYKR0WNLeb9Tq7rCkPac9RmxkekLIJIFP222Fo1oCT2OQ8JTK+LmU912yppH6BxpQDGH/jUoc08KC+js30/LV7Kzfe8pwzEAsGZxIfww/evtu94s/DIGV0mHJxR/ejghGhb0/iJndHZa053TEiFWnJ4RM4uqPHSEik7P94X+nibH3bY+H2Lebrkgc04KC+l9souUr4yg20h/FR3ZW+piGDTtY9bc3EblOS0r6nNQf+sxG16kTo2ZckyQ5WmWmnE/FbitgMpeKaTMSplnOkwoKEPVkj3ZpiWRfSVGlNeoxsHzZ9UrKywjYyKMtgulXEB41q4watYhWl2TtYfHsCmFpEK0NZJnW/2k7kVo/0fpAer1DBp5P2+jafzThqZVpo2SBtxoZc+0HjPxnF3pneu09rYJRz3RQseRDtn1nIp37eDhx8iKGKvfccw+TJ0/G7/czd+5cXnrpJdfrV65cydy5c/H7/UyZMoX77nNeAveHP/wBRVE4/fTTXe/51FNPEYs5rx+XDAzSH8MbJw8o2Dun6P4wBKbXxR9+G3+0RzC99v5QYibRqpKsPSyeljCWCtHR+flD/6yN0H6jCU9O94fv7SZqr32fmuc60dvTa+9tsRj9VDsVd33I1usn0THLy0mTLmGoMlz8MaSjgQ0Flj/wd1RVJvUaaDI7GKmbFQ3SkzfalVNSXgKIAolJi6zOj5lyXEs5nnyOlfg3dZ2xoamoCBRTpN3PVBKZjZW4iHyxeCELCHsVVAGaItCjdBvK1MFQVRQEnqhAi/Q8J+IDFQUNgRoBJRFZxdTB8MXfkR6OoYfj3TILiJZ7UVQFSwi87VG8LfGkXZYCkUo/qqqghaJoIRNtRyLZV4UXq8SDMC08O0P4d4S7y4Rq/GgeFbXDwNMeRX+3KV6HMi/hEQEwLRqna1gXTo1XfO1W6v/cheL3YIWiNFxaRvPCcbBwZvc3p/m4OopOP/x9+Oijj3LVVVdxzz33MH/+fO6//35OOeUUPvjgAyZMmJB1/YYNG1i4cCGLFi3i4Ycf5pVXXuGyyy6jpqaGr3zlK2nXbtq0iW9/+9ssWLCg13qcccYZNDY2UlNTg6ZpNDQ0DIt8K4Md6Y/BgduMei7lsqIy4vzHV2qZ1EGy5H4VgY1b1HggFEVQdH/o0Z7nhH0KKgqqsNCioIQT/tAy/BFN8UeJF0WLh+3XO6N4W1P8UeFHQ0WNRNDCJlpTwh/lnrg/DAvPrnCaP8I1ftRUf7yf8Eeph2hlAGFaNMzQ0/3xx05UvwczFKVhcTlNXxoHX6rs/px2HVPr8N3YA6Q/ivY+QXZW+px1b6yXohkEJBv4TFKFkGtyLIX4L47T5nrVoYyixIWSWQ8F0Mz0zPbd50S8U5SZ60QFfFGRnvE+gWaAimVfb0VDC2fP92gGqEYsqw4qoCoKnrZIVhlVgCdmoHYZWc/S26OI9mh22EkBYlwVvreyo69onVGab9qPaE3GJ3TQOLYflHV5GuFxPvcL8qS/4uT//Oc/5+KLL+aSS+Ije0uWLOG5557j3nvv5fbbb8+6/r777mPChAksWbIEgJkzZ/Lmm2/y05/+NE02pmnyjW98g5tvvpmXXnqJ1tZW13rU1NTw2muv8cUvfhEhxLDKuTKYkf4YPBT6G2FXrrc/vOzabqGkZJrPOKdZDuWK7A+PUNGiNv4weyJ7ZfpD0xX0dgd/GA7+6IghOmK2/qCuCt/bNv7oirHzB/sRGZOxN6Y3fyjSH0PBH3IZWB8T7sr+JZUMLvrrz7KCMtpS5Pr1spzE7lnCLRJRIctT3MJsegp4twqIYg+75LoZ0u4FtLe3p70ikex2IBqNsnr1ak488cS04yeeeCKvvvqqbbVWrVqVdf1JJ53Em2++mTYNf8stt1BTU8PFF1+c09tdvHgxX/rSl9A0DUVRGDNmDJqm2b4k/Yf0x+BgMCzCGxT+6KUStv5Qi+sP1zDNHudzjigKViHecUP6o+j+kDMrfUxVTQWKogy7hGuDjeT0uW1j6nB8T7C7p1pAODK3euNy3LFeuhJfw+Z03uaeasw5l4rwqOCSpMsWl4FiNSSgUsnvjVmghov9+2U35phrORg/Pn3T/0033cQPf/jDtGPNzc2YpkltbfoShNraWhob7eP+NzY22l5vGAbNzc3U1dXxyiuvsHTpUtauXZtzrX/4wx9y9tln88knn3Daaafxm9/8hqqqqpzLS/oG6Y/BQW/tcL70dq9B649ecpjk7Q9dxX5npzOKSwdHCyU21Oczsm+JeLmiIv1RbH/IzkofM++Lh/D68rcGuhrDnmSj7dRw5yuOGOA0cez0jGQGYrvnmQroNu1lMvKK3T0tr4ISzc7w61YPNWg61s/y2y8R0ztjzvUu86G322+sM6q96C3ZS8F8n+3G0hVUI/sNl60LERljnwzMERXK3u3Ir0wfs2XLFioqKrq/9vmclxlkTpn3No1ud33yeEdHB+eeey6//vWvGTVqVF51njFjBjNmzOCmm27ia1/7msy5MgiQ/hgc9OYPN5z84c3jeujFH9j/MVd0f0Sc/SF8GmrEZolYl4s/Sj3oHfn5w7OxFUtTUE0bf7wfIjQ2z8SEqiL9weD3h+ys9DGfP+co7vuv38rp/EFAcmN7JqmhGLMadIcyhaxwVXBP6uXYoGug2ww+6VHhKCLTp6BFshtzN3kpYWcRmeVe9I7sKRnf9s74tLyw2URa4cXTkl1Gb4vQtd9IStbtisfDT6HiqY3sOmY/+w/dATUi+Pr0fXIvkAspU/J5lwMqKirSZGPHqFGj0DQtaxSsqakpa/QryZgxY2yv13WdkSNH8v7777Nx40a++MUvdp+3Ejk6dF3no48+YurUqa71uummmwDYuXMnH330EYqisM8++1BTU+NaTlJ8pD8GD04u6A279tSpo+J0ffK4oz8YeH/g0pGxSr1oXTb+aAzm74/2CMGZIwl8lO2P8uWb2Hl8nv4ICc6eNSP3Arkg/VF0f8g9K31MoCzA2d89Y6CrISG9oc3ELgY99EjAroxToL7ecqzYPcdNRG7ZhA2bUJQAekI0tvUu0W3PqZAVbrL7XKdLdmKHmZDAxk6MUt22jNZlZoU/BtBbw1StDufV0I94qYurfprb2tqc2cM1x7ng9XqZO3cuK1asSDu+YsUKjjzySNsy8+bNy7r++eef55BDDsHj8TBjxgzeffdd1q5d2/067bTTOPbYY1m7dm3W8gI7gsEgF110EfX19Rx99NEsWLCA+vp6Lr74YoLBYO5vULLHSH8MHtz84UTqjEwmToufclntNZD+MP0u/tDt/aEE3fxhPwLv5g/dwR9aW4Tq10J5bfAZ9a8OrpT+GPT+kJ2VfqC6rmqgqyAhXQJOnRK7czgc95CbcFLLpa7QtauDU+Iu05N9HMATE1g28fHd6u0NGpgOHRYS2Ymz6iDAqPLb36+hC2Ok36GtFQivmnXcv7GV8KzRCF3J2jA58t73KV1vuDfcyRGotyJob29wubBAhFL4Kw+uueYaHnzwQR566CHWrVvH1VdfzebNm1m8eDEA119/Peeff3739YsXL2bTpk1cc801rFu3joceeoilS5fy7W9/GwC/38/s2bPTXlVVVZSXlzN79my8Xrcx3ThXX301K1eu5KmnnqK1tZXW1laefPJJVq5cyX/913/l9f4ke470x+DAzR+54Nbe5/qsweAPPWw4dlgwXPxR6eSPIEa1Ly9/eDe3Etm3xtYf1Q+uo/yDmHuHJXGu6vUQ6jufOV9XKNIfRfeHXAbWx8SiMR687n8HuhqSBMlkVnajV8nRqdTY9qT8v5Lxb/J+MXpyrqTeS5A+2pUsoxGPF5+6Bjm1nGlTBy3mXEa1wNQVNCN7NCz1/1PrrQUNYqU6ni4j6xoLwKuiRq20e+itYcwKP1pHOMsD+q4wsbpSPA1d6fXuMjG9ClatH08iVn6SwDs7iOw7EiJR/Bs7ElmIFRSg9va3aT1/Jq1HBLACSrrVVdC6BNUrO9i5Xztd/282z//vSk78xjEUC7vEx7mWy4ezzjqLXbt2ccstt9DQ0MDs2bNZvnw5EydOBKChoYHNmzd3Xz958mSWL1/O1Vdfzd133019fT133XVXVoz8PeGxxx7jz3/+M5/73Oe6jy1cuJBAIMCZZ57JvffeW7RnSdyR/hhcpM6a2/2qO+2HTC2T+nWmczLLDFp/hA1iAR1PyMEfHhU1luGPtjBmmR+ty8YfLRFiY0rwNAbT6+3iD//7TUSmjYRoFP/mHn8AjP7pu/i/vg8t80sxS1VI3d+iKegdgup/ttF0UAf6ZQdIfwwBf8jOSh/zyhNv0L5rcG3eGu4kG/Lk37+pgknKxG5K3elcshFPJvpKPRclvr8lVSDdN4Os9boi5ZihpMfVVxKiESooln29Yx4VPZY+Xie8oETBSpRLkoyXHy314AnGumPxKwoYuooStTDLdLQuI+UBBkKAVeZFMUyUqIWCiIc3DsfXQEfHlqG3hlEMKz5VX+pBa49hBTRiY0pQO2MolkB4VEQ0im9jB+FJVZgjfCgxMz7A5NXxPr8Rq7WC2kgJ4X3LsLwKatgi8F4HjQeG2fmFcUAFJnDTb/5aVNn0J5dddhmXXXaZ7blly5ZlHTvmmGN4663cN13b3cONYDBou+Z59OjRchlYPyP9MfhIa6tJb4dN7LdLpJZxckvmnhiFnoGwgvyReb9i+8OI+8MIeNDC6f6wNBURs7ACOmq4xx+KkfBHqRfVMFFiVvwvdE2FSPyyaF0JelsUxUzxR5uDP8wovs0dRCdUYYzwQswERUF4NPR/bMbsqmRMl5/I9FIsn4IaFgTeb2f73BhNp48DqjCAH/76L9IfDgwWf8jOSh/zwqOvxDMQWzKxV3+QOmLV2/HUTouSeOkZ50TKuSTJ0aukCJJlPCllktf5E/+vEY/4lex8JMsmG/jUmRo18YWeci6tDokfpeRIGYAnMSrmyRjNUqE7VLFigVGiowfjoSS1RDZjX1csvYyILxUDUDsNzICGGjJRBOih5PH4TY1KL1pbFNWy8O6ON0S+bZ3x+yXqp6WENtY2dNB1YC2lb+8gFf/GVtgIW340h/BYNfFmxwKQfiW0zc0Ib2BBx2GVFBX7NQm5lRvizJs3j5tuuonf/e53+P3xn+BQKMTNN9/MvHnzBrh2wwvpj/4lH38k2+RUf/S2rzt5XWonJ/lvakb65HFvyrlU5+Tkj5T7Fc0ffg09ETEy6Q9PKNsfajgRrjhkYPo11HDcH1riuJrYbG9UxAeyFCPFHw2J2RUHfwT3H03Ju02k4t3cinczbPvhAQQn6Ik4z+MAyAziu/uIQPoBS9B5aBVFRfqj6P6QnZU+ZldDqxTNIEDgvEErv1WiuZWxO2+J/KPJ2C0R6EZkj+z1Vgell2zYdmVMvxctFLK/PuZyP6eGV3f+9MxSJf9viApmSZG33xWwfri73BDnzjvv5OSTT2bcuHEceOCBKIrC2rVr8fv9PPfccwNdvWGF9MfgIZ92tjfc2m2nexYSjaz4/nD/a9q23n4vWriI/nBJUmyUqYmOSh6oCob0R9HoK3/Izkof4/UVklJV0p/0Jg6nMvmi2G2UyfFZRWvCekmWZfcs1w5OoaZ2up1ZwLsVoOSZl7I3FNEzaplvuaHO7NmzWb9+PQ8//DAffvghQgjOPvtsvvGNbxAIBHq/gaRoSH8Mfgppn/tigMyJAfeHW5b6fDsWvVCQB4SQ/igifeUP2VnpYybvP4H3X/0Q05CjY/2BU8OcOfWeWcZtXMWuXG9til0ZzWUky+m4W1x7ocXDUuZTB1NVUTHt5ZWMPpBZ7/aIY/2sEk98T4vd8z0KxLKTjmldTkGfwbvTJFqt5henUIC3yfmeBTGMp/EBAoEAixYtGuhqDHukP/oXtz/s852FKHY9NIfjTtdD8f1hqQqqw7OcYi6r7WFnfwT0vP2hdjq39b4dBpGaPGdXBPikP4pKX/hDhi7uYxYuOl6Kph9xk4lT++E24eEUu153uZ/hUCZ1E2Q+OF3vFj/f0hXbOniD6euLU4mW24cljMfPz45pD6C2hRzrFxlnn0nY+9EuorUB2xnvyn/uzL9VUqDy2W15FpJIBj/SH/1Lsf3RW2h727bboR6DwR+exJ4TuzKxMmd/oDn5I5y/Pz7ZRXS0vT8qXtiZ/2yNAhXPS38MdmRnpY+ZcsBEZs3bB9VlnaWkeLg16MLhnFuyr4jLOSepeFzKmA5l7MIldz/Ha3/OTZKqIZzr7dNs6+BrizrWwajw2xwFLQpWiX3iLm9jp+39VCA6rtL2Qf5/N+BpEc4JCDKxILDFZOWnv86xQI70U5x8icQN6Y/+xc0fTufcOh69rS6yay3cEhEPCn947P3h7XDxR5n98h8tVpg/jHoHf7y1A+9OM75BNBcsQckGgxc2Ls3t+lyR/ig6sgXsBy797/MQuf7ySIpG5ieeOo3u9N3IPF6CcxIut05OcmI785yO89/hTp0pPRqP3GJ3zk2ghk+1PxcxHesdrfDalvG2hLACDqtGE8Hhsz7vkCAytcL2XMnqRkL7VGW1zSpQ98Dm+N6V3josFqgRQe39n/ZyYQGIPXhJJEVE+mNgcPrj3s4fTu2wlx4XOD0jn4G1QeGPmLM/jDJ7f3jaglh+B39Y+fvDv7aR8PRsfyjAmAc2oxj03mGxBFpIMPqBvkgKuQcviS2ys9IP/PaHf0SRn3S/4ba/JHVbRmrboGZcl/r/OukdluQ5t+d4iMfItzunJOqQ2Tap0J1NOLOMJtKFk1MdIhamR8kq4/Zefe1RopVe23NayMBMjIKlHzcxKzzdlUk95/+0nXCKcFLr4N/QRnBmdfxcyvv2fdjMuDu3oIUTV2faOfG13iGo++/PaLpoKkVHykYySJD+6F8K8YdbGR37Dkuuz8k85+oPxb6Mmz+cXFCIPzydUWLlDv4IG1h+G3+ETczy/P3h29BGeF8bf6xvYfwvNqEFk/7IaJQTX3vaBGPu+JTmCydTdKQ/io7cYN/HbP5wG2v+8e5AV2PYkWyEU7PVKynnkkm2lIwySQlkbmbUgE6gFPtp+Bg9+1iS51OXg2U+B+JJH3WRfl61euqtZpRVBcQ8oMfsRWd64tPqafWOCUwl8V5TGsLk+WipB09XLK2Mty0a/xwCOlooXbNq0CBS4cMbikHM6nlOewwLMOpL8ezoArPnfv5P2wlXeFBG+PBt6ey2sBIT+Da3E9xnJMIDgY9aUJPx/jfsZvQNuzA+N4W2+eVER/XYyLfDourFVnZ4g2y7aSoIOHbyxfxrQxGn8gsVx14km2g0SlNTU1bo3AkTJgxQjYYf0h8DQ+bSrkx/pOZWScWp7daAMPGZltR+Z/K8aVPGaXN8tz/Idk4yQ30+/ki+V9OjoMVEfv4IePCE0v3h6Yj7A7/ek28lWS5sEK3w4wlGwUjxR0fCH3WleJpy9Ich8GxtJzStGuFR8K/v8Ye+uY3aG97GXDCJ1qMqiIzpCfoc2GZSubKVxtIw226ZDkLw+YkX8c9ND1E0pD+K7g/ZWeljlj+wAlVXseQmyaLhNJoF6RJJlYmVUk4lPWGWQU/HJTWOvUXP6JYGlKeca08cU4gvFUtNCJlcV6yQLqbk/ZLlPCK9TLJuqWUE8WSSiogf96ZM10QATQfVSJSLpZRJZB1W6Eku2V3GG99rogDeRGQuK1EGQI0mPodER8UArFINvcuMz4i0R7rLRLwqukdD7YqhAdr2ru5zsQoPajCGbkCgPQbt8WcFy8CjxpOB6Z0G+se7usu01foIj9RpP6sUJo7vqfgjG6h9L8iOQyqJnDGOHWdVA9Xdb7j1pJRrJXvE+vXrueiii3j11VfTjgshUBQF0yxynE+JI9If/UumP5JkzkakJnBMzRSf2XZbKcdTd/1F6elIeDPulyxj54/kcYVs5ySfk+WPlHu5+0P0vCcPKLGEK1P8EQOUVH+EUvyRkKpqJN5PuMcflGiowbg/fO3h7jLRhD+UpD8aUvxRnvCHme6PUAnoeoo/PmnpLtM22kd4pEb72WXp/vi/DdS+18WOQ6sInTGO0Lmj0j6ktuPHIikOfeUP2VnpY9av2SBFU2QEhSV4tCuTbMTtzmWOTKVSin2CruT9bDtSSnrDn1km+f+pCOIjYnb38wCKkX1OId7hsCuj+zW0cHZjoRLv3NjWodKHty1iW8bjUVG7YtnhJQFPe/ZxALHPGDxvZeYVjpdp/c6stFmUbr4+OSuTfTcKRMYUOR/FME7qdcEFF6DrOn/729+oq6tD6SW3gqTvkP7oX5x+0t2OO7X3Ssa/qXgKKGNh/wdbr/5wq4ODPzSb2XsA1aehRRz8YTjUvdyH3mHvD93NHx0O/pg+Bs/b9v5o+6+ZRGptzHxOL/6ok/4oFn3ljz7rrKxevZq5c+f21e2HDNFwkeN3SwrCbXa1kF+lQsqIAqd4i9p89VIJWzmoLgvmC3lPLm9IaAW8WwVEkVuy4ZzUa+3ataxevZoZM2YMWB2kP+JIfwx+CnJBAeUKbVqK6o9eNq3b+sMtkl0hgSNc/ZH/7VAULL24nQTpj+L7o8+27Z1xxhl9detu7rnnHiZPnozf72fu3Lm89NJLrtevXLmSuXPn4vf7mTJlCvfdd1/WNY899hizZs3C5/Mxa9YsnnjiiT2qY9XoSpQiZ2kd7hTyaWZuCkylL5aW2p0v5MfArd7g/lnYleutM2BXRotEna/XC2hCXASlhUT+3xCLns2UxWIYb5CcNWsWzc3NA1oH6Y840h+DnyKP1zjes5A/1oruj15+Fm3rHXHucAtP/u9KMV38ERT5jwpaAj1Y5NlL6Y+i33ePxiPPPPNM2+NCCFpaWvbk1r3y6KOPctVVV3HPPfcwf/587r//fk455RQ++OAD2w08GzZsYOHChSxatIiHH36YV155hcsuu4yamhq+8pWvALBq1SrOOussbr31Vs444wyeeOIJzjzzTF5++WUOP/zwgup59FeO4LW/vrlH71WSjdO+leSekExSN0xmlokBvl6eZTdN7TQ65rRMLZnBPlmfVEyHJWJudcAHIuJcB9tp/KDpXIcSHT2YHbdGcyljlXnR2+07M7EaP56d4awy/vUtWB4VNZYtiLJ3g4Tr7JOBOaJC2Zq2/MpIHLnjjju49tprue2229h///3xeNKXSFRUVBTlOdIfvSP90b+4LTF2ohB/9Hbc1h8U1x9Wyr7GXOumRlxc4NfRwjb+CBnOZUo86O32nRknf3g/c/ZH+dtdhMbn2T6pCqVr2vMrI3Gkr/yhCFHo4hSorq7m97//PWVl6X9cCCE466yz2LHDcZXgHnP44Ydz8MEHc++993YfmzlzJqeffjq333571vXf/e53eeqpp1i3bl33scWLF/P222+zatUqAM466yza29t55plnuq85+eSTGTFiBI888khO9Wpvb6eyspK2tjYqKiqIhqOcWbeIrrZgoW9VYoNbNJYkWQ069h0ZN5JRvpwadLvnONUtec5Jhk73MzTQHfakOZbxgebQkbESG/ad6m17vwrnTolI9AQzy4THl+Hf0mlbpmv/UZS834yS4Ruz3MuGX8xG5DotL+KzMT8xZ/O1y78IZP8O5kOy7IQ7foQasE+E6YYVCrP5u98v6NmDBTWx7C9zrXGxN9hLf2Qj/TGwuPmjEOyiTmY+y8lhe5M/zDIvWmfx/BHcbxSBdTb+KEv4w5OrPwRaUPATa3/pjyLRV/7Yo5mVz33uc5SVlXHMMcdknZszZ86e3NqVaDTK6tWrue6669KOn3jiiVkRCJKsWrWKE088Me3YSSedxNKlS4nFYng8HlatWsXVV1+ddc2SJUsc6xKJRIhEejaPtben99C9fi/n33wm9161LId3JsmV5K9B5giQgnOnJHVK3G52xW6LnVP4YbAPD5l8jlN2++QMj10dnESkmi4jal4VLWpl1UGPONfb9KvooewyKmD6VLRI9oiV2h51rEO0rgzv9s6s+/m3dGJUetDasjdK6i0RhEeDqJm2TlfriDJiVZiWBfYZj7NQYOTfO/jan7+Y2/U5olDgmuOi1mJg+Ne//tUvz5H+kP4YbCi4dwiccPJHajZ6p7bBzmH95Y+YV8WTpz8sb9wRtv5I+Cir3p0u/hhTirehKz9/7HbwR2eUkS920XxcjrPzikLNs6187Unpj2LRV/7Yo87K448/7nju2Wef3ZNbu9Lc3IxpmtTW1qYdr62tpbExO0oEQGNjo+31hmHQ3NxMXV2d4zVO9wS4/fbbufnmm13rK1zWWEoKI7XRzmzkNOw7EqkSyCzjwbmTI7CXhIZ9fpXU+mU+SyE+ZZ+Mh29X79R7kHgfTmX0qIWp9QgpFyF6QhbREg+eYCy7DhELS1NQzfR4+ypgVPjR2sNZZXzbO4nWBvDsCGV/DsEYViJsZep78m3rIDhrFP5PW+L5WlLe+IilHxCr2J+OA73ObyrxDa5+KVTYpsreGMbRXOw6D32B9If0x2AkOdjkttQ3Ezd/JMn0h5vDXP3hULdC/OEtwB96xLLNrwKgRV38UeZH67TxR0MX0dEBPE25+8O7vYPwvqPwbsz2R9XvPyJWuR9th/jj+1fsolFZAlSFkf/sxPT3gUCkP4pOzgMIHR0dfPvb32bGjBmMGjWKqVOnsnDhQn784x/z4Ycf9knlesNpmimf6zOP53vP66+/nra2tu7Xli1b0s5HQhF+d/Mf3d+IxJbe9pulflcy96glG/XM85mCSD2nYp9pODU5V+b9dLIzGqdel5mFWNAjDad6u5VJvX/3ORMMn+Jch5QPKnnOE4wRK/HY18EUGAE9q4zWHsYKaAhdySrj2REiWl/W3aJ01y0GSsQkOr60+xuWPBf4oJloXQWh6SO623ahKiiqQs0v3qXmr+3o7aLngzR7Phxvi0XtH1poOSrArs+X8+CtuS2zyZnMN5jPay+gtbWVn/3sZ1xyySUsWrSIX/ziF7S1tRV8P+kPe6Q/Bid2GeR7+/XWcM9Ub+cPbI716o+Ua4rmD2+e/gjFMPy6fR1Mgem38UdnGMuvITQbfzSFiNaV5uUP30fNGLUVhKel+wNVYdSv3qf28d14dif9IcAU3cFdfDst6n7fzK5jy6Q/+oBi+wPymFk5//zzWbNmDf/v//0/ampqCAaDXHvttWzatIkbb7yRU089lXvvvZf6+vo9qlAujBo1Ck3Tskasmpqaska2kowZM8b2el3XGTlypOs1TvcE8Pl8+HzO2+tW/nEVwfaQ6/uRFE6qCETG18m1wjF6EjFmXmdhL5MIPbHwk2WMxLHUPSmCnhG11KRdmfdLTTwp6AkEIGzKKEp8CjmmgWam1yEpqtR1wwKwrEQHzQdKylpjofRMR5u6gmKIntE6w0SB/8/em8fLUZTr40/1NjNnzUlOdgJJAAEhCgTB4EUUkUW8fN0uoP5wA7yIyiYuiF4Bl1wQEZWLC4uIcgU3vKIYCLtoECFhj4CQkJDkZDnLnHNm661+f/T0THV3Vc1Mn5mz5PTz+Uwyp6reqrd7Zt6n31reF3aGQCkBpGzIqUpAqFdndRtQRy0Ql4IqBFRTQAoOXBVARgcpOQCloLoKZdQCcb1lfaVog1gOqELgthvQN+dACVDcpxNKwQEcCmooILaFto05mLPbYO3ZBThu+QYQFF7aBvsNw5h/m4bS67vhpgiUoov0ukFsPW8htp/mJYV0MwQ3rFmHM/FBJBg7Hn/8cRx//PHIZDI4/PDDQSnF1VdfjW9+85u45557cOihhzbcZ8IffCT8MXnhz+SynMG+57mg/gOViD8MBJMG+1yggc9h/vuG+KP8shRAdevkj/J7yyBQTRrgDwj4A65b5hwCYjH8oRDAtT1u6TSg5Mv8QQigKkDRAVUBmtJBTAcABdVUqDnb44+5bR5/2G6ZP3Qxf7gWMi/mYM3KwFrUVSZCAkBBbuMOWCM5LPilgtIBXaApBaToIP3UELZcuAjbTvcSQyb80Vy0gj+ABpyVe+65B3/9619x8MEHV8ouueQS3HnnndA0Dd/85jfxpje9CY888giWLFkSS5l6YRgGli9fjtWrVwdCXK5evRr/7//9P67MihUrcOeddwbK7rnnHhx22GGVaAUrVqzA6tWrA/uO77nnHhx55JGxdf3L7/4OohDQOPHEE3DBW25njT8QXDLU4ZFHClGC8WeiCILJIf3HB8qUsWOyByeN0Jg8HTRUl+IJUxd2bBTmD92J6lCRoUGnKVXOPqyWQv0xXzvVprDbVGh5zxFRynuLtQINyji0siylZ0045WV44lAoI1b1BoxasLsMqMMmFNuGWs54b/TlKv0RAPBlKJD51whyh8xF+7rg4WljZx7Gzjw2f+sQFOcrgQ9q238iiLe2Bf92gdFDmnwYMe4s127wM7/gggtw8skn4/rrr4emeRRh2zbOPPNMnH/++Xj44Ycb7jPhj3hI+CM+wg/ybLlsu1NYxp/wMgRyvIAqMv5QEeQPlgtEHKZJZCCQ0d2oDhUZAX/oZogLwvyRUqGVgvyhlkIyLgXKZ+m1ERNORoNSsEEoBXIMf+Qt2F1eJnrFtisZ743t+Up/PP4oLJuLzDNB/tD7C9D7C1z+2PppBPGOzuDfCX80Fa3gD6CBbWBz585FLpfj1u2555748Y9/jE9/+tM477zzYinSKC688ELccMMNuOmmm7B+/XpccMEF2LRpE84++2wA3vL6Rz7ykUr7s88+G6+++iouvPBCrF+/HjfddBNuvPFGXHTRRZU25513Hu655x5cccUV+Oc//4krrrgC9957L84///zYeg7tyCZEMwbwyKHZNiDOLtFaMlxSi6G46KClj0avSxajXiTjpAxOabl9nOzaklj9ToY0/oEoZbkmwk/qFec11fH444/ji1/8YoVoAEDTNHzhC1/A44/HC6Ob8Ec8JPwxNuxW/BFjnKbzR4ykkG5KnB2ecMIP14IsV1jCHxOPVvAH0ICzct555+ETn/gEnnrqKWGbD3/4w7j//vtjK9MITj31VFxzzTW4/PLLcfDBB+Phhx/GXXfdhb322gsAsG3bNmzatKnSfsmSJbjrrrvw4IMP4uCDD8bXv/51fP/736/EyAeAI488Erfddht++tOf4g1veANuvvlm3H777bFj5AOAkRb/UBPEQ7PJYbwmQCRb15s+VlwleGMRV0Ioca5JckG1nClRf4S3WXwsoGN4TXF0dXUFbKePzZs3o7OzkyNRGwl/xEPCH5MDk4I/YsjEHUuIGEkhiSPhjzgJTxP+mNRoBX8ADWwDO++887B9+3YsX74cxx57LN7znvfAdd3A4cFf/vKX6O3tja1MozjnnHNwzjnncOtuvvnmSNnRRx+NtWvXSvv8wAc+gA984APNUA8AsO+hS/HcX/8JJ84MdALukn2thIyNmr/wMnmz+gzL+EkhG9GbPdwfrnc1QJMYWZ6MQ4g47r9KuMZezZaE+jkdOtQcXwmaUoCSG70PWX68fQBIbXNgzlQaix1KgVSfOEtyLEzjZfxTTz0VZ5xxBq666ioceeSRIITgkUcewec//3l88IPx9nUn/BEPCX+MDTy7RQTlovb1jNEof9Qag8sfgnJRe6AWfxBotthg8WQoIWIdFMJdeVFGJPzRHoM/BPm+gIQ/JgNawR9Ag6GLv/Wtb+G9730vrrrqKnzuc59DoVDAQQcdhDlz5mB4eBjFYpFr5KczTvrksfjtd/840WpMWXCXoiE2wCKjKMswzO4Tjhh08MNRyrIJWwQwOEZHprcMovaqLdbB0RVonCV2o5xNmNen1a7D4BCBAsDVFS9EZKhO2xUNN+mjtLADmVeimYHTLw+gtLAdxtZcZNm7+94+jCxbxOlNAgL0rHqtMZkEQlx11VUghOAjH/kIbNt7kNB1HZ/61Kfw3//937H7TfijcST8MTY0yh8iiDfCxuMPWc4vP5CLaJzm8QdtmD+0opg/nDYdGifxowKAagqo3Rz+SG1I+GMyo1X80XCelTe96U24/fbbYZom1q5dixdffBHDw8Po7e3FMcccgzlz5sRWZnfEov0W4tBjl+HJB56DK1sOTRDBWJwIngEsAuDllJUlcZSRikgHg0qcCMJfYZHNgPkRvcJ1RKK3arliHTIatEJ0NitVTvzIJaJ2A9pQNEa+4gBOhwZl1I7IpDYNC7MT23M6kdoSPcOQeWoHUtsXojS7ztkxF2jb4OD+jTfU0bh+xN0/vDvsOTYMA9/73vewcuVKvPzyy6CUYp999kFbW1tt4RpI+KMxJPwRHzL+EJXJ7LAfpIUn00z+8BMR82T84DITyh+GCs2MZiHXR8X8YbcZ0Dg5uhQ3Hn84szpAEv6YlGgVf8ROCmkYBt785jfjzW9+85gUmA44/WunYN19/zXRakxphI0cm/hR1DYsk0Z9ibvCBl2U+FG2LC9ySjQqTjgm0lulgKN4hp2nn0hvM6VA52Qa1sqrKzy97XYNWi5arw8VYXcY0MqEFBirYAMEXv4tplyxgcLSTqRfGYnIZNb1IXfQLLQ92x/RY951r2DzxXvDNYiccFxAy1HMuu6fwGWSdnEwjZN6+Whra8OyZcta0nfCH/Uj4Y+xQ/Sg3sj2rRSnrJ5x4vCHiCM0Sd248YfpiPmjTYfGSTasDxfhtBtQcxz+yDfOH+mntyP/+lnIPJ/wx2RFs/ljTBnsE9SHn3zh5yAKQKOTEQlqgJ0dA4JGlTejRcrlfpjHsIwfNjLssIT7YmV0VAknXOf/zY4PeE6JRbz/ZXqzs1cyHVQ3SDi+jOz+GCW3QjjhOgoEcrBUrjVnw+w0oI+YERlt1ITdaUAdMYN6O4CTJiAWBQllQs68MlIhHLY/BUDmuX7k3zgbbU/tBBRUshAbr2axx5UbsPW8JbC7SfSpovy30e+i9+qXsOu8/dF0TLM9x+973/tw8803o6urC+973/ukbWWZ5xM0Hwl/xEcc/givOLAyPGeA19dY+UOVyExq/shbMNsN6Lkof6g5E067ASUX4g+3nB/MRkP8kX6+H4Vls5F5ZifA6JLwx/hjPPgjcVZajJef2oj1a16caDWmNHyD6v+Ow8Y9nKQRqBpuiuD5Er+uhGjcfN+esTNhPvwZLZ6BB7yw8kZYhlbDTSqI6m0jmKzSLwcAm1mZqejn8mUqOqQI9BINlOsl19NB9ZyKwDgUKLZpSBXsQGf6iAkHAO3UoY1Ygf7UERNmmwqNEijlFRoAUIoUVCMoLmiH3p+HUqpuWUlvGIGpA+6eXUhtHgXK8fkJBYzXhpHfoxPozSD1rwHvsKVCQEYK6F25FjhwL2SP7kFxgeIFALApMptcdD+wE9veaWPryv0AChy9zyfx0L9+gmZhui3jd3d3Vw67d3V1SbOuJxg/JPwxdvi2MvyA79fx7LpfzuMPoGrvWYyFPywEt3+xMn7fLecPg0A3G+OPUlqDUQrxR87jD7TrUHNB/lByJqyMBpUCSpHhjxIAlaA4LwN9sFgffwDQtw6juKATdFYaxobBCn8owwXM+cZauG9I+GM8MB78kTgrLcZd198LVVOSaC4CyJbiwzNAPvwMwLxkjSwhhMvByLB7jx14Rtwv00N1fvQstj+fkHy92P7YsWQ6hMdhr0ljjBZ7TQEZ1ZuJ8mWMEuXKqP4AAMzyNi9fJpOvnl3xJ27V8rX5ybhY0gaAVL46xWvNyUDZUfAI0KZIbx2t1FWIkQIpC8DL3mHJ4n6zYLzQDwWA3l+CjhLwmjdztvVDPcgdu5T/pXj1NdC99kB+bxX5vedVyymQPaH1mc93Z/z0pz+tvE8OuU8eJPwRH2H+8G112A6Lkv2y5f5kWcXWhmQcpqwe/mDLw/1NGH+YtfnDSmtQi1X+SBcl/JHj84fBnJe0etNQdhXLY1Ck+/KVuob4Y2uZPz7Yg9w7E/4Yb4wHfzQS4C1BDGx8dnNCNGMAzz9nSSMMYVjelCKUIRBHexH1Jyr3++ON5Uq20Ir642VHrozjyHXg1akOlerAq3O6UsJzPkrOEvbnz+BF+mvXhTLWoYvFN2mvPfjlBDBnNzkfBR3Da4rjmGOOwdDQUKR8eHgYxxxzzPgrNI2R8MfY0Cz+gOS8dtgJqKu/GuVc/hCUy8ZpNn8obuP84XalxfxRsJvLH8sXJ/wxwWgVfyQrKy2GWWpy/O4EsUCFZllssGWII0NjGqKmLqjWUIJL7qpkTiPONUkuSJadWNYflUVNiIOYy/i7A9k8+OCDMM1oCNJisYi//OUvE6DR9EXCH81H3J91o06HDLH4I4ZM3LHESsTgD1niR05elppI+GNSo1X8kTgrLcasBT1QFAWuLAv4NEYt08IjiTi/Z6UkjmAiIyKZXrXqI4cweRuFa0CWAJM9VFivDq6qQIX4u8iTUYvRCC6V9rrEkRFdryTLsJqjsLolS1A8uICaa/LvK+4s1xQmm6effrry/vnnn0dfX1/lb8dxsGrVKixcuHAiVJu2SPhjbODZrVjOBSeSlmyMevSqVR/hjwbH8GWayh+aApWTf0Umo8j4w1CAgiByRMIfUwqt5o/EWWkx3n7qW/DXOx6baDUmNRolAfbwYrjeBv9LLc3kC3E4Y9E4vh0VLb1zc8BQid4KoEvsJXcsHaBmY/dOzdtCHZwOHdpodCZXLTjCe+ekdWFGemtOBvr2QkQmvX4n3JQKpRQlqa51oygu7OL2J4QCdD421JhMLUxDsjn44INBCAEhhLtcn8lk8IMf/GACNJu+SPgjPkQ2UJb7xAGfP3x7L5o8EzkronFkTkScRMQ2AfQGHQ+qAdRqkD8KYv5w23SoeQ5/FCX8kdKggr96mPDH1EKr+SNxVlqMI9/zJnT3diK7a2SiVZmUkJGAbNldlMxK5KwAYiLynZ9GHBk/2hhPTxkZivZL666EiDRAj+ZwhGJKCNQACMeHUOCdneHNqCmjlpjEu9PQssWITGp7DlQBqMvRQSV8wiu5yL1xDtqe2VEJV+yj465XsPOEN4Lqdc5VUm827YqvnVpf+wRCbNiwAZRSLF26FI899hhmz55dqTMMA3PmzIGqNnu/RAIZEv6Ijzj8IZtfF/GHLBGx6KHf14GnZyz+kE2ECfhDtSR66wDh+BAy/iB5CX90paENc/hjRyHhj90EreaPxFlpMXRDx1lXno6rPnHdRKsy6RE2mjJSEcnIEj/KSIANHcnWyRwZv0ykN49U2JXtiEEn/ESSui2RSRGo5XDFbJ1qiu8d1T1HJiyjALAF2e2VbFGogzmvA8bW0Uh/qa15mL0p6LtKERl9Ww5uRoOStwPEpxYczLo/h13Hd3A054AAs/80iLeec3h97evEdAs9CQB77bUXACRbjiYREv6IDxl/+GVhm2VAzB/SlQ3IEz/WWn1pCn8w49XLH3ZKgcZJ/KhaEv5QAWLz+cMpRwsLQxmW8Mfcdhjbcgl/IOEPGZJoYOOAnZv7J1qFSQ3WEIV/q6yDQUPlLqccqGa3D0MB4KaqMqycT1K8/qjgvQqPpER68/RjySYsw+ZlEekQ1lsrUTjlWaSwjGgs1fRCGHN1KNhwdSUyjgLAbVO5OqS2jsKa387vr78Ep1uP1Bk7crDmdcFt00BDVqjnly+g52/lWTiR3SvHw+xdNQKrNyVoNDVw3XXXYcmSJUin01i+fHnNQ4gPPfQQli9fjnQ6jaVLl+JHP/pRoP53v/sdDjvsMMyYMQPt7e04+OCD8fOf/7xufVauXImbbropUn7TTTfhiiuuqLufBM1Bwh/xIeIPmR0W8UeYp1g5HWL+YFEvh8XiD0l/Iv7QSy4crUH+sAE7E7XpAKAWJfyRqXJOgD+25WDNb+NfU8IfNTFd+CNxVlqMwmgBt13x+4lWY9KD57CwM1JOqN5fDg8bU7aOs/INtRQ1lv7fGlDZPcv25xttVyATJilWB57eJPQ3+16Fd36FLWN14OptUTipIOGws1iuEpUxcjbsjMLX23LhdOoRGSXvwFUB11Cj/W3LwVzQDqqR4L2jgJK1UFzSWYnU4sukXx6A3ZlB/sBeuOWD+pR4r96fPId5/9uP1A632pkfsB9AeouD+Tf2YdcJnRj8tzas/GSTz1LQMbwawO23347zzz8fl1xyCdatW4ejjjoKJ554IjZt2sRtv2HDBrzrXe/CUUcdhXXr1uHLX/4yzj33XPz2t7+ttJk5cyYuueQSrFmzBk8//TQ+/vGP4+Mf/zjuvvvuunT68Y9/jP33j2Z1PvDAAyPElqC1SPhj7GC3XAH8lY56+YOE2rMyIv7wwdOB55SMmT9IsKwmf9gUVirKBf718vhDK1iwOTI+f7jtHP4o2N6WLy5/5GHOS/gj4Q8xkm1gLcZ9tz6CUqE00WpMCbBGODwh4seRdxBdRvYNMVvHLqMXUc1W77fTy23YpW4KQEkrIEUXNoKePA3Jh718nxDD5ETKL5MEE3XZBDBoVG8KQCvv37XLWYMr/THbxFwFlf26FAB1veDMbgqgZlURt9wHAWClFKgmE9amPI5DAGgKiON16GoKlBELBECpOwWtYIG4FCAEbpsOPVuCC+8cCyl5G5+prkIbLEKxKcyZaRAVIBYFVQA3rcPY4O25LyztBHEoQCmopkDJWmh/ZgROh47csjkAKatnU/TNygPzZmHht19FadlM71BlwUHqiZ3Y8tWl2Haml9jLTRH8fvtruBjNw3gt41999dU444wzcOaZZwIArrnmGtx999344Q9/iJUrV0ba/+hHP8Kee+6Ja665BgBwwAEH4PHHH8dVV12F97///QCAt73tbQGZ8847Dz/72c/wyCOP4Pjjj6+pU19fH+bPnx8pnz17NrZt29bYBSYYExL+aA58ex1+HvTtM48//EfqEqIJGHWODAWgl+1ymKdYJyPMLb4eYd7zOYzAc4LCSSVFOmi0zB+Kl7We7c/fJhbmD+K6HheEzjpK+YPSCu+5LH+oCkjO4w+zMwW1WOUPJ6NDHynzR5fHHwSAq6nQhsr80ZMGUQBiu6AqgZvWEv4QYDrxR+KstBhr7vwHCAhooy7zbobwTJYP3v5glkD8v32oqJ4vCcuwe4pZMkgjmP1XCbVhnQql6LX0fxg8vf0ZL5X5m/2fp4NBq2dS/L95egfOyzgh3RgZuEGHTClP6SmlkN6Ms6OXXNhtKrS8A1BAKXnXqlIAllvV26wyXCpbgt2hQx21AFAo2VLl2pRsEXaXDnXYAmEisxgDxaAOqD5sZV4ZQe6QeWhfVw1rCADqqIX2tX3Y9N+HoDRXCdyILZ/fK9AW7+oM/u0Co8tCZROM4eHhwN+pVAqpVHC7gWmaeOKJJ/ClL30pUH7cccfhb3/7G7ffNWvW4LjjjguUHX/88bjxxhthWRZ0PfjLoJTi/vvvxwsvvFD3EvyiRYvw17/+FUuWLAmU//Wvf8WCBUmm5/HE7sYfIh5o1VgIjec/XPM4R8QfqVBfSkgmkHixbDr9/nnX66/G1+I9VgcdfM6R8ocb4g+2jYA/VDOkd5g/0iq0Ypk/yhnvFQCwGf5gkpgaIyUviWOuzB8jDH8MF2F36lBGrEBkL2Mw4Y+EP6JInJUWI7trBDRuNsBpAN84iwiMVx6H7FyIv+xx+qslw6t3qThEsgj+dgBZfSP6E1v+XeTqrbMbHELtJTH3hZBckJsmjX8gSlmu2RjDz3bRokWBv7/2ta/h0ksvDZTt2rULjuNg7ty5gfK5c+cGYtSz6Ovr47a3bRu7du2qzGhls1ksXLgQpVIJqqriuuuuwzvf+c66dD/zzDNx/vnnw7KsSgjK++67D1/4whfwuc99rq4+EjQHCX+MDc2yCjI7PG78gUnAHzWSOIr4QxSemNgJf/CQ8EcUibPSYmQ60hOtwqRHo2Yijg2QjRFnti+WDuy0Wb0yaPJspCybsGAsIovuESdjsOQeECvG1VJww2yOCTH2D1fkAGzevBldXdV4/+FZMRaEBK+XUhopq9U+XN7Z2Yknn3wSo6OjuO+++3DhhRdi6dKlkSV+Hr7whS9gYGAA55xzTiUTcTqdxhe/+EVcfHEzN0skqIWEP5qPKcsfMWUmnD+chD8akkPCHzwkzkqLsf/h++Lph56HK/vBThPwzIgoRKNMxt+vW297QLytzJeJM/skM4u8Ol5Y4lo6SJNZ6gSaJbaIXBlCK/ugI+1V4u0HDuudLQnvndNhQB3lHUUF3LQCpehG78NANN6+j/QWG+Yso+HQH+nXmruvf6x7jru6ugJkw0Nvby9UVY3Mgu3YsSMy++Vj3rx53PaapmHWrFmVMkVRsM8++wDwknWtX78eK1eurItsCCG44oor8NWvfhXr169HJpPBvvvuKyXMBK1Bwh9jg2gbbyPt2Tpw6uPwRy0ngssfEpnx4g+KaJj8Sp2AP5SRhD8akQMS/uAhiQbWYpz0yWOTvAUQG1k2j0kYtkAmLWgPAK7BN/9s5LAwfOLg1YtizbN7ksOwBQzkF/NkZIsuonunWlSot2MoXBm94Ap1cNr4LqACwDX4pkLrywt1MBd28D+/jUMo7tUJyqnsvntb41aJAr33N/ngNx3Dq04YhoHly5dj9erVgfLVq1fjyCOP5MqsWLEi0v6ee+7BYYcdFtlvHLgcSlEqNUbIHR0deNOb3oSDDjoocVQmCLsbf4zXeRVAbLtlXCBSUGaf4/BH+Hwji2he9qBq48IfOj8ho1aU8EdKzB9U529gS/iD86oT040/kpWVFmPe4jk48uQ34dE/PjGtZ8dkKxuivbh+tBOejAUvwlcYmlk1wKJxwnV++MhGk3AJ9aaS2SziHZQP17GEx5u54+lAmP7CUE1XqLfTpkHNR2ez9BFTrHfagGIWuffO7jagZk2EkXpl2CMUytFhRhvIq9Gs3On1u5DesgjF+Up9pOMCHS9auGfD9XU0nny48MILcfrpp+Owww7DihUr8JOf/ASbNm3C2WefDQC4+OKLsWXLFtxyyy0AgLPPPhvXXnstLrzwQpx11llYs2YNbrzxRvzyl7+s9Lly5Uocdthh2HvvvWGaJu666y7ccsst+OEPf1i3Xv/4xz/w61//Gps2baos5fv43e9+14QrT1APdkf+iLFRJxZkSRxFtluR2G7WWaiXP2SJiEU6yBIR++O0nD8sKuaPtAq1GHWp9LyMP3SolsPnj3KQljAS/qiN6cQfycrKOOADF/77bjU7FgessQnbRpVT5kOU5Mpg6uqVkSXN8svC5QqisfN9iJKHyXTwt4KJdBCN46j8OkL5ehMgEgffh5a3K4m0wnVO+aBhuFwfLsLpTnHrlKwJqnLKKVBa1MGVaX9qO3IHzwEP837wEtQ8Fd9cHy6gD1HM+J9/1mjYOPxl/DivRnDqqafimmuuweWXX46DDz4YDz/8MO66665KNuBt27YFYuYvWbIEd911Fx588EEcfPDB+PrXv47vf//7lbCTAJDL5XDOOefgwAMPxJFHHonf/OY3+MUvflEJb1kLt912G97ylrfg+eefxx133AHLsvD888/j/vvvR3d3d2MXmGDM2N34YzxXV0S2Vma7RTIEjfMHgThZZCz+kOgHQXl8/uBzgVZ0Kqsa4TpXJDNShNMl4I9hK+GPhD9qgtAk1EhTMTw8jO7ubmSzWXR1dYFSik8d+nm88swm0BqRNKYD2NkadpaFNyPjv7fgORq8fb6iGS1bB1SLL2PDM+DhunD+FFbGhLfSI9KBp7dNPOeEJ+OPFa5jQ01G9FYB1eHfuzD8OjOlQi85fL1De4wrMm0q9DxfxupOQcuWoveOADCUSjhkVqawZwfSr42CuNFrzS+fi/Ynd3ih0pgZNHN+B7ZdsC/MOUr0Qy7/ndnkoPOGfyL7qQPwwhe/XKkO/wYbgS/7us99C2qq8cPNTqmIF7/z5VhjTxa84Q1vwH/+53/i05/+NDo7O/HUU09hyZIl+M///E/Mnz8fl1122USruNsi4Y/mQ2ZTRfwhtcNonD9qcRiXPwTjsGO1nj8U6CW3If6wMyrUAp8/7O401Gwx4Q8BEv4QI1lZaTFe+Me/8PJTryZEUwZraIDoEjWbWNGv8+PS26Fy33AVEZ1E0bxchTA5Mn6uFidU5y+vlzgyenn8sIyvA5OHsTpOedbK4sj4Kzbh++AbcYuTxV51yjqQYJ2PUiqqg15yvHtnKJH+iENRNACqkKBM3gFVCUxmJaXyWWRLsAhgdxrBe0cBUAJzThvcdi0gk9oyipILFPbugtOueVmGARAFMLYMI9+VRv6QebDmZOBkVNgdGlwdmHnVOiy4dis6XrSg5iiISaGOUnQ+Z2HhtzehsKeKHZcfiNJ8BW99/afQVNAxvKY4Xn75ZZx00kkAvCg0uVwOhBBccMEF+MlPfjLB2k0v7Db8wUQaYn8msqtq1hX7Dzki/rDBT0Is4w8eF/gyvHF8DhPxhw0OfyA+f5hN4w/X4w+dRPojDkVJj/KHWvD4w+qM8oeaLcIigNMe5Q9CCczeTMIfUxyt4o/kzEqL8ecb7oOqKXDixBPfjeD/BgnzAjzjyX4JRXHk2TYlFUgxW2bZ+YtwEiyDUw4Ez7uwEy8k1B+rN3v8LJyYkj0+Fr5WXvJJng7svWEzD0d0KBc4CqAy7dIlvowGAOVEj6V2HalcdX9wxqy2ZvcHE4fCKCeAdEl5idrXmwIY8Wjc6k1D2+XNlKmmA3VHvnpNGgGxKRSnfE9f9hJd5V8/G23P7wRcwOgrePdhsAAA2HrJG5HbWxNOozgGMLJMx8iyPQM3L/vOeXyBBA1j5syZGBnx9oMvXLgQzz77LJYtW4ahoSHk8/ka0gmaianKH0pXJ0h7eyRMqr1jO2BXIzzJtoQ1c7tYxQ4iOtHOcouIP1h7CgRtf5g/2L4jdriMcFJItr9m8IfRbP4oRw0z23QY+Sp/pK1q6zB/+AkgI3pTALkyf8xKQev3VuqJ6SC1q1C9poQ/piRaxR+Js9JibH5h65QjmlaBRz5xlvZUUbgUwRhAdTZKBH9Wqt4+a+nNk3EJoAlmTkRjy0I7h5fF6+lTkx3SFejmdKehD/HDRSoFfthJACA25epAM+LUZubMOg9GBgYCrJnNNWVjDT05lXHUUUdh9erVWLZsGU455RScd955uP/++7F69Wq84x3vmGj1phWmLH+ogt+jHQ1FO96Q2lSJnKyuUTtcC1z+QOOJjZvNH2oc/uhKQx8W8YeYzBP+mJpoFX8kzkqLYVuSJ+sE44Zm24BYhDsZDFGMI2q8EJHVyviqcFEj6RgXBJWAAU1D3CX5yfAZjxHXXnstikXv4eLiiy+Grut45JFH8L73vQ9f/epXJ1i76YUpyx8E0sR0Uw2yh/o4mLJ3ZrIfcU74Y8LRKv6YkmdWBgcHcfrpp6O7uxvd3d04/fTTMTQ0JGxvWRa++MUvYtmyZWhvb8eCBQvwkY98BFu3bg20e9vb3gZCSOB12mmnjUnXOYtmQVGn5G0eF8T5bcahb2ls/RrgydWa6+TJxLGjtXICNOpHuDG+i2rOFOpABflXAAiVI7b4k9BG3No3NwwX0EabPPs8Tfcc27aNO++8E4rifa6KouALX/gC/vCHP+Dqq69GT0/PBGs4diT8MQ5wXHBj95CJ/4nEXemYcP6IMXbT+UNrXAulmPBHwh9j548paAWBD33oQ3jyySexatUqrFq1Ck8++SROP/10Yft8Po+1a9fiq1/9KtauXYvf/e53ePHFF3HyySdH2p511lnYtm1b5fXjH/94TLoe8+Gjdpv4+GOFyAA3+vtMSWREiSRlSbhkn45oHDYzMK8/rg5ULOMojRMHFLF+otUQNW+LdejU+Uv/pjgRmCtI9gUA5vwMVyb9/E44bfxF3a7HRxq3SgrQuaa/QSE5xiv05GSDpmn41Kc+1XACsKmEhD9aD1oocFdWtPnzvfrxVoiBLFS+iD/8Mp6c7FpEdb4ODfGHRMaO4Xg0mz/c9hj8IXHEE/6Yemglf0y5bWDr16/HqlWr8Oijj+KII44AAFx//fVYsWIFXnjhBey3334Rme7u7kjWzh/84Ac4/PDDsWnTJuy5Z/WwVVtbG+bNa95hqyNOOhSzFvSgf9vglPeax4JaSRd5y+wlBA+usxAty8sSSTrgf+FrJeHiJg9DfRmS6+1Pc8Uylk5gWNEvj+KI9a6ErOHprRAQTnQhZcQS37ueNLTB6L5jY2cBrgrA4ejg8L/wiuUid1Av2p7cETHOHX/egJ0nHQzXqMG+PiigDVP87y3n19E4QT044ogjsG7dukqs/t0JCX+MD6hpgto2oKpcp2U8tkGFg6CwiMMfItvNOh71XhdBc/nDT0TMk4nFHwq4Wxhk/EFyEv7oTkPLcvijv5jwx26GVvHHlFtZWbNmDbq7uytEAwBvfvOb0d3djb/97W9195PNZkEIwYwZMwLlt956K3p7e3HggQfioosuqkQ1EKFUKmF4eDjwYqGqKj713Y9PKaJpBXwDK5uZCteJHBUgGOI4DFESLtYpCYMNN8mT4UGkt09EvLpas2M8GaOcTZg7MyVIwqXY4tkxWrbwPN2cDp1fN1gU6z2Xn7grtb0Ia36Gq4OxcQR2txHZK6xYLnr/nK2baABg9h27sHjx4joEGkB4ab6R1xTHOeecg8997nO49tprsWbNGjz99NOB11RGwh/jBzebBYDIdjCiKuPyUxE5KoB8W5csiaNsNYQnI9s5IFqtmRT84Uj4A2L+cNsE/JGV8MfsNq5Mwh9TE63ijym3stLX14c5c6KZS+fMmYO+vr66+igWi/jSl76ED33oQ4HEOx/+8IexZMkSzJs3D88++ywuvvhiPPXUU5FZNRYrV66smeTmhX/8qy69phPY2RwF1QRY9c5OaQBMDdDtaHs/pr1v2Nl69m/2vY5q4i6eDn4ZW+cvXrDbCtg6lyNDQuWsjE4b11stUdgGgWrSiIxCqytaARkHsNo0L5N9SAdt1IKTUaEUnMhn5KjV2ThWxtg6itKCDhhbRyM6aNsKMHtTMHaVgvd7sIDSom4vdHXWDMyQdf/+ZTjt+6H/2A5vMFHWNgBz7hhEcZ8OToOxYTpHczn11FMBAOeee26ljBACSikIIXCcKXroGwl/jCdoyYQ7OASlZ0bluwMA2ty5sHdsB7Vd4Ux8q8E6BI3wBwsZF7AyLAfUywWTmj9cwMpo0ApR/lDzFtyUClKK8oerAoTHH315lOa3w9iWS/gj4Q8hJo2zcumll9Y02v/4xz8AgLuszBpDGSzLwmmnnQbXdXHdddcF6s4666zK+4MOOgj77rsvDjvsMKxduxaHHnoot7+LL74YF154YeXv4eFhLFq0qPL36FAOv//BXTX1mg7wDWw40y9QmyDCcekBwLA9gvAz+bIyfuLHsPPhr26H2wOew+Jnqg/L+Drw9GZ1YCFywvwtZODIqPASQvrbwsIyvHujmRR2ClBL0XoFgKMTKFaQjPS8DVv3VmDCBlItODBnpKAPBQlCdbx7QNt0qEysfQBIbR1FaU47tJEilHL2Yn98bVcJhX26YLw2CrVY3X+f2pyF1ZNB/uC5SL/YDzVX3bc289YXYLy8ANnj5iC/RA1elAu0v2ij+65t2HrhIhCL4tyTLsf3//RfaBriznLtBmSzYcOGiVahYST8MTlBi0U4O3dB6WgHMpmqwzJnLuy+PmGyy3onreoFbxuyzzW8OpHz4TsLXDsskQH4/CHjgrj8YZLqtrCW8UfBhq15k1dh/lBKDqyOFLTRIH8oZf5ARodSCPHHthxKszPQRs2EP6Y4WsUfk8ZZ+cxnPlMzcsrixYvx9NNPY/v27ZG6nTt3Yu7cuVJ5y7JwyimnYMOGDbj//vsDs2I8HHroodB1HS+99JKQbFKpFFIp8Yale372IGxz6s5ENhsEVafDf8gHqkaSdWh8+8Lu0y3BS4blt7PhJYzyzRSvP//uszNLfn8WglvUNFQJid1T7KC6rSAsQ1AlA1bGJ8GwDr498kkofB80l68Dm6fF0gjUclQUSgBSYu6dApCyoKMR6OX9ylZKgWp5XhBVCIhCK7NnbloFsbyBXV2BNuQl6rLaDRDXBXFcUFWBoytIDZtwAdgzMyCmA1AKaAr0nTkoFHDaVbhdacB2vOzGqorMv7ztLcU92oGU4kXAVAB9Wx7t67bDVQjyb5gLqhHvnuYsbDvRBvZSseCrL8I5qBdumwYlZ0N5Zju2ffMA5Pb3HuqoRvBoGz+Of2xMM7I59NBDcd9996Gnpwc/+9nPcNFFF6GtrW2i1aobCX+0GIQAhgFt1kwAgD04BBQK1brODmgdHbBLJWBwEPCdEEUBurugpFKwh0dAbBuUEM9mgICAch+im+2s+PbVn8DybaWJasAW1pnwOYJnh1n+cDjlXNuN6sMWj/d8x4nHe43yh04FHFZ2YoCY/GGUZShAFYAo3mSXC3irKeV8QK6uQB0t80ebAcV1AccFVAWOpsAY9fjDmZEBsTz+oLoKfWfBc47aVLidKcB2QVUCKCThj0mO8eCPSeOs9Pb2ore3t2a7FStWIJvN4rHHHsPhhx8OAPj73/+ObDaLI488UijnE81LL72EBx54ALNmzao51nPPPQfLsjC/HMEkDh6/+0l+CMdpBtFMFG9Wy3doWEPuI4XqwUmCarZ5/4vs32lWxl9SVzh1OqoGnS3nkQkrw1vpCUccY39cKjyyUEJfBYLguZiwDoGMyoysbtOq3kw5gbdMT+HNeCnMwUq95MLOqNAKjnd40WHGLDpVve3qw5GeM2G3G1ALNgAXWqEqowwUYHfqUEet4KpWzoGaiy7pA0D6tRxyh8xD+7rglhvFpWh7ejs2/fchKM0Nhkbb+vXXBTs5bWbwbwrk9p86D9aTEevXr0cul0NPTw8uu+wynH322VPKWUn4o3VQZ/eC6METIFrPDNiUQpvZEyxPpYB58+AWCiDpdGC1SuvqhL11W6R/UeCVsYJnf3R4D/e+w+K7ibXssMh5Yrdr1csf/i6CMH+wvOe3Y8eZcP4wKcMfCPJHieEPZpuPnjdhtxnQijZguZVrVgAoQ2X+GLFAUF1pUfMO1Hw+4Y8phPHgj0njrNSLAw44ACeccALOOuusSljIT37yk3j3u98diOSy//77Y+XKlXjve98L27bxgQ98AGvXrsUf//hHOI5T2Z88c+ZMGIaBl19+Gbfeeive9a53obe3F88//zw+97nP4ZBDDsFb3vKW2PoOD4yO7YITRBCHyHhL8GPpr5YMr96l/EguMoi23MYFEURYqdRzylxDBXKC9nGya0suyE2Txj8QBXDSzZyH5W8TrFduKuLggw/Gxz/+cfzbv/0bKKW46qqr0NHB38v9X//VxO0S44yEP2JA4f9gpbvmynlmJhLNHF220pPwB1PP08FQgbygfcIfEbmpiPHgjynnrABexJVzzz0Xxx13HADg5JNPxrXXXhto88ILLyBbjkby2muv4Q9/+AMA76ayeOCBB/C2t70NhmHgvvvuw/e+9z2Mjo5i0aJFOOmkk/C1r30NqtqoiaiirZMfzSJBfMSZZ5QZ7DhbDuLoENd4NXVLRI3MlLyxpASlxtBM0h0xY1wtBVSzcTVq9TmdlvFvvvlmfO1rX8Mf//hHEELw5z//GZoWpQdCyJR2VoCEPxqGaGVHtuJDad3ngFoFkSVptu1O+KOKhD+qfSb80Vz+mJLOysyZM/GLX/xC2oZdOl+8eHHNpfRFixbhoYceaop+LJYddQCeevC5KZnYq9ngmREL8hDFPMiihsmW6/1vQLie3U/MQ6NjicaR6S3L5+LbvbCckyLQS+LvNX8sV6y3RrjZgdXhovCanM4U1FFOQhcAbrsKJedEZLTt/C1iAJDZZMHsTTU8HZje2Nw9x9Mtmst+++2H2267DQCgKAruu+8+btSs3QEJfzQGWipx86VQIyV0SKhpgaTTkXKoCmj5Wlq11asWRBEfZf2xZ0WawR8yJ2LK8odKuI6JMiLjDyPhj5DcVMR48MeUy7My1XDime+Y8D3HkwGiBFgyR0V0rDR8NqReiNrLYtezBx5Z1JM3JgwiqZfF4xcZZr0kjp9vp1W+3kUq1MFJ8c2BAu8AJQ9aOdwkrz9rXjv/M98yguLeXZEY+QAw464tjVslF9jzsYEGhWqAjuE1xeG67m7rqEw1TAb+cHN5fjb6jnbQfIGrHxVsHdPmzm3qT0TmqIjcO5kdrrWCwkMc/pBx2KTgD0NpmD9cCX9QIX/kE/5I+KMuJM5Ki9G7YCaO+eC/gdRYPt3dUcug8yBLyOiv2obr6yEBnozIoMfRW+bIiJJ9+TNtMsLh9qcIiKh8YJ5LRO0aVwc953DLAYBqCrdOAWDPTHF1MF4ehqsSfn9taS/iTFjmlUG0bXDETxqRjoDOZ03csf5/6hRIkGDqYFLwh23DLZW4TolrepY4XKd1doDm81yZuJNNjYJdTQ9DlPixHkemEf6QJZKc1Pxhug3zh5ZP+CNB65A4K+OA4z/2tgmfHZtosMvo4TvhR1ThwSV845eC2HBDUK5CTFKiMj/WPa+edabCdfVca71EqQCwNcLvz5UQcppPEHrO9kJCcupcha+DlrNg92T4OgyU4BpRUlEAWHP5Mm3P7MDoofxQsXO/90/ow7Q24bhAaruL7pueq9EwJqbhrFiCyYfJwB/u4BDgOFGnpGcG3CHvbE+4ztU0wLSiMguq0dFafVWi/v1IlLI2PPvcKH8QyTiTnj9S/P5k/OFnvY+Mk7PgdCf8kfBHfCTOSotBKcX3P3PjlI3y0EyEl6rZ96xRZ6FSscMSdnL8NmHHiJX1E3eFZdg8KWHboQtkeHqH++Pp0IjeFR1sCksnERn/PU9vrejCTitcGcWhcI1onep6cfZ5euuDhYrDEr5WalK4HVpEh9TWPMz57aB6lNza125Hbvk8uCnFkylfnjZUxMJLn0dmU/nRIPzFKF9sx4sWjF++hB1feSOaDX/PcZxXggTNwqThD9eFs6sflLOSovbMgDs0VMmt4tdpqRQc6oIWvK1irIy+IBjOWcQLYwWbDDGM8eIPdpKMlfHHmrT8UQLsFLgyIv5QaHDCK3DvsgU4M9qiMkj4I+GP2piSB+ynEp75y3q89sLWiVZj0oBNfsXbZ1uEl/QxkCeFVsO666H2CqpJGtn+fMPNi2evwdtGpoVk/PfhxGG+jFP+m7c9jSfjE46fjCtg7FFN5sXTm5YTc7H96RaFCUBVvWzyYRm7XB7Qu+jCVQFXV6ExuVQAgJiup3f5YD3rsLiGAiujITVigjLGXhsqeDp0p6BmS4FrdSiB2WVAVQA1a1aCBen9eViz2+EaCrRs0Yur71JQTYG6cwQmUWAvnwNtVw7qqAmqENg9afT8bhtm5k2MnrgQhcU6XJ1AMSna/mWi/U+bsOXrrwP298LN/tshn8Yj65q4lB9m2kbkEiRoEiYVf7gu3OFhuLoORdcBwwAIAVwXUDU42SwIAUgmA2gaQMu2rViCWyxB0TUglQIUBZRSqF2dsIdHKt2HH6ApGp9NpYhua/L78LPBA1UbqaB6roRnh3l5wHyHJWy7w9zG2mHfYRHxB5sPhecAhWVq8YdLPMdhzPxRgscfmgKt5NbFH4rPH+kyfzBCajZf5o801GwxeK0OgdmpQ1UJhz/a4Bpqwh/TGImz0mKs+un9UDUFTpx44pMQrIELgzW4MhAEMwCzBMLGj6Gh9n47P6mXD0Mg4xMQTzf2YD9bRyT9sWP6iSlRQ0aF52zxdGAdr7AO7GwQS8ApoDJN5yieY+GPpThBmYoODqCWE3WZ7RqMnB3Um8lkjDLBqaYLtTyL6qhe36TccQoAsiXvPsxMQxsoevcnZwXukVuODqOYFKmt1XwRhYNmI/PsThDTRXpTOYHLE17eii1fPRj5JarkKYXA7E1j6M1Moi8XGH67PPt4o5hu0VwSTE5MBv5QZs0EMYzIIXt7aAjajBkAANUwAnXuaA6kvc1zUJioYPaO7aC2W7FnMmckzmoSKxPmKt82+ZwTLg/LsPwR7ktmu9lrEvFHmPdYiHivEf5gEz02hz+8hs3jDy/6FssfasEKTCwG+aOa6Cvhj+mJxFlpMbb+q2+3cVR8iEhkrOTSrD55qBVeslEdas34iZw5kQ6i6+TN7FVk3CAR1dOnyglNXIGgyu1MQx3ih3ZUivywkwBAHMrVwTHEeSesHqXx6VQCWDPi57JIAPT09NSdE2NgoMmRcxIIMRn4g3DCFgMAcSW2RBP8HhlHZTzAGyvuhHfcPfPN4sWEPzwk/DH5MB78kTgrLQaVGfQECaYAmv4NbsXTSrP7nGbL+Ndcc03lfX9/P77xjW/g+OOPx4oVKwAAa9aswd13342vfvWrE6Th9MTk4I8JPzGTYAoj4Y8G5aYgxoM/EmelxZi3ZA7++Y9/wd2NVldEszGyWRpZX42OFef3LAsvWQtxdBDtnW703kllFG9/cCM6OKoCVRg0kw+1nNSLq0NKBfKC2TEFoC7nPphipfVh15vlamR2jAJ6trFrqoXptoz/0Y9+tPL+/e9/Py6//HJ85jOfqZSde+65uPbaa3HvvffiggsumAgVpyUmA39QxwZUJZoUUhZO2RH8HstJIcfL/eHZrbgrHQl/JPxRLxL+aD5/JNHAWox3fuRtu5Wj4hsO3m+q2dvARIkkdYgnLmyBTJwkXLI6Wf4Vkd5SHWr8EnkyiiTsJCV8HYy8Jb53XQZ/6b98ApV7rZKlX3NBG7c8/dwO2J06t7+uR7ONWyUF6Hx4V4NCNUDH8JriuPvuu3HCCSdEyo8//njce++9E6DR9MVk4A+aL/Ar2tpAOaGJAcAtlfiJJOd6ZwPG42cissOy3Cci/pDxngyi9jIdYvFHDfIdH/7g50tJ+KPB1xRHq/gjcVZajEOPXYa5i2fvVkkhRbMkMoNeEvTFhmjk1TVq0OM4MnGy0csyBsfR2z/oyJMxdf53R0qgVKyDHyM/osOwKb53PRk+eQ0UQTV+4i5S4H+yiguU9p3JvRHtqzdAKdD6jbYL6AMUt/7qwjoF6sQ0JptZs2bhjjvuiJT//ve/x6xZsyZAo+mLycAftFAAQqGHAUAzDLi5HF+ovR3U4jsycVYorAbbA83njzjZ6EXwV2p4iMUfVKzD+PFHKeEPHwl/RMrHyh/JNrAWQ1EUfPYHZ+Ar//7fE61K08AaWhHphAkpJWjnt+URGEEwxDEJ1Yl08KOFhWX0GDJ+THsZufJ0E+kdDjPJwo/OErl3FhXaMJoCSIk/jita5ncoV28FgNVlQCs7LYH7MFAIRHthYc1sg7EjF5Ex+osoLmpHenP0gSb94iCs3jT0/mIgE7HiAHPuGEDfB2fVfqopf3Fm396HxecvljRM0Aguu+wynHHGGXjwwQcre44fffRRrFq1CjfccMMEaze9MFn4w80OQ+2ZESmn7W1e4kdDD6ykaKkU7P4BqDN7QCkN1CnlrWBA/U5LOGR9PYhrh0VcoElk/IPsXDvMac/20TT+IJ7T0jT+IMGoYtWBJPzRaUAbSfhjOqNV/JGsrIwD1vzxid3qjKJs/y0RlANebhMeVHgEwbOLKlPO1odXZNg6HeJMw1TwXkd19i48wRFO0CUqZ+tEmYZl905z5Xpz70MJcFOEr7dbTagZ1ttJ82X0YRNOhx4dB/Di3pOoDqkdOZgLOrh6G5tzKJWX8wP3Z9QEiAqrNxPRofOejZjzf0NeoWjZzQWIA8y7dRfyb5opaBQf0zmp18c+9jH87W9/w4wZM/C73/0Ov/3tb9Hd3Y2//vWv+NjHPjbR6k07TAb+oIUCnKFsJLmjZhhwLKuSqT5QN2sm3MFBT54tnzsXROUnq5UhzuqKiD/CKxv18geLsL1nZerhD1aHpvAHbTJ/0Bj8MWLCaUv4I+GP5vNHsrLSYgztzGLVjffvFst7LHxDy7ssgqCN8A2swciEvWSfIAjz8uEnfgzPdqkhGRYavK1nRkjGTwLmv2dhoJpoi7fK4ycVq1cHFZ6Dpodk/PvDm/jRANicGbJKwjGlute4okOJwtEBYkX7UyhgpasJIf16rUg9vcux7AM6jFoodRswhr3EXIF7RwHaZUAdDrqeqa2jMGemoViOl7SLkTG25pFf2gVjZx7qiF39PuzIwW7XkFs+D6kNg9AHqpsFu3//MvSXepE9cQFGD9ADmT2JRdH1rIWOP76GLV9dCmJTfHj5Bbj1ie+iaRB9ueuR2w1wxBFH4NZbb51oNaY9JhN/0HwejmVCaW8HMpnKaonW0Q67WATJZr06vboOos6cCbt/AEoqBbRlQBTP6mpz58Lu6xNGO+PZxjirK0DV5odXKtjs9jw7zOMc33aLkkWKuEDEH6wj0zT+EIwTmz9S1YSQdfFH3kKpOwVjuJTwRxy53QCt4I/EWWkxVt30AFy3iQckDR1QVZC2Nqgpb3OVncsB2eHmjVEnZKsovsEOT24QVI28n0Xeb+fAcxhcVJ0GlP/WmffsjBOb3MtEMIGWjuqSOmtoKapf/BLz3neiKpnnGRlWBzZuSFgmrLfBvBfdB5tUZ1RceDNk/j1hz9kQJomXaRAv5j2FF9XFqhKSrZcJhADQCPSip7GTgrc3jFJQhUAxXSgO9fRu16CUD/I6qopUtpzQyyBwUzqI4wIqAXVd6GWiMWdmQFwKQimoqkAbKFS2D5gLOrw8DARAyUXbK973s9SbhtutV25e6tURdDzRBxdA4XWz4Ga8zNcYLGLr5/cCAMz5r+eg7tkLmtFAchby2UFkP38Asocs9bpRCV7Yvx3NBKHedcWR2x3w8ssv46c//SleeeUVXHPNNZgzZw5WrVqFRYsW4cADD5xo9aYNms4fcUAIoCpAKg2tuwsAYOfzIMUSKmdZTG/rj5MvAJoGoqoAIaAKgTbLm7m2C0Ugl/P6ozSwJch3JgDxNqcw2MS8vL9ZsOcS2dUHlj9YW+tzhO8YhM+rsEmNwcioEhmfZ0QcxurmIxZ/QMAfJCZ/lMq8YACgBHApKCFQbCrhj1JZhsA1dCiO60WPoy60hD+kcrsDWsEfibPSYjx5/zNNjZWvzpoVibKitbfDHidnhTebA0GZXy6q8x0Tv41vmP2Hf38sNl2Tv8rCIyV/FQUhGX/1g5f2KRXSgR1HpIOoP1YmrIMCj1Q0zldBo3wZn5AqujFsZZi0qrcTlNHKe5QJUNlfDABqCbAzBFrBn5dj9M7ZjA7VgVSTgqYINE54SWOgAKdDgzJqB+8dBVJbRrnfldSuInJ7zkD72r5AuQIg82I/Nl1xCEpzg3OWOy4PG7d5EV0Ke2ciZWPCNJ4Ze+ihh3DiiSfiLW95Cx5++GF84xvfwJw5c/D000/jhhtuwG9+85uJVnHaoNn8EQdq7ywQPWhxtbY22MMjAM+Rsm2Qzg6QdDp4jiWThl3eFhYZg3lf7463MAeE/+bZH9+R4PGH6EyKEvqfBXumUcYfCNXx+MPnGZEOY+YPRjYWf5iAnVGhFYKbtuvhD7WQ8EfdclMcreKP5MxKizGazTe1v3qzhE4lNPr7jHMHZHOTcfqrJcOrjzNpInIO40KadVowlqtJzIQT46IkF+Tvn260P0cWwSFBQ/jSl76Eb3zjG1i9ejUMw6iUv/3tb8eaNWsmULPph2bzRywogt+/zKARMmm5KuGP+CCOfJUv4Y8EreKPxFlpMTq6+bHC44IXCnKqo1HzEucOyL7orZgA4dXH4W5/Ob9ZkCZyE4ylyPI8CMJYSiHpTinFY2RVFBs7JqbzAclnnnkG733veyPls2fPRn9//wRoNH3RbP6IBdE2NJlB44Q6nixI+CM+qCp/ZEz4w0PCH83nj8RZaTEOPmZZU2PkU9OccBKIu8IZpy9enSoo99vz7rYsCZcofr5Mhzjl/qF5Xn2cRGBuijSsN3Ecsd463xwoI+L4+U5XWqiD06lxZfRto8IkZm2vmPJpTAHa/tXkGWg6htcUx4wZM7Bt27ZI+bp167Bw4cIJ0Gj6otn8EQe0JOCcri6pDBeqMq4/E944mqDcby+yw83kD5kTMan5w7bFeif8UUXCH5HysfJH4qy0GCd84u1QRMvoMUBzOX5m4AXzmzaGDDIjK7MRojpZQkZZskiRTK1yno2LkwgsTiJJ2b2TJgIjfB20EhXKWBmVr7cp1tFV+dopgHdwkafD1lGxDnPauDoYfaMo7DsDlPOz6PrjpoaX2ohDccArosDY8TCdZ8Y+9KEP4Ytf/CL6+vpACIHruvjrX/+Kiy66CB/5yEcmWr1phWbzRxy4eQHntIn3+XuJJKPlfgZ77jgSHZq5ekEk9bWciGbxh4zDJgV/pBQ+f1hiHRP+YPpM+KPp/JE4Ky3GjNndOPHMY5q2f5cWS6C2PaGrKyJjH/fLFI6A4kPmENiCOpnRlk1eiGam/Fm4RnSQ6e0PEq7ziYgbvEA2o6byZYyCIyaBDo2rg1akABERIuXKKADs2fzsxKmXh+Hq/OzE0DTuQMaWEbS/aNc/O0aB7idN3PLot+sUmHy47rrrsGTJEqTTaSxfvhx/+ctfpO0feughLF++HOl0GkuXLsWPfvSjQP3111+Po446Cj09Pejp6cGxxx6Lxx57rG59vvnNb2LPPffEwoULMTo6ite//vV461vfiiOPPBJf+cpXYl1jgnhoNn/Egu3ALRT5nGMI4m9RCprLcWXi8EcctpOuKgjqfDssc1imBX+UQxVz+aM94Y/JhOnCH4mzMg444l3Lm+pcOP0DgOtG+hzP1RUf4auSrYaInBJ2Wb7e1RIdYsIRyagIJn4M14n6EY2jo3G9FSpO3CWaOVPghZrkyaiOWD/X4JcbozZcQ+HW+V+pyDgFB/bMDL9uZwFOJhprTQFg9aS4BNb2/C7kD53H7W/u99YjtcutTTgUyGxykLnjuRoNY4CO4dUAbr/9dpx//vm45JJLsG7dOhx11FE48cQTsWnTJm77DRs24F3veheOOuoorFu3Dl/+8pdx7rnn4re//W2lzYMPPogPfvCDeOCBB7BmzRrsueeeOO6447Bly5a6dNJ1Hbfeeiteeukl/OpXv8IvfvEL/POf/8TPf/5zqCovpl6CVqLZ/BEH7tAQwJkk03p7xTIjI6ClkpSnGuGPsQRw5tl7mcPCK/cjUYr6q8UBU44/NH65kUv4oyYS/mg6fxA60VZwN8Pw8DC6u7uRzWbR1dUF13XxkX0+g+0bdzZ3IFWBMmMGlFQKlNLKzBulFM62vhrCzYH/xQnP6ojCBNdT54daZPvjzVr5bfz+eDqEPXG/zk8WGZahzN/hOj/RVrgvimqCsEb0thGMh8/KICRX0dsADLMxvR3di6HP1TutQik6dd87CsCemYE2UODK0B4D6qAZqSvNSkMrWFDzTkTv/CHzkPpXP7QRC1QBSPkG2G0Gdn5mf4zur1cFWDZ2ga4nSxh+dSf0ty/E6v/3QSxevBhA9DfYCHzZ5ad+E6qRbkgWAByziCduv6TusY844ggceuih+OEPf1gpO+CAA/Ce97wHK1eujLT/4he/iD/84Q9Yv359pezss8/GU089JYy04jgOenp6cO2119a1DH/55ZfjoosuQltb8HB3oVDAt7/9bfzXf/1XzT4SxMO48UccEAKluxskky7/yXBO33aIwlUpXZ0g7e1MNwSUUljb+vg2C2KO4CVjrAWeVrX4Q2S7gWqixobscEjvKcMfKqA40b4S/uAj4Y/W8UeystJirL33mdYQjePC7R+AvWMnaC4Ht1TyDkIWSxUyaTVY4wZUjV4lWy5Hxp/R4tUpoTrWiNqoZu0F87+/YmOHZNgkjWH9/Ez1Vqjcf+8gONtEUV2VCevm3wdHoLevW1hvFXy92W0IEb1NL6ejqZNAuQ8XXobiwH2wADtF4KTVqN4lBzYQqVMAOBkVZrsKqgVn5NRsEWanAWtmBlQhARk4BLYBWLMzoCpAiffSCjbsrjRKC9pgzcvAyShwDQVOhwZ1KAeas5B741wU9u1BcY8OFPfqRGlpF7r/bwsWXPICZj5SQPo1B6ntLjKbHcy6P4c5lz2H4eUp4H17wOoh+PApV6OpGOPM2PDwcOBVKkXDzZimiSeeeALHHXdcoPy4447D3/72N65aa9asibQ//vjj8fjjj8OyLK5MPp+HZVmYOXNmXZd+2WWXYXR0lNvPZZddVlcfCZqDlvFHHFAKt1iEk8uBjo7CLZa8gC+lEkhGfH7FHc3BGR6GO1xeaSnLqB0dIIoSsbU+f/C+zb5dF02a88pZnuLxh//wz+pAmHHCdliHZ7sb4Q8FVQ4Ly7COU1gHn1eaxh+kQf5wyvyRisrI+UOD1aaCqlH+sDoS/kj4o3EkSSFbjHt+9iAUVYFbIz55bNg23OGR1vTNATuz4UO0m1o0O8Z6yOwsGgnJsMZclKmYhPpjZVSmv3DiLrY/ti6sM9ufwZT7M1u1dGB/YOw4YZlaOvh6qy6glnOlOExG+8pn4EZ10EtVKrQ6dOijnlEitKxDOcN9QO+CU9HdX50hAIhDkRqpHka0e1JQB0uezHDZoO4seP2VnyzUvA2VSSqZXzYHbc/sgGK60Ea9767+1HYAwJZLD0Z+T1U6jVLYsx14B5PoywVGjp4tFoiJsRx2XLRoUeDvr33ta7j00ksDZbt27YLjOJgbOnQ8d+5c9PXxV0f7+vq47W3bxq5duzB/fnQr6Je+9CUsXLgQxx57bF26syu1LJ566qm6CStBc9By/qgDyqyZIIYR+U7YO3cCVjTZHwAo3V0gbW0RGWvXNpDQWWbRT12UjT5sa4mkDky9XxfmHLYuzB8+wvwhsusy/mCvJ8wfIpmm8wcFVCsOf3iw2jXoOe8zl/NH9XsR5g9jNOGPWkj4I4rEWWkxtm/cMaFEM9khcnRk9bVkeHAR/8seRwdevUwHUX+ybQ/E5TuPsj6lMe8FcNrTUIeK/DFKopNIAFzBfTDETGJ1KY2v95Ky3CTC5s2bA8v4qZQ461jYsIuMvaw9rxwArrzySvzyl7/Egw8+iHRavuLa09MDUk7m97rXvS7Qn+M4GB0dxdlnny3tI0FzMRn4g6ga//toS377gr3primexIoDmf2DoC7OM2Rl9j8Gdi/+aPzuJfzROBL+iCJxVlqMiY6RnyDBWNH0b/AYZpzGrU9KES9ltCfT1dVVc89xb28vVFWNzILt2LEjMvvlY968edz2mqZh1qxZgfKrrroK3/rWt3DvvffiDW94Q03Vr7nmGlBK8YlPfAKXXXYZuru7K3WGYWDx4sVYsWJFzX4SNA+Tgz+a9+NqxU8/weRGwh8NyiHhDx4mlztZJwYHB3H66aeju7sb3d3dOP300zE0NCSV+djHPlbx+vzXm9/85kCbUqmEz372s+jt7UV7eztOPvlkvPbaa2PSdcE+86BqU/I2jwtq/ZxF9Y2aAX9vchzw5OoINNKQDqJyqYwiJwKenCNI3AVA2JkyUhTq4KbF8x1U5YedVE3xbJo+WEcUl8hAgD4kmaGLgfGIk28YBpYvX47Vq1cHylevXo0jjzySK7NixYpI+3vuuQeHHXYYdL260eTb3/42vv71r2PVqlU47LDD6tLnox/9KD72sY/hgQcewKc+9Sl89KMfrbw++MEP7jaOSsIfjYE6Dj8amS6Z6xSsuqh6cx9da/XG+zkSQXmtcRL+ABxNIpHwRwUJfzSfP6bkU/SHPvQhPPnkk1i1ahVWrVqFJ598EqeffnpNuRNOOAHbtm2rvO66665A/fnnn4877rgDt912Gx555BGMjo7i3e9+Nxwn/hf5hI8fAyfG1pvJCplxi3OV0mRWgvF888aTsQQytRJJiiBq74eq5Br0GDrwElzV0kNxGyciY8QS6m3NMLgyavmCuMQv2QpiLmzjlqee3Ql7hsHtr/uvg41bJQXovq/JEfDqPQzJezWACy+8EDfccANuuukmrF+/HhdccAE2bdpUWS6/+OKLAxFYzj77bLz66qu48MILsX79etx000248cYbcdFFF1XaXHnllfjKV76Cm266CYsXL0ZfXx/6+vq4hx55OProoyvEVSgUIoc9pzoS/mgMNJfnJ4WUhS3O82X02fPEMvHUa5g/auXvGg/+kOVfaWYiSVl9LP7I2UK97Y6EPypI+ANAc/ljym0DW79+PVatWoVHH30URxxxBAAvic2KFSvwwgsvYL/99hPKplIpzJvHN5bZbBY33ngjfv7zn1cOEv3iF7/AokWLcO+99+L444+Ppe+yow7AHvstwJaXtoG6u8ciOIXcmPIg2jvrR10R9UeZduH+eHufDYmMheAh+fA4ousCp86PSiMiQ5EODvg/OtUV62AZBLoZ/e6wkV8iQQ9csQ5UJ1CsaH/GoCmUsXsy0AcKERlt2IJrEMCkERl12DvIz7t3hb1nom1tX8Q4tz3wKtT3zYLTRuojHRcw+l385anr6mhcP4iLShjMRuUawamnnor+/n5cfvnl2LZtGw466CDcdddd2GuvvQAA27ZtC8TMX7JkCe666y5ccMEF+J//+R8sWLAA3//+9/H+97+/0ua6666DaZr4wAc+EBiLd0iTh3w+jy984Qv41a9+hf7+/kj9WB6+JxoJfzQOWiyCOi6gkPqTU9o2qGkCuh6REXHBZOAPNvdJK/lDpncs/pDo0HT+0Aj37Io+kvCHj4Q/ms8fU25lZc2aNeju7q4QDQC8+c1vRnd3tzBcm48HH3wQc+bMwete9zqcddZZ2LFjR6XuiSeegGVZgbBuCxYswEEHHSTtt1QqST1HQgg+e+0ZE57Uq5mIs7oiW472+wvX+wZdVCdcIRDIGJwyH6JEW77esm0APN1E/UlXZFS+jGFS8diCxF0EgBsKO1yBRcX3rsfgyugDBbiCZXm7I8WXGTJRWtLJ/b6kn90Bc0F7ZEVJocDc28q/y1o/GRcgLsXcW8a21Waicc4552Djxo0olUp44okn8Na3vrVSd/PNN+PBBx8MtD/66KOxdu1alEolbNiwIXJocePGjaCURl71EA0AfP7zn8f999+P6667DqlUCjfccAMuu+wyLFiwALfccstYL3dCkfBHPLjlbXIRPWaJo/s4Q1mg/N1jYUiSF8v4Q4Q4/CFb0RdllpfxR61xePwh0zsOf7iEL9N0/rDF/GF3Jvwx3pgu/DHlnJW+vj7MmTMnUj5nzhxhuDYAOPHEE3Hrrbfi/vvvx3e+8x384x//wDHHHFOJX93X1wfDMNDT0xOQk4WBA4CVK1dW9j53d3dHQs4BwN0/faD+GakpApEd8FcceBA98IfL2feiTMNh54d97+dR8cvD7ShHRpPIsNckKuf1x5OJzFKxMg5gMw5L+F5xdbABN82XUW1aMeYRvZnpucC9GzRhd1cJJ6CDQyNx8wHAGCjCnNfOlUlvGEFxr46obiUXas6GuaDDk2NuTPsjr2H+rbu8WSbel4l65cSmmP+TbRh6p3hrSWzwlufrfU1x3HnnnbjuuuvwgQ98AJqm4aijjsJXvvIVfOtb38Ktt9460eqNCQl/xAMtleAODnnvGedDS6UAUThS24YzMFBxWFi5OPwhgsymivhDJqOjcf7w857wZET8IeO9OPyhUsBhHJaW8gezLBW4dyMmnM6EPxL+aD5/TBpn5dJLL40cYAy/Hn/8cQD8EGu1wrWdeuqpOOmkk3DQQQfh3//93/HnP/8ZL774Iv70pz9J9arV78UXX4xsNlt5bd68OVC/a+sAHrjtr7WX8HUdEI2jTpqPqQIZddZyWHh1/hWK6ngzTbwkkj58wuHNdAH82TMdwSRgYb15Kza1klw6nNkun6S4ejt8HRRfB85sl1oU2zniAhaTtKsiY5f15oSB1LImzC6jQiwBHRwKqycT6S/Vl4PTrsGeGQ2xmHp1FMU9O2D3poKEPFSCuj2H0UPnwVzYHpBpv/9VLLj0Jcz4RxFKKXhlaoFi5l8LmHfZC9h6zgKMHGjg/+3/Kc7Vx8d4HJCcrBgYGMCSJUsAeFFpBgYGAAD/9m//hocffngiVRNi2vPHOIAWi3DKSYipW7XUWjoFzOjmC5kWnB074Y6MAoyMvmA+1Hn8iEW1HBaRfYZArpazIOIPB43xh+9giPTjyfiTbkL+aFBvjY4Tfzhi/lBHTFgdCX8k/NFc/pg0Z1Y+85nP4LTTTpO2Wbx4MZ5++mls3749Urdz505huDYe5s+fj7322gsvvfQSAC+km2maGBwcDMyO7dixQxhZAfD2MctiYP/5+vtAQMBbiFV6ekA0FdCqcexdy4JbKgDdXUAmBaXdO2TmOg4wkAV2DQKaDpLJQC1HY7GLRWBgsO5rHwso+I6Kf3W+IQ07H+yyO1sX3ovMJu7y26sIHqpnD9GrzPtwejI/toVblvF1UJj+fIfGX3pPMTL+YUtfxhDUqYze4QP+Go3qTcvjV+4DAZxyO0K8GbKKjOIZMEq8cy16ea+wo5X3PFPAUQh0i1aJNEUAl4ISApVQGOWkXS4AO62CUAqXKNCLNojpfUJmhw7FdQFC4IIgNWxW792MVFmGQM+WoA96e4/tdgNuSvX0A4UxWARyticztx1U805VatkiMptGyzIqzFkZEI2AlFwYW3LoWOvNPJcWdMHuSoG4LpyRPPq+vi8AQPnpy1i4Swdt10BGLGw5WMXA8XsAR+3vXbACvHaE+LBvLIwx9ORUxtKlS7Fx40bstddeeP3rX49f/epXOPzww3HnnXdixowZE60eF9ORP8YNqsdRyKShtXl8ZFsWMDjkOSCUAjY/OSQAwHVBR0fhjI7C1jRoimefWBkb1Qdq9lxGLf6gTB2YOvbhmyB4qN63w6TcPsw5vgyPc8J85Ger9+tYzmGdCZWp43EYTweWp+Lyh10W8GXGzh8KiAu4CoFedKr80aZBKTOpS1FJ/OgCsLtTIH5/wwl/1JSb4mgVf0waZ6W3txe9kugiPlasWIFsNovHHnsMhx9+OADg73//O7LZrJQUwujv78fmzZsrGTuXL18OXdexevVqnHLKKQC8w0nPPvssrrzyyhhX5OGZR9bDdTnzPaoKJRNNsqPoOmhaB2bOCMTYV1QVbk8XVCc6M6il05EH9VZB5KyEDw/6YMmDCOp460b+3tzIwb/y/7wvrh9hhacDm80+LOOTjRaSYfcbI1TnE1AY/gFNnt7hMSr90Wp7dmaFwHNQSKgc8FZGKn06wUqtRMs6BMsVAK4CaPno+rgxalVah+8DcQl0P7swO07OhK1oUEfsyP02tue490HLOSjt1+EdjgwhtXUYfecfAnNO8BvhfnxvbIm0DqK4WJ6wqlHEneXaHWbGPv7xj+Opp57C0UcfjYsvvhgnnXQSfvCDH8C2bVx99dUTrR4X044/xhHqrJkgWtByaboO2zI9p6MBUIFTowne18MfgJgnGuEP0cF236nh2Xt/soynm4g/avXXTP7w68L9xeUPCsINHWzkbSF/OJRAG+Hzh0M0KKMJf7ByUx2t4o9J46zUiwMOOAAnnHACzjrrLPz4xz8GAHzyk5/Eu9/97kAkl/333x8rV67Ee9/7XoyOjuLSSy/F+9//fsyfPx8bN27El7/8ZfT29uK9730vAKC7uxtnnHEGPve5z2HWrFmYOXMmLrroIixbtqwS3SUO8iPRCBgAxFu+AECUCMxxJnzvcisgcoBEiHMHRNFf4vYXRyaOHWJnEpsBUuPhgrtqpoqvljhxLLK4ihox7iwBHF6YngSxcMEFF1Tev/3tb8c///lPPP7449h7773xxje+cQI1Gzt2G/4YT4g4J8ZPfzJsaG627Z5O/FHLOW2UP5Dwx26HVvHHlHNWAODWW2/FueeeW4m8cvLJJ+Paa68NtHnhhReQzWYBAKqq4plnnsEtt9yCoaEhzJ8/H29/+9tx++23o7OzsyLz3e9+F5qm4ZRTTkGhUMA73vEO3HzzzVBV0WNubXTN7Kiu3bKQLfc5Lv8Hqao190BPRciuhufIxDHask+wUWepHh24KzK870EN+CLN+sSpIt8RztVbkoSLauzO6johmShWiuU9b41csAuoxSZPSbH7SBqV282w5557Ys8995xoNZqG3YI/xhOu620FC4OQhretuIq3WjxeaBZ/yB74pxV/yBwPwVgJfzQot5uhWfwxJZ2VmTNn4he/+IW0DRt5JJPJ4O67767Zbzqdxg9+8AP84Ac/GLOOPg47/mA8fvdT0T3HjuMtiatq1Pkwba5DomgaaKkEpFIT5rCMZVSR0Wy0XLRUDohXUGQx9x1U9xPz0Cy9FSrWoZbeXNJNEZCS2LpxtxMUbPG9SytQilEmUHKS+PldKWjDVlgEAOB061CzVkTG2DzsJR3jkE77SyWU5ma4/QlBgPb1+cZkanU5zZbxv//979fd9txzz22hJq3HbsEf4whaLAXOVVbQMwPoH2isszE4Kjy7Jcq/ImoPVLcLN4s/XMgfpETbuqYmfzjie5dSoJQa4w+nKwVtJOEPVm4qYjz4Y0o6K1MJx330bbjhS7+AVYru1XVzOShdXZFyoqigJRMwogm1KAEUXjbhBfNhb93WPMUFiOusiJajpcZUMB67NtAICcRJBGaD78jIknCJQBAzgZlgNlItUaEOVpsGPR/9zqlUcr8t/pOEAsBu16HmoqRivDYq1NualYaWjcrou/LI7z8TmRcHIoTT9adNGHjLfg190YhF8eZCc/ccT7cDkt/97nfrakcImfLOylSCjD/GC24+D7WjPVKupVINn5WUTQrJIEqGGGeLk2+H/ffh/hrlD9FZSUDMHzLeE2E8+cPOqNAK0RUPKe/F4A99S8IfEbkpiPHgj8RZaTE6ZrTjvZ99F3511R8idTRfADo6QBUl4JQoigJnIAsyf3ZUZsFs0G0DgM6Z6RonxFlWZleweQadN9slM4wu+CssviPD00+mQxxHxgL/QH09RBSu84mIR76ybMKO5h2WDPfnH3jk6VDq1JEasSJ1mgO4KgGcaDZhYoqzCZfmtyG1LTozlXplBE55ti3Sn0MAeFGO2Dq9bxSdz5oYWWbU9yWjwIzHivj+ny6ro3H9mG4zYxs2bJhoFRJwIOOPcYPjgBaKQCYd5Zx0CihGD0/L4NvNRiDL6i6ym/VMhIns8HjwhwMxh000f+jlFRSeDma7DiMX5Q/Vjccf5rw2GH0Jf/hyUxHjwR+T4bzbbo/XHbY3v4JSOP3VpFksVEUD3RldYlc0DU5X2iOQkIwmyQzcTLAzU43I+Aj/HlmDLvqthss1iLP8isoUAKagXpQ8DJJyQ1In0kOBOENyONElW24ZfBnNFt87l5PQCwDSIxactmjsfACVA4+R+2OBGx8fAFLb8rC79UidAsDNaKCczMWZl/qRP3guQBBI6AUAc77/HNKvOXVt7G5/0UbqgX/WaJggwdSFkD/GEW52CDCtKOfMnCkOCCOAn8+qEbCOAs9u1gLP3ov4g3DK/HFEtjsOf4gSP7KYSP7w7XLE3ucsuJzcKwBi8YfRl4fdlfBHAjmSlZUWw3EcXHfBzeIGtg1n5y4oM7pBUqkAGSgO4G7bCfT2gOhapY50dcIhBMrACJAOLl+q8+fB2SbOmNws+CsYjXi7MsIJzwyx9ewsGAsV1VUZVoadHSMhWQOew2IIdHA4OrDXyiMIf4asXr01ePHv2RwsLFjd/TrdBCwd0C3BvVMAxQ3p7QZn71ioeQdOhwZ11A7IVO4dM0NW0WGwALMnDWOwGO0va8Gck4G+oxDoTxsyYXVqICqBNhRc0m9b24fCsjnQ+oZh7Cx62ZJdgDguFlz2FAbOPhDZg8vJxdibqQDEpOh5tICBrgGUzjsYGzduxOLFi9E0+NOKceSmOD7xiU9I62+66aZx0iRBTf4YL1DA6e/3ti2Xc3/5UOfOhbNjJ+DUf1A6bDfrlak1qSXaFsyTEfGH/943OWwfGoI7AcJj7U78QahE76IDp02Dmm8SfwxbsOZkoCX8MeXRKv5InJUW4+9/WouBbYP8Sl0HUVWQVApEN0BtG7Rs8AkhoJRCURS4O4dA4XpL7qoCuC5IyQJSBtxiCXBsELUc8d11QVKGd+alxWCTc4WdFtZgsvD/5i0x+0YxTGJhEghvFQO8rQXh+O42+MvsfnZiF/ztCE65fTiOv18evi6/jtXHL2f1CuhNy9dKqkm8fPj3M3xfNcsjFUcDNJO/FQGq5+RWxgHgprzlcsUMLjMr5cP2docGbbS6+1wBYLdpUHIWYKhQitUO9VETTobATaehjZRAHG+PAVUIiO3dB2dBO/RdBS/MJfG2NbqqAmuWAtqmQR328jNQXQHJF6DsLCL/+tmASkFKNighcNt0tP3fZiirLCjL90BxaQauQaCUKNpeGEW2bxcG/nMfAG2wAXzoI9fgbw9fg2Zhui3jsxgcDNory7Lw7LPPYmhoCMccc8wEaTU9IeWPCYDrukChAGJZILoBLyKYC6LrFe6qF76tFDktvO1ivs3jnTlht2/xbDfvPIqIPwiqTkl4W5jPKaJzNCL+YHXmbeGiaID3YvIHBWAb9fOHAo8/QCmIFeKPoscZVkaDVgjxR1qFUrABXYVSCvFHmsDNRPkDZf5w57dD6y94qzRKmT+UhD+mClrFH4mz0mLcf+tfoKgKXCe08K0o0GaHkpgpGoimwc3lgbZM5SC96v9ruaAlG0RRAKJ61ibj1cY5XB823PXKICTHOi2+IQ47L+Gx/HY8xyQ8M0UQ/KL6KyOsHjpHhiW68EqLIqgLkyY7O8bKsDNyIhnC6Al4mYYVGtTb/1t0rX5dheBcQDWj/VUOUDrR/lQm4ovVqUEf8YiFlNsqnNUV3Y/QUnTgGAoU09s3TCwXigWg4M2AWT0paIMlKC6FMuCVaVtzXn/EM75KlnGc+0vIv3Eu2p7ysojrA16d9vxOAMBrlx+MwiJV+sUc3X8GgBnVAhcYWTFLLBAHLm044V1FborjjjvuiJS5rotzzjkHS5cunQCNpi+E/DGOUHtneRNrofMqcTiHN7ElWl1h7TdvYisMmR1WmfesE1QPf5BQedghqpc/WNTLHywXNIU/ABhj4Y92DXquzB9lGb3A4Y9yG5QE/FEsZ7GfYUAdMgP8gW0MfzgJf0w1tIo/kjMrLcaOzf18olEkt15wcN5xHM9RaRIadVRqycXpr5YMrz7OHRAdmqwHcXTgycSyXYK+AHgzUg3qoAgitsjgtBtCHdhZswgEylFN/EnYnUrjH5QC2B1NNmV0DK/dEIqi4IILLqg76kuC5kDIH+MJXnj9mBjvnwdP6zhXkvCHB8VuLn8QU9Jfwh+7DZrBH4mz0mJoevyEYAmah+ZQbRWxbEqzlYgBGuOhQ7o03exrisPIFNyY+wmai5dffhm2PXEhdKcjJgV/TNFwqpMZU5U/RBOpUpGEPxJg7PyRbANrMRbttwDPr3kBTnhGohzNiztjJSAHVVVBbQdQlabMdMXZBiaTq9Ufr77WIX2eDO+8Sy0diKQOknJRn6JQlTIZP6lXI3rL4ufT0P7ievp0NAWaKGMwu9+BgZotCvVz2zQoOb4BohoBbE4YS078fh/GgAurR2lsGoUC+kBzH6IJYu45bqoWE4MLL7ww8DelFNu2bcOf/vQnfPSjH50graYnhPwxjhAmL46BscyOxt223Ii9rWWHdyv+YA7V16uDrSlQm8kfmYQ/wnJTHa3ij8RZaTFOPPMduOuG+6IVlIIWCkAmEyUBQxyF3i0UoHASdMHQAZOfCVaERg2sLyNyFuL0Vyt+vix2PfdhHOLkYXGcHAq+3asVP79RHahaPUNSr27Ekdw7jUC1o9bSz6/Ck7FmeJFaInpTwFUI4PKIQ2zkzYXtSL06GilPP7/TS/jVX4z01/1QP3L7zhX2yYUCdN/T5ISo0yypF4t169YF/lYUBbNnz8Z3vvOdmpFeEjQXQv4YR9B8Hko6mjQvTiLiuA9jwoddiJ9LayVxbMSJkPGeTL/x4o9ApuQ6dVMk+VdclUB1OPwxKuGPbgPGUDSwj4w/lIQ/onJTHK3ij8RZaTH2e9M+2PuNe2HDs5sje4/dXB5aW1tERtF1uIUCkOYk4So7MuFVGa23t6kZ7OOcJanHkQnXEbQmCRdPB1EysnocGZ4OIiKSZTQWEZHqSBywcvSWcJ00AaZNxffOUKBy9gqzISUjM2ozMjAGoom7tFEbTtqLGBaW0fo9x4end2HxDGj90RDbmUdeg3bKHNgdpL7ZMRdI9bl4eP2P6mhcP6ZzNJcHHnhgolVIUIaMP8YLtFjyIn0pE7eiL+IPmYmQZnVH444MeyieN85E8ofveHD5Q69GAquXexUnBn8MmQ3zh5pL+CMsN9XRKv5Izqy0GIQQnPO9T4C6HKKxLLijo5FEWwAAVQVcN5osMpWCOzpaCW0cgGRFRgTZb0NU5xu5Rvplt2Hx6ngyfgx6WX88HUVJs3i5VXyIknMpkjq/rBEdWDKM6KDyZXRTfO9YPVgoAFxd4dYR0xXqYPWkuDLGQB6OwTcXboqfIEwbtVHcu4urd2bddpQWd0YSeikA5t681ROo9SVzPads7k0bazRMkGBqQsof4wh3cAgA+FzVIOK4OzL+EN0Zlqca4Q8R5+icMh809L+P8eQPKkgCrFvieyfiXoKY/NHdOH9QI+GPBLWROCvjgN9c/UeIDqa5wyOgo+VQfWxCSMPLu+In2mLr1M5OOMMjkXKtNxQKuQ7IHoBr1fGMcFiGfS8y3DKZcKZ6vy6cqIp9r8NbRQnLsDqEy9kMxOH+RNmJa+nA0zuMwDgO4Kj8OtG98+LgC/qz3ADh1HPvjMESrJmpiAzgkZRPOIFrzZowezPc/jIvD6OwtDOqt0uh7yyguLTbq2MsUdvaPiy4oQ/EpvybVy5TShTzv78ZAx9YhKaDjuE1xdHf349Pf/rTeP3rX4/e3l7MnDkz8EowvpDxx3iBmibcgQGAUtDyC/C2gsXqL4aMzAaKTjGEM9XXYwP9xI9hGVYHUTmvv/HiD8X18nBx6wQ6EACUia8f5Q8S0UHKH9kS7O7G+EMbNmHNSvgj4Q85km1gLca2Ddux5s5/SL+Ebj7vzXwYOmAYlaV2Yhhw8nmQ0VEobW2AUbUqSnsbnMEhKIYOtLVVQhrH3Ufsz7zUW0cQdDLYNrW2g1FE+/WNKZsx16/zCUcJ9ee34yV/1CFPTOlvCeON4/fNgk0yCQT1ZmXC/fF08K/dT0QWkJFsByMATJ3AsGhQplTWgXNgUrFcWBkVWsGJ3DsH8JJs5e3gvRsowW5ToNoIhJZUAFDThTUrA3WwGNiDbOwqwFUBZ3YG+vZC4PuefmUEpXkZEAUwthaqeuds6C9nkXvDXCj5ItIvZyvL4JnHt2HeC/0w/30xht7UBqeD2fKYpZjx6Cjcx1/D1q8cALjAcUvOwj0brkezQCgFiTGLHEdmsuH/+//+P7z88ss444wzMHfu3KaFrU3QOOrhj5pQVX52eYXA21tfXze0ZMLZsROkLQOlvd3rF2M7uyLiHRHYB2We7RZt7eIlkZTxB5sMcirxh2w7mIg/FLPMvVz+oLDTCtSiy+ePjAa1EOQPLVuCnSZQHQJi1ccfen+ZP3oz0Hck/DGV0Sr+SJyVFuOu6+/zMrBy9hsrvbO8zPPMXmDqUtgD/YDtAK7rzWQBcPIFLzcLKRNMeVuAWywCwyNeZnuQSnmjYL9OFNEkin55eFaGrWONcTjpIjsrwyZ9ZDMHh2Xscv8q08aFlxTSLydMnYPqjJjG6OKPQ5n2BtNfidGL/UH4We5R7ktnZExGwYHVZgAASZ5JREFURg3JsNnsWR1sVPdKKwgmIrOIp5tCg/fBUb1EvgSA5gIpi1b1Nrz2hHhbxXxlnRTggIAQArXowihHT3EBFNs0775RQC3YUPPeHbPTgGPoAAW0kgU971bH6TagqASuCxhDJej9hYpudlcaBARqyYSad6D2lZNFzkzDTamgrgtjZwHpcrmrAvneNDRdhZKzoA+a0J72Enw5XQaKPW2A7WDXIgLz7H29C3ryNSz4TR4ko4MWTGw9qx27TtgDOOEAr54Au94Rb4ZXCN9zjiM3xfHII4/gkUcewRvf+MaJVmXaQ8YfQhgGlI4OQFOhaFWL5pRKXoSlTAboaodSdjbcog1s3OxxiGFAYw7T2/k8MJSt9u26oKM5OKO5IB/VifC5QZY/RBno/Xq/PSvDZngP2GHF20JHCQGx3QjnKExfYf7wuaUe/qDw+CMsAwRNCGvva/EHyzlj5Q9X8fok8BI/8vhDgXemxR/YNQCHEE+mRKEX3co4hbQKtXwcRC04lQArjgHY5W3oWtGCXvSeFlwAZqcBohC4lMIYNuX8sb3MHzPSoGX+0Hcl/DGV0Cr+SJyVFuOfj73EJxpVhWIYkWKiEC+qF48AZI5IEw9fElQNXnhGJ1zG1oVnjXyIwj6yDkcYLJmEy1lDzcI34jy9w6TA9mcIxvL74+nAOkP1yJDyPwbnYyUANBrVGwCI410rTwfDFOhdAhRQkNCUqXffKLR8dJZVKwJq0Yro4N03An2gFJFRHYDaNtQRO6KDPsAPV6k4ABbOQGpt9HCkOmxi11cPhDk79Khy8B7YenCkeQDFhSl5gwYxnWfG9t9/fxQKhYlWIwEk/CGBkk6DpIzojKamgcyfFSlX0hqopkLp6IjUaW1tsFlnhUWMiTGqKQAnDDPLHyKI+IMH1QUoKJSQDWRXLSIyaJw/CKr2mWejReXjxR9wvbEa4Q/FBAg4kR8BGMRzUiJ6m4Bi8vlDUQj0LJ8/YNlQRjn8MZTwx1RFq/gjObPSYhRGo6FgAWCi9yDXg/HSsNGfZxy9ZLQap784MrEiGcYcSwRSI2kWbyyqiM0E4YS3rK2EuIrqMa6WALTx2BIJBLjuuutwySWX4KGHHkJ/fz+Gh4cDrwTjByF/yCDhlone0qfGSdrXZMSy3U3ub6ryR62ki43yBxL+2O3QKv5IVlZajO7eTn7kLpnlaXBpvVVo1NDFMYwUco+Z12ecOxPek1xrjFqopQN3VojUIRiCLyLSr2G91fBR0FA9p0/FEsfCp5rseKsAEs9RKVCgmzR2YS6gFJv8e4l72HHif7ZjxowZM5DNZnHMMccEyv1w6Q7v/EOClkDIHzJQye/bdSvnGycCtkagmuP3I2kWf4h2B4jGqEevWvWTkj80BSg1kT90BZAkeeQi4Y9JjVbxR+KstBgr/v1NeOzP66IVjgNqWd7SfGi2i6RToPmJ3YYhciJYe8kjgUYdjzjlcRJJ+mTD69MBf1tZXP1E5X4GYvB0INWl/IAMxHq7aQJSjC7Xy/RQcrZQP7d8WDIMTZIIzO5MQRvmJyO1ZxrQBsyITOqVQbgqgcKZVetYX0BpHifpqQwK0PHMSGMytTCNk3p9+MMfhmEY+N///d/kgP0EQ8gfErjFErSOjki5qqpwsiOgM7oinynt7ow3y90gUpz8HBUdIH/G5NpuQfJCWX+TgT9kTkQs/kCMRMQpAlpqIn+kVahFzhZjCX84HQl/ROSmOFrFH4mz0mK848P/hh997mco5aN7Nt1cHkp3V1SosxOYYGelljHlOSUyErAhPmciIwGRARYms5LoUCuRZLP0FoFArLcqIyIV0DiTEVqRCnWw2zRo+ajjISMvUhQTkdNhQBuNZidObxn1Yt1Tjg5dBvSBqIyWLSF34Cy0re8HCT27dP1xI/qPPrD2BnYGSoniPXP3qF+gDkznpF7PPvss1q1bh/3222+iVZn2kPGHEKYJatmApkadkpE8lJ7uiIgyuwd041Z+IuJxgmxUke0WOSqA5KEfE88fMjssgpQ/JDqI+EMtSfgjrULjOB5y/nAa5o/U1oQ/wnJTHa3ij+TMSouR6cjgtC++h1tHCwVvhSWc+FFVgVT08P14Qrbi7Nfx6kXzZmxm+TBECbB0QTk7TrhO5mTJ9K6VCKwRvcNx6Hng6S3OJizRIUW4/ellR4UnY3bq3DqFivf8KjlTqIO1gD+Tldk4Crtd5cqoOQdUCYcAALSBIrrXlhpaDu/5Sw4X/+Sz9QvUA39mLM5riuOwww7D5s2bJ1qNBJDzhwzuyAjX6SCaDjqa424rc9WyLQnVxc2lIgJ/Hl0OdnIoDFfh2yw2klYYVFA3nvwhkhlX/jAE/FF2VLj2vkPAH4jJH4KVkIQ/piZaxR+JszIO6F0oSIRDKZz+AW6mem3WrEoc+4kC67DQUDk45UCQVMJ1IgdDl9SJSCWcuItFOBY/W25xyv3+RGOJysOJu1iI7h17woN370TJw2xDoEOJVhKB1at3asSC065x61COwR/RgQL2jDRXxtiSg92bFpIRNcIxeYD0xiEUXz8bUEkkC3Hvdc+i/V+2nHDKdV3rTGjrXpE0TNAoPvvZz+K8887DzTffjCeeeAJPP/104JVgfCHkDwlosQinfJg1kFBYVYFsDiiWIpyjLJoPNxdNUNxs6ADsMZybidj7Og7t82xtHP6olVmeVyfjsPHiD0fny+immD98ROz9aJP5Y1sO9syEP3YXtIo/CG2lVZqGGB4eRnd3N7LZLLq6umCZFk5b+J8Y7pfsiVQUKF1dIJl0pMoZGABK0aXQ8YJvQAjkqy1h+MkdwzJ+OQ9+Ui+eDAFfBxPVmPc83XkyouRh9eodrrNRDQ/JllOmfaQ/4hnwRnRw1HK4Rw4cnUC1aEP3zurSoXP2C7sAkFKhlJyoDl1pqCNF7hfBXNAOY2suKmMQuD0p6NujkY1K+80CiibSr46UsxB7hyNdl2Loowdg6M0ZuGlSTRYAAAqgjlL0PDyKXa/PQu1diJ8e9A689a2HA4j+BhuBL/u2I74CTYv+HmvBtot48O/fiDX2ZIHCeZD0D3knB+xbi1j8IQFJp6F0doLoWsABcW0bSGnArBnlgC6+AEBf3QpV1UAMIyDjbIuGix0LSpqCFCeMsQwy/pGdvRBxgW+7RXK7E3/wkj76cDR4SYAb0MHq1KGPCPjDUKCYbkP8Yc1vh74t4Y+EP/hIzqy0GH/7/T+ERENSKUBVQVIpkHTKW2GxbS9+vUtBHcdLDlkHfMPabLBGKrzMzI4ZHt//urKJIsG0EWUHpogSCGu8ws6OFpJh9bHgOTJOSMZ/75ezOvj7ktlklb4Ovs7ha/XL/Gvl9RfWwb+g8H2gQGUPr028sywVvcvL+a7qvQ/cu3I2YCtFoJWqQgQATQEoAVQFCPN1UgveHmOrU4c+alX2yxICWIYCveTA6dCh5qzqRdu2l5O0wwCxHRDTBQEFVRWQgjerVtqjA/pgEcRyvaX6Th3qiAU3o8Ka1wZl1AJxXFBDBbVMpF4dQXHxDDg9KRDLASUANTQY97wKd7ATc802FF/XATdFoBRdZJ4dQd8bitj1rj0AdMIB8MXLfoU19x2OpmEaH5DcsGHDRKuQoAwZf0ihqiCqCtLWBqoqoPm8V06I91IU0Bmd3kRYdsRLTkyIxzfFEpxCFtA0KJmMl/wRFKS9zQv80qTvuO+olHQFKSv4FO2WbWD4sUdmhyuZ6BUSWG0hABxNgWq7Qi4I226K6pnJ8eIPXn9+21j8gRD3+kkfFe99gD8ouPwBAEgBtARQBYEzImrRW2cyO3TouSB/2IYKYrpwOjSoObt6IZaYP1A+g2ku7IA2xPBHhw51OOGPqYJW8UfirLQYD9z+Vy8DcTiBlqJAnRVa3i8TjN23veGEW812VNgZFf9/EioPG1yeA+J/wWiozjei7CxVOJM9Df3tg11N8WXZjMK+TIppE15hYMdj6/xzMux5GZ5TVa/ebHZlh3E+VAT/D8iU//Cjg4V1UMoOBzu75h+g1MtEE7h3/tlcB7A6NOijHsmo5QeEVHl2rCJDq2XKqAUno0IpOCAUlUP7SvmwpD3DgDpkQnFdGIPeA1H6tVGvv7J+bIQYdcMIcm+ci/antoNFeuMQsBHY/I1DUFyolC92IQAg2BLIHhpK4OUCI4d3o6kQ7UmoR26KY6+99ppoFRKUIeQPCZTOTpCO9uCZlbY2AIDT2w2iKlV7outAJg133fPRjmwb7khzoySVOtqQGs0HysKOChBdNWAf8HmJG4GqHfZzSbFcpZUdIxV8/gifjwzb7oniDx+x+EOgg+o7LY3yhxvmD69VajTKH8aozx92kD8KtfkjtSXhj6mMVvFH4qy0GP1bB/lEI9uzO0m8a5ED1Gh5nDFk9XF2O8vCS46X3i5tKEiJJwPJ9VL5ihr33tXYdsGTsTMGUoKMtEQSilRoeDXx3XPaSeMfiAI47c09fjfdMhD/4Q9/wIknnghd1/GHP/xB2vbkk08eJ60SCPlDBgG3OI4Dok7sMVVtnLc0J/wx8fzhpA2oCX/ULTcVMR78kTgrLYaRmrqpUVu1tWwiECf/iwxxTAoJb8ZtYKxmfQ60RlhS3ljSrPdxFJN158S4Whrc3pagcbznPe9BX18f5syZg/e85z3CdsmZlfFFPP6YvA88rkIafuCeDEj4o9xXHP5wJA5Jwh+7BcaDP6ZkNLDBwUGcfvrp6O7uRnd3N04//XQMDQ1JZQgh3Ne3v/3tSpu3ve1tkfrTTjttTLouPmgRVI1zmzkhiyuY4ChgtSCyFbXsKK++1pwhT0a2wioql9n5WuaNJ1fr58aTYXOp1NMeiG4NCMjoMfTWopFVIoOFoA0VhTJOu/hhiurR0JIAvDMwAhi7nNpfishAgLEjTkBUWZ80/msKwnVdzJkzp/Je9NodHJXdgj8koIKM4aqqgpbMlkb6qgW3FP932qxdNQl/lMvU8eEPdbiU8EfCH2PmjynprHzoQx/Ck08+iVWrVmHVqlV48skncfrpp0tltm3bFnjddNNNIITg/e9/f6DdWWedFWj34x//eEy6nvTJd8LhLZ1SCprPc4mDzGh8/+R4fcXZg3yiOh5kxlREHmwADxay/Cuin4JsnDhEZEj6swUysnsng5BALbEOrq5wdQifT2Fhdqc4peV7pwmIYyQaoaXS3x7RLNoAYLzQD3NuJhJyEgC679vZuFUiQPfdWxsUqgEKj/QafcX4IV533XVYsmQJ0uk0li9fjr/85S/S9g899BCWL1+OdDqNpUuX4kc/+lGg/rnnnsP73/9+LF68GIQQXHPNNY0rtRtjt+APCahgyw0A0P4hfkUX/7ffbMhGkV2l6Gclk4nDH3F4aqryhyz/SsIfY0TCH03HlHNW1q9fj1WrVuGGG27AihUrsGLFClx//fX44x//iBdeeEEoN2/evMDr//7v//D2t78dS5cuDbRra2sLtOvubtxxYLH0DXvh9SteB4WzV9jN5bmJu1Sj8YSQzd6uFccwymSooE6WAKvZjkytuPoywhGtCol0aJSIZDNgTopfJ7t3xHLFeqdVPhENloQ6WF38MIxqgcJt07gyet8otz8FgLlHN3eg9D+2QR+g9c+OuUBms4OH/vWTOgXqg7/nOM6rEdx+++04//zzcckll2DdunU46qijcOKJJ2LTpk3c9hs2bMC73vUuHHXUUVi3bh2+/OUv49xzz8Vvf/vbSpt8Po+lS5fiv//7vzFv3ry6dfn73/+OP//5z4GyW265BUuWLMGcOXPwyU9+EqVSA5nUJyF2J/4QQjYR5pbrwzlW9t57THo2Akfj7xyQXaGIW2QycfhDlkhyqvKHK8jRJXOyms0fdsIfCX+MkT+mnLOyZs0adHd344gjjqiUvfnNb0Z3dzf+9re/1dXH9u3b8ac//QlnnHFGpO7WW29Fb28vDjzwQFx00UUYqRENpVQqYXh4OPAK45NXng7K2/dv25XEXWGQ3ll1XQuLVq2uNGLkRDKihFUymXqTcIUhSprlz2bVmgkL61ArcZdshSdcxybAFI0dkSmVQ3kKxuKVKwDsjMqtI0VHqHdpRoorkxoowM3IMhJwPvMCRWnvLm5d2xN9KLxuRmR2TAEw7/pN3t7jWoTjAkqJYu6PX67RMAYoYi7jNzbM1VdfjTPOOANnnnkmDjjgAFxzzTVYtGgRfvjDH3Lb/+hHP8Kee+6Ja665BgcccADOPPNMfOITn8BVV11VafOmN70J3/72t3Haaachlap/xvzSSy8NJO165plncMYZZ+DYY4/Fl770Jdx5551YuXJlYxc4ybBb8YcE7vAIYEe3Gqu6DrrFi48UcWbGaXVFlYTjdwWzbn60SSD6ExOZibj8IbLDU5U/NFPMHzInsJn8YST8kfDHGPljyjkr/iGeMObMmYO+vvqSVv3sZz9DZ2cn3ve+9wXKP/zhD+OXv/wlHnzwQXz1q1/Fb3/720ibMFauXFnZ+9zd3Y1FixZF2txy2a9ABHeajubgDA+Dhma7VMMAmTmTLyRAzPN30v5YsH0rqK5asuUymbDD4teFVwjY9zqChBNuxzPQbHZ7kd48HVyBjBZDb5kOPvGGr0cBKtmEwzIqDRJOQAfCl9ELDpyU0pDe6aESzJ4Ut04t2HAyWkRvNe/A7qoeoAn09/IwigzhsDqkN2RR2N/7jlPmutPrd2GP72+GUiob7zDplP/WRinmX/kKdnxi/GaF60X4AZQ3o2SaJp544gkcd9xxgfLjjjtO+OC8Zs2aSPvjjz8ejz/+OCxrbPuun3zySbzjHe+o/H3bbbfhiCOOwPXXX48LL7wQ3//+9/GrX/1qTGNMNHY3/hCCUjj9/UD5O8Fyi2akQV/rg7/C4tcpe+8NGBP7SKBQwDX4D7Uy/hA9l8bhDxkXTHb+oA3yh4yvE/6YOCT8EcWkcVYuvfRS4SFG//X4448DAHfrlJ8dsx7cdNNN+PCHP4x0Org0edZZZ+HYY4/FQQcdhNNOOw2/+c1vcO+992Lt2rXCvi6++GJks9nKa/PmzYH6zS9swdp7n4HriN0IWjLh5vLe4UcmTKWSMoCO9rquyUerHBYeGchmuyj4s0MKvIRZohke3kyYn5iLN/sDeHHzeTIu+GdYCLzQ8bz+XIEO/iydSG//Zx7WgWcr/Zj3Nsc4K67k3lHATHF0KE8i2QaJyKglF7YO7gwUAJidekTGGCzBJYDDmQlTCjZKPWlQPXjQUhu24FKgtLAdVI0STqHLQGnP9oDFIRaFsXkYuf1moXDALLhMSEr9lUHMufgpzL5zGMZA8A6mdriY++tB2H/aiC1f2xv5pSqOWXxmRNcxYYwHJBctWhR4COXNKO3atQuO42Du3LmB8rlz5wofnPv6+rjtbdvGrl27xnTJg4ODgb4feughnHDCCZW/3/SmN0Xs22TBdOYPIVwXzsAgnJER0FIpOBlGNNCXXgXdugNgDuST1+83rk8FthE9YK2YttD5UBF9SAfkvOevRjTCH/4D+VTjD1KDPyyDf58oAFuPyrSEP7oT/kj4o3FMmtDFn/nMZ2pGTlm8eDGefvppbN8eTvMD7Ny5M/Ih8PCXv/wFL7zwAm6//faabQ899FDouo6XXnoJhx56KLdNKpWSLpX96Sf3QtEUuJxDksqc2V6WYYYkKaWwd+4CbLvyxW0UrF0RGcdaYGde/BdQNZx+GbvHl82yy/IdDcn5J3JceEZfLb8UBBNt+QaXLQeq5KKV69j+LEYvdme0TxSkLJcOyfhJwFgZXwf/WvWQjJ/xWAnVsXqz98E/Q+fL6bR6f/y9yGEZCsBWPBJSAKRK1b5MAig6oJplvU1a1SHthWMkFqAzEyYFAFqaQC1S796VD0y6AMw24t2DPPWSj5UTeNkEcDoNaDkTqgOkB4sVmWJGha4qUEYt755vyVX7m5GGWipBKVC0DZvAsAkXQL6NwDB0qFkT2qgN7YV+T4YA2bkp2J0KBk7vBPaqzjKnfvwv9GwqYcc+aZQ+vje2/0cPgJ7KTRo8YQ80FaIN5fXIAdi8eTO6uroqxTIbEX5IrvXgzGvPK28Uc+fOxYYNG7Bo0SKYpom1a9fisssuq9SPjIxA1ydnKPbpyB9CpFMgnZ0gqgqFybniui6cvu1BXskXQHf0gyrEi40rCzMbgmh2XoYSgofsNdOqlKOjzfvu5wqBNi4AamiAaVdspD++Cc+esxzly7gaARzqJRZk6urlD9YO+zbdH38q8Qerg2FW+zIBqDqgWGU5i5FJeZ0qTgv4I1vlj1JGhaYoUHIJf1TkkPAHD5PGWent7UVvb2/NditWrEA2m8Vjjz2Gww8/HIB3sCebzeLII4+sKX/jjTdi+fLleOMb31iz7XPPPQfLsjB//vzaFyDAS2tf4RONpkLRorefEDImRyXSX1N6CfYnilSiMO95OoTLfaLgTeQpkrHCzgtb7v8UeDqIZHhkJ9PBJyXeMVGZ3i6pZqUP66Yy71lQeBmHeTroFCCmQO+i4PBmmwotH50vVAAYecrVwe1OIzUUjdqiADBUAmXU4n+2Q0VuOfafC31tdOZHocDQ518Pszf6jSj95z4QbtIhQGlecx+ix5rUq6urK0A2PPT29kJV1cgs2I4dO4QPzvPmzeO21zQNs2Y1fs6NxQknnIAvfelLuOKKK/D73/8ebW1tOOqooyr1Tz/9NPYex4PYjWDa8YcESioFommRhw+3L+qkVSt56xW10Si/KCkd4IQvTgFAKLN9RQYALTsq4bF9h4MnQ2zaNP4AxPZ+svOHwin3r5VYfK5USs3jD9qVQiob3cakANAT/ojIAQl/8DBptoHViwMOOAAnnHACzjrrLDz66KN49NFHcdZZZ+Hd73439ttvv0q7/fffH3fccUdAdnh4GL/+9a9x5pnRJb+XX34Zl19+OR5//HFs3LgRd911F/7jP/4DhxxyCN7ylrfE1tcsivYBNtuNaD7GS8NmblsTQUb3ca4zjkxc/7OZn4M0waNgLKpINGjsOUo8iD+WGuNqCUCbPe0yxmX8emAYBpYvX47Vq1cHylevXi18cF6xYkWk/T333IPDDjtszKse3/jGN6CqKo4++mhcf/31uP7662EwkQlvuummyH7nqYbdhz9kmLzcojQYKKAeJPzR/LGEfcXiD8ljZsIfCX/UiUmzstIIbr31Vpx77rmVCz/55JNx7bXXBtq88MILyGazgbLbbrsNlFJ88IMfjPRpGAbuu+8+fO9738Po6CgWLVqEk046CV/72tegjiFJ44w53SAKiUZzcSW/UkKatrIyFvjbplrV3peRecy8PuPYN3+vM9eYCsplqPXp8PpUSB2CYRlBX4BXSCT98eSoJr9SnowmSeRG9RjzHRLCUwsU1gzS2AfienJTERdeeCFOP/10HHbYYVixYgV+8pOfYNOmTTj77LMBeGcatmzZgltuuQUAcPbZZ+Paa6/FhRdeiLPOOgtr1qzBjTfeiF/+8peVPk3TxPPPP195v2XLFjz55JPo6OjAPvvsI9Rl9uzZ+Mtf/oJsNouOjo6I3fv1r3+Njg5+7oOphN2CP2QQcIu2YD7srdti6xJGHLtpGgYygqSVcXVI+EMMKX9IykU6xOEPtWTymnrtE/4YE6YTfxA6kelsd0MMDw+ju7sb2WwWXV1dWH3LQ7jyY9dy26q9swBdjyzXO9lh0FyuoXHjGEwZXIhJQLRX2UJ1Gb1eGf+cC3fZG/wlcXY7KK8/0aOBSAcb1X3A9cqIdPPrGr13tgJoAhYVymQAtdC8e2e3a9By/AcJkQ6lBR1IbR3lypiz09B3RpfynQ4dpORAsaIXPPjB/bDruI6Gv8zzftmPR//sZRMP/wYbgS/7jtdfBE1tPJSr7ZRw3/NXNTT2ddddhyuvvBLbtm3DQQcdhO9+97t461vfCgD42Mc+ho0bN+LBBx+stH/ooYdwwQUX4LnnnsOCBQvwxS9+sUJOALBx40YsWbIkMs7RRx8d6CfB5EMj/CGErkObzd8S12xnxUczuMdVFCgCRyvhjxo6CLaISXVIAUSw3Svhj4Q/wpho/kiclSYj/EU3iyZOmX8WctnoflySyUCZ0R11VhwHdPuO8VKZCxkRjZcjIyOByUxEsoOnsYhIBTRBagLh/c6o0AqOcO8zoQ0SUZcBbZg/Q0bLM35hmeKiDqQ384kot6wXbc/t8pLUMXA6DWz47rL6l+UpoBQprrAPwn98+t8BNIlsDvhcfLJZ/51YYydI0Ah/yKD29gJ69NxKM50VQP7Q3UyM10SYjPcS/ijrkPCHEAl/tA5T7szKVIORNvCRS0/h1tFCAbDtaOIuVQXS/IyvMjTT6/RXnHnGyq/jjSdLZiWSiZO4SxQy2Q8f2Sy9NUE5ADiEXydbrZfpINqeoDhiGStdjYPPQi843HIAcATJvhRAmJ1YGTaFOpgLO7j9pTePwu7WuTLaYAlUVyMhMdUREz1rChwJAQjQe+9IhWiaBncMrwQJmgQZf8jg1khG2Sz4P99GuKcUw7OJwx8qp8yHiD8Iphd/OEbCHwl/TA0kzsoEw+kfAJxopmFtZg/Q4IGnZs9uhRNg8caRORjhOhFB6BAbe7aMrdPAj48PRJOH+VAk44QTd4nGD+hAxUTJEk743omyCbM5BCJ6q3wZo+jC1eR6h9/reQcWJz4+AKhFB275kGJYB7s7zZVJvTYKa16GW0fyFtyMypEZQXHvmV68/ZAV6rnxeXQ+bfIvKnRBPY80QEwJEkwT0FIJbjYbSTasLYgfmYwHGReIkKJAqa3xWWcRryT8UdVbxB8iGc104Qq4pdn84XQl/JEgPhJnpcUoFUq45VJJ1k7XhbNzF2g+D+q6FXKhlHpnWjITu8LCy8zLM2rsiyCYTMsvVxA1pv57FcHEWGxf4QkHv84QyMh04BERq58TKvP7Cl+3/zebaZhHFDy9FRodO3y9Eb2dKHlV+rMBO00i/fl6uyTanz5iBRJ6BeQcCrtNi8ho2SLcdg1UJREZva8Ac4+OikWp6GYBxHRg7tFWmQXz6zLP74S5oAuFfXtACbyXSkBUgtlXP4PZfxyGNkyrN9Kp3hx9gGLu7QMYfEsGu47txK//5040E37oyTivBAmahZr8IQHNF+D2DwBmNYs9pRTq/HnNVDHisPBeYaTyJRTb2xoaJ+GP+PwRlgnUOcGEwq3iD3W4CLdNTfgj4Y9YmJLRwKYSHv71o8L9xiSTBlQNSioFGDpAKahpAqYJgIA6DiCJpCECuzTP2w/b6AqMTxK+EWRnfXhj+UadIJj8im3n98cm+IJAhpVl9xr7BMH2FW6rhvoLG2rCkfEJJNwfqzdB9D7wZCinjtWNex/Le4It1XNSeFstXOIRUEXvci4WK0OgFpj8AuUBKQBXV0Ast3rvyu+tNhVqyQUpZ8mmGgGFAwrAmpGCNmKCuBRUIXBVAtWhcA0CauhQSl5OIKorIKM2iOtlIVZzFmC5gAI4HQb0rXlQAMW9u6AUbC+ii64AloW2jTmYc9phLuoEccoXQhSU1m+FvawD83+tobh/N2hagVJwkX5yEFtPdrH9VC+Rl5sh+M4vH27uUn6DYSQDcgkSNAky/qgH1DTh9PcDmgaSToEQxUsal0p5We3RHI7g2bxafaVz3nWVunWkssGIg1bagF404aoKFCZRpT95NtH8gVBfrMyk4Q8lmKMr7IAE+IOWHbQUASkF+YM0kz80BWregaujyh8AqKZAGbES/kj4Q4jEWWkxHv7to/zQk4oCtacnWEYISCoFu3+gKWOL9t36aOSwHmtcWQPN9usbYbZPfyObLxc+oOgfWiSoZhJmZcDR0Sy3ZccKkxabyVdH1dHiXS+rt68Du9dYpLeCoA7h/cnsWBq8PcoKDcqEHRGF+UN3JDrQKuEQAKly9nq9HIKx0h/7tbNc2B06tHISLqXoDaDng/uTFZtCKU/3GUMlOO0alJwN4lAow+WHCpMCpgm724CaNaHYDtTyPudUOQuxr7c2Uo0Qk3l5GLlD5qJ9XTBBnbEjB2NHDpu/dQiK85XAF2xbOK3FUZng3y4wemg3mgqXQhoXWiaXIEGTIOSPRmHboKM2wr3U4ggZePwR5oZwX4X2NmRyQecr7KgAgF70Jul8R4V1esKZ5UX8IdKhWfzh2+NJyx8uXwdSLmQdtJTlCaklPn8QoMwfKrRRZ+z8YQGwrCB/FBP+SPhDjGQbWIsxtCPLJxpZoiQBmv01rkVKcWfXGpWRXRevzzhfWkfQl2iMWohz7+LYIdFhTwCVFZNGdCDMLGW9MnZKfHaKcMJH1oQkyaSTIbGW/pxMnE9RgnFI6pUgQS0I+WMSIM4vzjAb3ylQS4dG707CH1U0zB+2XImEP8pI+KPpSJyVFsNI1wq4OHkxXho22UxwUStxWKOII0NiXmgzP4dw9JR6xpJmLY5jQSTd+VsJGu2PNC/PXIIEkwZi/piacCUPmnGR8EfzxxL2VePzS/gjQauQOCstxr6HLIGicW6z7YDKsthz0Aqj3ExDyy7V1ysnCP9e11iNlPszcLz6OHrX+uR4MqpkJURUHt6eENDBII3fb1UR66Dye9OGikIZp8MQ6kBT/LHUrHh2NdXnNB6+kQKpvuhWkrEh7qzY5J+ESDB1IOSPSYJGv+1uKf7vNA5/NNKXrHy35A8tBl8rCX/U3WnCH03F5LWCuwlO+s93wrX5vx6aL0RCFo8nwvtdmyHTqCNTK34+T0YWu14Ya16im4yIRMvocfSW3TvZZyAkIpOK9dYVrg6pEUvYp9lpcErL907nE4e2Ky/Ur7SwnatD+uUBlBa2c1d5ulf3NW6VCNCz6rUGhWogWcZPMAkg44+JRhz+kAUslk1ux+UPHnZH/pBB1F61JXqL+GM04Y+6kPBH05E4Ky3Gov0W4pB3LIOiRm+1m89xZWSx8Jv9VRaRQD2GMVwnm8XxyxuZmWq2IyOKkT+eROQIppH8w4/ccQRx8KV6W65Yh3aNTwJDJaEOdofB18EGnE6NK5PaNCL+/OZ2cs8fZp7aAWOHW//smAu0veLg/o031ClQb780/itBgiZBxh+TASLbLYNt8Le21Yr20yh/yPrb3fhDxr1+HpJwney8T8IfY0TCH03H5LSAuxk+cukp/EOStgM3OwwAda+wNHsrmMzQ+hAZOZ6xo6H/2XFqJe7iztYIZNgILWGIxmFlZA4VCwX1Je6K6ED4MhoV3yORbpoDuALC8cHT2yrHuo/0l4tGBfJhdfFJxRgsVpbsI2MVbIDwiai4tJPbX2ZtH3IHzuLqMf+6DVBMWptwXEDLUcz64Ys1GsYAdeO/EiRoIoT8MQlQD3+EoZniLTei1ZCx8kcYMv6QJS+ezPwhKlddMX+IJiUVAFYm4Y/YSPij6UiclXHA9V/4udDLoPk8nKEh7z2tZhtudqbhWvBj17M//rBhYd+Hs9tTTnlYRkM00ZYvIxonhWriLpEOFEEdROMA0aRe9eigI0iiYZnw+EA5OzGJyrB6h3VQUSWViN4hwqlHbyNvw85o3DoKL/RxWG9j2ITZbXBltFETdmcqqoMNL8swZwUo88pIhXDYOgVA5vl+5N842+uLuW5j4xD2+PYGaCPl1mH7Xf7bGHAx579fxK5zX4cECXZXyPhjsoDHH+D8XQsaAEsXnH1A4/zBJn4MQ8QfqmAcv26y8odMBxF/hO904N4VbNgZlVuX8EeC8UbirLQYrzz9Kp5f86J0ZozmC3D6tsMdHgEcp5ppeO4cad8yImiEJNi47WHjSYBK8le2X7aOhmT9vnjZgVV4hjtsbH2jWeLI6OXxRTqUODr4hBOe1fL1szgyvg48vX2iDM+sVfQmHBkqv3c2RwelvIJtk6iM4nrlIr3NjBLVoWB7904jEb0JBYpdhpf5l5ExsiYcAFaXHpChANSREswOHW6ZxCp65x2AEBT36AA1ghmK0xtGUNKBwtJOUEOp6k0BY3MW+UXdyL9hDux2zdNFIVCyBfSuXIt5v+hHeotTidZCbIq2jQ7m37AdZm4rtn5zPxQWqTh6n0+iqUj2HCeYBKiHPyYaMv4A5BPcBU4We72c84Nywl9p5f4a4Q8/SSPvDirwVl92F/7wJxiF/MGR8WGmSKDc4w9Hzh+desIfPCT80XQkSSFbjD/9ZDVUTYFT65AkpaC5HJwc/xwLD7xZESKo4yGcgZd9zybBYhNaUQSTgbF1fix6NpkWW+cnoGLL/cy8vkwqJOOPwX5RrZBe6VB/hCPjE5wvwx4FrFdvtj+z/Lcvk6ZBGV9v2b0Lj+ProPqNOTrobLkOEKvan1FwuTIaAJTj45e6dWhZq6JXZtgMyPh6awBQTuAVvnf+IUsAKJavQ4VHBOnXRit1tl9OgbQF4JURT2a/WTBe6IcCQB8woQ+YwGZPZuuHepDbVweWLEIEr74GutceyC9VkV86t1pOgezxC6LtxwI3TOWNyCVI0BzUzR91IvyQ7EOWIFgEGX9QThngPZSnmJ8ImyDSdzR8DiDMg5sFzyb5tt2HjD9YDpPxByuzO/AHQTAZMPs5BWRUgDgMf5QoVybAH506tBGGP0aqXJDwB4OEP5qOxFlpMTY+u7lpRFMLje4U8GeJGulLJiMjO9+I8spl/YkO/4czAvsQlVNJXZhQ69FBReN6O8SbLRPJ8MBmGQ6DWDH0Nl3hfRDpYM9Iwxgqcuv0cnZiHvzM0mE47bpwLOvQxUCvoHavPfjlBDDnNDkfRdxZrmRmLEET0Uz+YCez4tTz2jfKH4qhA4LwxbJIYb6jEulPooNMt+nEHyL9iCPWWySj2gl/1IWEP5qOxFlpMcwxxJWfaEzybdINgZ3ZCSPOdcaRiWuHmvk51OqLVy9NBBbnmmTdCWL11+qPij7cuKCISTZN1iPBtMZU5g8elCk6c5zwR7mvGkok/OErgoQ/mozkzEqLMWtBDxRlat5m3u9GFsEkzu+slgyvPs48oyoZa7z0jpO8WRpth9SeKQ3DrZFgjiejFUzxvTMk/YmUs8V3T83Rxj8QF1BzSRSVBLsfmskfMtvt17caVoqfiyMu4tjuhD/KSPjDQ8IfUwJT8yl6CuFtpxwJt8FM9XEQx2Cy0Uh4/Ym2YInGqzUH2CznR4NYb3/vbhjSTL6Ccfz2cfTmLqFTcX+2GoM4UhIdBJ2pI5ZYh06dr3fBEergZMTL59acNFcm/UI/3BR/KqvryVFuuRQK0PlYtnE5GZIDkgkmAVrBH43aMxFk/CFCejQvrHM0+fR2s5yF3ZI/lMb5gxox+GPUTvijHiT80XQkzkqL8Zb3Ho6uWZ21G44RjRoqX6ZWncg488bz9xzzZEyBjDSZlUA32WyRjEBFRBQ3o7FIRnbvRESkOxIi0gV6F8U6uGn+N0IBAmEeWUgdmZ40/zPflgMUwbWq/CzIatFGYf+ZXD06/vQKiGTmLDoIoI5SXPG1U+qXqQeuG/+VIEGT0Ar+aHQSStZPMx+tVNsR1on4Q5ZMV9TbbskfroQ/NMG9K4l1cFMJf4wJCX80HYmz0mLoho6jP/GWcRlL5niIICIp32CKHAyZQef1l5LIiJJmyZbefYTrwzHow3WN6hAno7Hs3smIyBEk7tItsYzFhCtmoRWo8N45AhkFgNPGz06sDhaFOpQWdHD7S23Nw+xNcWX07Xm4GS0yg6cWHMx6oP6IeCDA7D8N4q1vPbx+mXqQzIwlmARoNn/I7I+PRh2WRmVKtZtEEJc/amG34g/C70+3JTJpAX8UJfxRnghL+EOChD+ajsRZGQfc+e1V4zLOWFdXRIY7bGTYccJ1fhx8Xn/sOGwdmzRLNJsUrvNj3fNkWCcnLMNLpuXrUEvv8HtNooOI9GT3W3Mbv3dGwYWT4pNHWM6HnnNhhmLgV3TI23CZWPaB62mvZjRm69KvjcJayCccrb8Ep0uP6t2Xg7WgyyOckBXq/t8X0LOmvGwkmmgqfzF77x6FNau5++ABJGSTYNKgVfzBs+si2yRCHJkUgCInv0otxOEP2crGbscftHH+0IouHL1B/shTmJ0Jf0iR8EfTkTgrLcZZB39u3Mfkfd1lS/bssnjYqLFGky3njeXX+Ykf2TIK/qyQ/zfPcPsyBNGMwhTVZJEivZ1QmV8u0kEFP6GXrzfvPvCIkqcDW84jHFYHdoWlnnunlmggISQYGQLAVaMyxrAFqyOatIsCUEwXdpcekVFyNlwNcA012t+WUZQWtoNqwYRehALKsIXSks5KpBa/LvXSAOwZGeQP7IWrl/VXAEKA3h8/h3n/24/UTrc6EJOBNL3Fwfwbt2PX8R0YPKodKz/5AyRIsLvhrIMvaHqfvNWQemyTCDL+ECGdy6PYIV774EWQktluDd5WsbAOspUfA7spf5BgWU3+sCjsdIP8MWLBalcjMgl/JGgVktDFLcbGpzeN+5isgSahchr6n63zy9m9uWyej7CxI6gab1bGJw+CavIrH25IJuwt+44JCcn4+lkILu+7CGY15uldQjSBls6RoYxMWAf2voX1JgIZmd6+Drx7p7plGR1QmagFruJlIabl94RhP8V1QQDYGUApovJBuRpA7HJ/GQ1qkYlp71JPbxWAqoCU8zm4mgJ1xPLu3cw0tFETxKWAQuB06DAGSnDh7UNWihbgAtRQoQ8WodgUpVlpEIVAsRxQhcBt02Fs8BJ6FfbuArFdEJeCagrIsIX2Z0bgdBjILZvphZEEQGyKvll5YO4sLPz2JpSW9cBNq1AKDlJP7MSWryzFtjO9xF5uiuD321/DxWgikqReCSYBNj79Wkv6Ze19ePJZVMc+wIbBnsEQ8UcY6VHvUbyEaI6VUlsG6dE8zJQOgwnfzD7Ax7HDNoJ85F+TjD/CHDbp+YOWr1UFFMbbcUn1kH6YP1DmDycFEBNV/ignjvT4Q4VadJiLIZ6MAkAL8cdwwh8JfzQXibMyCeF/XcMGXlQugmjPq/8/Df3t/88ug4eNKi/evC/DS0DlL7D6Y7H9+cQTzgocPjTJyujwDLcekgkfWGRlUiHdlFAbXh2rQ/g+hvXmyfD0thVvqxdbF77f7Fi6FdItQC4e+fiZipXyRnCtENKb8U30gg27Q4dWziBs5L1K1QHguFW9zepAqYEi7A4d6qgFgMIYKFX0VwaLsLt0j5hKVVZM9ReDOgxUd6lnXh5G7pB5aF/XBxbqqIn2tX3YtPIQlOYFQ9ts+fyegbY4MXTg2AVGlzX3EDKlLiht/LBjHJkECcYDPBtTmT1H1HaLHBMf4Qd/X0bEHwCQ78igbbQQKOMlg/SjhvmOCmuHWbsr4w8e/BV8nz/8/2X8YWBy8IdTdjbYOhl/aE5IN8q0CfNHeVmKPXRPgMDSjl5wqvxBGf5wAZgJf7BI+KP5SJyVKYZ6HZVWjSXz++PoxlvlqdVnnHH8mah6x6iFWjLcexdj0sSfrRNVyu4dD0QScQeCvpyUVnFwov3FMK6SzaduhjT+gSji6GexQWm8Wa5kz3GCSYxm/kri2MBU0eSUjk2Hhm1gjHEmA3+4tL6gASwS/qivv4Q/Jj8SZ2WKoVHj0uyxWjH2eFyP7HBWnHs6XiYlDhnLIM0mLBhLRiixMgZLbh6xYlwtBUizE33TmMv4CdkkmMSYaP5wdFUaojgOpgt/EJ8MGpFBwh/19Jfwx+THlDxg/81vfhNHHnkk2traMGPGjLpkKKW49NJLsWDBAmQyGbztbW/Dc889F2hTKpXw2c9+Fr29vWhvb8fJJ5+M115rzZ5hqa6Qryg06+ss60dUJwvfKMq/IuvT5pTVo0ej5b7R5tXXMm+ia230/mlUrjdPB2kyshRpWG+qKGIdBMShZUvCe+d0GkId3DR/LHWgKJAA0ltseZY1oVycgKgJpiN2d/4QoRZ/NPrYKHt2FpYX4q+sxOGPRvqSlU8G/lCbzR9Gwh9VuYQ/JjumpLNimib+4z/+A5/61Kfqlrnyyitx9dVX49prr8U//vEPzJs3D+985zsxMjJSaXP++efjjjvuwG233YZHHnkEo6OjePe73w3Hae5MUC3EIoEY48g+fNm2rFoGnQeRI2NATAK2QCZuEq44RCTS298SwDXAgpsne1iQzcMIHccSFZNASuXrPVrdAx6G3cEP4aggGMEl0N/WvFA/c2EHV4f0xiEU9+rkZknuvntb41aJAr33bWtQqAaSpF67LXZ3/pAhjg0UoVZiQx5451N8yO5SHP4Q/RJ3R/6IU66YMv7gJ2RM+KNOJPzRdExJZ+Wyyy7DBRdcgGXLltXVnlKKa665Bpdccgne97734aCDDsLPfvYz5PN5/O///i8AIJvN4sYbb8R3vvMdHHvssTjkkEPwi1/8As888wzuvfdeYd+lUgnDw8OBF4vV7q8bvj6ZURLVNTpD4kP006j1YM2rk626xDXoMkeGp0M9RCTSm4c4emtUrIMo8aMsE7P0fgu26ColRzKbxd99bWRLQh3cDD+uvgLAmWFwZVIbh0GJQIcZbSCcivT6XUhvccVfzIhiQMeLFu7ZcH2dAnUiiZO/22J35w8ZZPxRa4WAhzj8YWb4D7Wy8xhx7LDs4Waq8odLxDqIZOLxh5vwx1iQ8EfTMSWdlUaxYcMG9PX14bjjjquUpVIpHH300fjb3/4GAHjiiSdgWVagzYIFC3DQQQdV2vCwcuVKdHd3V16LFi2KtGnrzsTWXUQqvLo4Kx71fAF4D9Yiwy2SURAvU304dr4P35HhXZtIN1nCSgjKa+ktyznF1cGt/fnxxnFUfh0onwwVAFa7IHHXiFVJpBWuczpUbrmeLcKZkeLWKUMmqMopd4DSnvxkX+1PbUfu4DnhqwEAzPvBS1ALtDbhuICepXjj3QM1GjYO6rqxXwl2L0xl/ghDxh9xVldECRRZhMsNyVYw0epKbP4QENxU5Q92K1jL+SOjcWWk/NGe8AeQ8EcrMC2clb4+L8zd3LlzA+Vz586t1PX19cEwDPT09Ajb8HDxxRcjm81WXps3b460+b/BW5Bqly2ARxGe7WB/rOGZHP993C1acXQIG1q/rlaWX5sjA1Rj1tOQXApeHH6ejGgsFdFkWqhRV0tviyPD6h7Wm03OVa/evCRgPjQHcLRoXTghG9ufkbNgtmkRGcALg+xqJKKDNupUkkVGdBgqwe5JC3VwU9HMxelXR1Hcs6Niadi6zJM7kDt0LqhCAjNoet8o9vjGizB2le9S2H6X/05vcXDQj17DrU98F01HMjOWoIzJwR/Ny7It4w+2LPxblk148exZmHPq+WWE7TMLGX9Q8O2w4QKW4Clnd+QPHicDMfmjYDfOHzkHVgdfJuGPhD/GgknjrFx66aUghEhfjz/++JjGICT4+E0pjZSFUatNKpVCV1dX4MXDH0d+0fCSPkGVQNhZL4roB1cPsdRyWHg+PasDOw6rQ9hw+saHl8VeK5eHswP7ybQsRke/LlVuz+uPlOvCMv5sGy+jsFruS6Q3L3OxT5ThOv/+sAcmK+NQ8X3wZ9xEepsaR2+7rAMJ1vmfj51RIv0Zedu7d4Ya6Y/YFKV2pRLhpXKtoxaoSmDOTEdktMEiTAWwO43gvXMAgMCc1w63XQvIpF8bRdEFint3wW1XK8RCFEDfOox8Twa5Q+bBmp2Gk1bhtGtwNWDWVeuw8H+2ov0lG2qegpgUao6i83kLi76zCf+85BLcsf5/kCDB7s8ftzZ1S5iMP0jo/zC3yPoL2x/ehBf7KnHm8PztYI4eDVbKJkIMj0PAt6maGywP67078Yevg6lGZcaPP2yPP3pSEZmEPxLExaQJXfyZz3wGp512mrTN4sWLY/U9b948AN7s1/z58yvlO3bsqMyWzZs3D6ZpYnBwMDA7tmPHDhx55JGxxuWh2XuQEySYlriyxf27FNxN0bWQzIxNCBL+SJAgQd1I+GPKYdI4K729vejt7W1J30uWLMG8efOwevVqHHLIIQC8iDAPPfQQrrjiCgDA8uXLoes6Vq9ejVNOOQUAsG3bNjz77LO48spWf7MTJEgwqUApYsXATMhmQpDwR4IECSYNEv5oOiaNs9IINm3ahIGBAWzatAmO4+DJJ58EAOyzzz7o6PAOZO2///5YuXIl3vve94IQgvPPPx/f+ta3sO+++2LffffFt771LbS1teFDH/oQAKC7uxtnnHEGPve5z2HWrFmYOXMmLrroIixbtgzHHnvsRF1qggQJJgDUpaAxZsZoQjaTHgl/JEiQoJVI+KP5+P/bu/+YqOs/DuDP+8odF0mXiXgHKmLZIVntgClIaOnCH6sQXZk5Rv84zbXyxyKyTWDNkuaP2hDMybRWmSugJMugBSh5BtlZFqltkZJFhjO4SkT09f3DvHnegXfIfT4fuOdju6173/tz93rx7nztdXefz1sz56z4Y82aNbDZbMjLy8Pff/8Nm80Gm83m9pvkY8eOob293XU/JycHy5cvx7Jly5CUlIRTp06hqqoK4eHhrjmbNm3C3Llz8dhjjyE1NRVhYWGorKzEkCG9XVSRiKjviouLERsbC6PRiMTEROzfv7/X+XV1dUhMTITRaMS4ceOwZcsWjzllZWWIj49HaGgo4uPjUVFREajwBxzWDyIaLIKlfuiErVy/6ujogMlkQnt7e48nSxJR4NzIe/DKsQ8MmYcQnff9BHrTLRdQc7Hc59fetWsXsrKyUFxcjNTUVLzxxhvYtm0bmpqaMGbMGI/5zc3NmDhxIhYvXowlS5bgyy+/xLJly7Bz507Mnz8fAGC325GWloaXXnoJmZmZqKiowJo1a1BfX4/Jkyf7nRMph/WDSF2sH9qsH2xW+hmLDZG6+qPY3K/L7HOxqZUKn1978uTJSEhIQElJiWtswoQJmDt3Ll555RWP+c8//zx2796NH3/80TW2dOlSfPvtt7Db7QCABQsWoKOjA59++qlrzqxZszBs2DDs3LnT75xIOawfROpi/dBm/RiQ56xo2ZXe79qdiIlIGVfeezfyOUy3nAfE/xMku/+72Om17//Q0FCEhrpfp7WrqwuHDh1Cbm6u23h6enqPGwna7Xa3jQcBYObMmSgtLcWFCxeg1+tht9uxYsUKjzmvvfaa3/mQslg/iNTF+qHN+sFmpZ85nU4A8LoTMREpx+l0wmQy+XWMwWCA2WxGfesnfX7doUOHerz/8/LykJ+f7zbW1taGixcv9rrZ4LVaW1u9zu/u7kZbWxssFkuPc3rbnJC0gfWDSBtYP3x7TqWwWelnUVFRaGlpQXh4uMdmYB0dHRg9ejRaWloGzVf8gy2nwZYPEHw5iQicTieioqL8fl6j0Yjm5mZ0dXX1OTZvGwFe+6nY1fzdbNDb/GvH+7KBIamP9WNgG2z5AMGXE+uHNusHm5V+9r///Q+jRo3qdU5vOxUPVIMtp8GWDxBcOfn7idjVjEYjjEbjjYTlk4iICAwZMsTjE6urNxu8ltls9jo/JCQEw4cP73VOT89J2sH6MTgMtnyA4MqJ9UN79WNAXrqYiGigMxgMSExMRHV1tdt4dXV1j7uep6SkeMyvqqpCUlIS9Hp9r3P6cyd1IiJST7DVD36zQkSkkpUrVyIrKwtJSUlISUnB1q1bcfLkSSxduhQA8MILL+DUqVN46623AFy+cktRURFWrlyJxYsXw263o7S01O0qLc8++yymTp2KwsJCZGRk4KOPPsLnn3+O+vp6VXIkIqL+F1T1Q0gxnZ2dkpeXJ52dnWqH0m8GW06DLR8R5qR1mzdvlpiYGDEYDJKQkCB1dXWux7Kzs2XatGlu82tra8Vms4nBYJCxY8dKSUmJx3O+//77YrVaRa/XS1xcnJSVlQU6DQqwwfT//BWDLafBlo8Ic9K6YKkf3GeFiIiIiIg0ieesEBERERGRJrFZISIiIiIiTWKzQkREREREmsRmhYiIiIiINInNChERERERaRKblQBbu3YtpkyZgrCwMNx6660+HSMiyM/PR1RUFG666Sbcf//9+OGHHwIbqI/Onj2LrKwsmEwmmEwmZGVl4a+//ur1mCeffBI6nc7tlpycrEzAXhQXFyM2NhZGoxGJiYnYv39/r/Pr6uqQmJgIo9GIcePGYcuWLQpF6jt/cqqtrfVYD51Oh6NHjyoYcc/27duHhx9+GFFRUdDpdPjwww+ve8xAWCMif7F+sH4ogfVD+2sU7NisBFhXVxceffRRPPXUUz4f8+qrr2Ljxo0oKipCY2MjzGYzHnzwQTidzgBG6psnnngChw8fxt69e7F3714cPnwYWVlZ1z1u1qxZ+P333123Tz75RIFoPe3atQvLly/Hiy++CIfDgbS0NMyePRsnT570Or+5uRlz5sxBWloaHA4HVq9ejWeeeQZlZWUKR94zf3O64tixY25rMn78eIUi7t0///yDe++9F0VFRT7NHwhrRNQXrB+XsX4EDuuH9teIwE0hlbJ9+3YxmUzXnXfp0iUxm82ybt0611hnZ6eYTCbZsmVLACO8vqamJgEgBw8edI3Z7XYBIEePHu3xuOzsbMnIyFAgwuubNGmSLF261G0sLi5OcnNzvc7PycmRuLg4t7ElS5ZIcnJywGL0l7851dTUCAA5e/asAtHdGABSUVHR65yBsEZEN4L1I0OBCK+P9YP1g9TBb1Y0prm5Ga2trUhPT3eNhYaGYtq0aThw4ICKkQF2ux0mkwmTJ092jSUnJ8NkMl03ttraWkRGRuLOO+/E4sWLcfr06UCH66GrqwuHDh1y+9sCQHp6eo/x2+12j/kzZ87E119/jQsXLgQsVl/1JacrbDYbLBYLZsyYgZqamkCGGVBaXyMipbB+BA7rhzvWD1ISmxWNaW1tBQCMHDnSbXzkyJGux9TS2tqKyMhIj/HIyMheY5s9ezbeeecdfPHFF9iwYQMaGxsxffp0nD9/PpDhemhra8PFixf9+tu2trZ6nd/d3Y22traAxeqrvuRksViwdetWlJWVoby8HFarFTNmzMC+ffuUCLnfaX2NiJTC+hE4rB+XsX6QGkLUDmAgys/PR0FBQa9zGhsbkZSU1OfX0Ol0bvdFxGOsv/iaj7e4fIltwYIFrv+eOHEikpKSEBMTgz179mDevHl9jLrv/P3bepvvbVxN/uRktVphtVpd91NSUtDS0oL169dj6tSpAY0zUAbCGhEBrB/XYv1QH+uH9tco2LFZ6YOnn34ajz/+eK9zxo4d26fnNpvNAC53+xaLxTV++vRpj+6/v/iaz3fffYc//vjD47E///zTr9gsFgtiYmLw008/+R3rjYiIiMCQIUM8PjHq7W9rNpu9zg8JCcHw4cMDFquv+pKTN8nJyXj77bf7OzxFaH2NiK7G+uGO9UM9rB/aXyO6jM1KH0RERCAiIiIgzx0bGwuz2Yzq6mrYbDYAl39XWldXh8LCwoC8pq/5pKSkoL29HQ0NDZg0aRIA4KuvvkJ7ezumTJni8+udOXMGLS0tbsVUCQaDAYmJiaiurkZmZqZrvLq6GhkZGV6PSUlJQWVlpdtYVVUVkpKSoNfrAxqvL/qSkzcOh0Px9egvWl8joquxfrB+aOXfJtYP7a8R/Ued8/qDx4kTJ8ThcEhBQYEMHTpUHA6HOBwOcTqdrjlWq1XKy8td99etWycmk0nKy8vlyJEjsnDhQrFYLNLR0aFGCm5mzZol99xzj9jtdrHb7XL33XfLQw895Dbn6nycTqesWrVKDhw4IM3NzVJTUyMpKSkSHR2tSj7vvfee6PV6KS0tlaamJlm+fLncfPPN8ssvv4iISG5urmRlZbnm//zzzxIWFiYrVqyQpqYmKS0tFb1eLx988IHisffE35w2bdokFRUVcvz4cfn+++8lNzdXAEhZWZlaKbhxOp2u9wkA2bhxozgcDjlx4oSIDMw1IuoL1g/Wj0Bj/dD+GpEIm5UAy87OFgAet5qaGtccALJ9+3bX/UuXLkleXp6YzWYJDQ2VqVOnypEjR5QP3oszZ87IokWLJDw8XMLDw2XRokUelzC8Op9///1X0tPTZcSIEaLX62XMmDGSnZ0tJ0+eVD74/2zevFliYmLEYDBIQkKC1NXVuR7Lzs6WadOmuc2vra0Vm80mBoNBxo4dKyUlJQpHfH3+5FRYWCi33367GI1GGTZsmNx3332yZ88eFaL27sqlMa+9ZWdni8jAXSMif7F+sH4ogfVD+2sU7HQi/51JREREREREpCG8dDEREREREWkSmxUiIiIiItIkNitERERERKRJbFaIiIiIiEiT2KwQEREREZEmsVkhIiIiIiJNYrNCRERERESaxGaFiIiIiIg0ic0KERERERFpEpsVCnqvv/46YmNjERYWhrlz56K9vV3tkIiIaABg/SAKPDYrFNRWr16NoqIivPnmm6ivr4fD4UBBQYHaYRERkcaxfhApQycionYQRGpobGxEcnIyGhsbkZCQAAB4+eWXsWPHDhw/flzl6IiISKtYP4iUw29WKGitX78e06dPdxUaABgxYgTa2tpUjIqIiLSO9YNIOWxWKCidP38elZWVyMzMdBs/d+4cTCaTSlEREZHWsX4QKYvNCgWlb775BufOncOqVaswdOhQ1+25556D1WoFAHz88cewWq0YP348tm3bpnLERESkBawfRMoKUTsAIjUcP34cRqMRR44ccRt/5JFHkJqaiu7ubqxcuRI1NTW45ZZbkJCQgHnz5uG2225TKWIiItIC1g8iZfGbFQpKHR0diIyMxB133OG6GQwGHD16FPPnz0dDQwPuuusuREdHIzw8HHPmzMFnn32mdthERKQy1g8iZbFZoaAUERGBjo4OXH0xvLVr12LOnDmIj4/Hb7/9hujoaNdjo0aNwqlTp9QIlYiINIT1g0hZ/BkYBaXp06ejs7MT69atw8KFC/Huu+9i9+7daGhoAAB4u6K3TqdTOkwiItIY1g8iZfGbFQpKI0eOxI4dO1BSUoL4+HgcOHAA9fX1GD16NAAgOjra7ZOwX3/9FRaLRa1wiYhII1g/iJTFTSGJvOju7saECRNQW1vrOkHy4MGDGD58uNqhERGRhrF+EPUv/gyMyIuQkBBs2LABDzzwAC5duoScnBwWGiIiui7WD6L+xW9WiIiIiIhIk3jOChERERERaRKbFSIiIiIi0iQ2K0REREREpElsVoiIiIiISJPYrBARERERkSaxWSEiIiIiIk1is0JERERERJrEZoWIiIiIiDSJzQoREREREWkSmxUiIiIiItIkNitERERERKRJ/wcNBqf6HrVu1wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sim_size = sim_batch_size * n_sim_batches\n", + "\n", + "plt.figure(figsize=(8,4), constrained_layout=True)\n", + "for i, t2_idx in enumerate([4, 8]):\n", + " t2 = np.unique(theta_tiles[:, 2])[t2_idx]\n", + " selection = (theta_tiles[:,2] == t2)\n", + "\n", + " plt.subplot(1,2,i+1)\n", + " plt.title(f'slice: $\\\\theta_2 \\\\approx$ {t2:.1f}')\n", + " plt.scatter(theta_tiles[selection,0], theta_tiles[selection,1], c=typeI_sum[selection]/sim_size, s=90)\n", + " cbar = plt.colorbar()\n", + " plt.xlabel(r'$\\theta_0$')\n", + " plt.ylabel(r'$\\theta_1$')\n", + " cbar.set_label('Simulated fraction of Type I errors')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "tile_radii = gr.radii[gr.grid_pt_idx]\n", + "sim_sizes = np.full(gr.n_tiles, sim_size)\n", + "n_arm_samples = (\n", + " params['n_stage_1'] +\n", + " params['n_stage_1_add_per_interim'] // 2 * params['n_stage_1_interims'] + \n", + " params['n_stage_2'] +\n", + " params['n_stage_2_add_per_interim'] // 2\n", + ")\n", + "total, d0, d0u, d1w, d1uw, d2uw = binomial.upper_bound(\n", + " theta_tiles,\n", + " tile_radii,\n", + " gr.vertices,\n", + " sim_sizes,\n", + " n_arm_samples,\n", + " typeI_sum,\n", + " typeI_score,\n", + ")\n", + "bound_components = np.array([\n", + " d0,\n", + " d0u,\n", + " d1w,\n", + " d1uw,\n", + " d2uw,\n", + " total,\n", + "]).T" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "t2_uniques = np.unique(theta_tiles[:, 2])\n", + "t3_uniques = np.unique(theta_tiles[:, 3])\n", + "t2 = t2_uniques[8]\n", + "t3 = t3_uniques[8]\n", + "selection = (theta_tiles[:, 2] == t2) & (theta_tiles[:, 3] == t3)\n", + "\n", + "np.savetxt('output_lei4d/P_lei.csv', theta_tiles[selection, :].T, fmt=\"%s\", delimiter=\",\")\n", + "np.savetxt('output_lei4d/B_lei.csv', bound_components[selection, :], fmt=\"%s\", delimiter=\",\")" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.0625, 0.0625)" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t2_uniques[8], t3_uniques[8]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sandbox" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.5 ('confirm')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "d8e1ca1b3fede25e3995e2b26ea544fa1b75b9a17984e6284a43c1dc286640dd" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/research/lei/lei.md b/research/lei/lei.md new file mode 100644 index 00000000..985fed83 --- /dev/null +++ b/research/lei/lei.md @@ -0,0 +1,712 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3.10.5 ('confirm') + language: python + name: python3 +--- + +```python +%load_ext autoreload +%autoreload 2 +``` + +```python +import os +import confirm.outlaw +import confirm.outlaw.berry as berry +import confirm.outlaw.quad as quad +import numpy as np +import jax.numpy as jnp +import jax +import time +import confirm.outlaw.inla as inla +import matplotlib.pyplot as plt +import numpyro.distributions as dist +from functools import partial +from itertools import combinations + +from confirm.lewislib import lewis +from confirm.lewislib import batch +from confirm.mini_imprint import grid +from confirm.lewislib import grid as lewgrid +from confirm.mini_imprint import binomial +``` + +# Lei Example + + +The following description is a clinical trial design using a Bayesian model with early-stopping rules for futility or efficacy of a drug. +This design was explicitly requested to be studied by an FDA member (Lei) in the CID team. + + +> The following is a randomized, double-blind, placebo-controlled two-stage adaptive design intended to identify an optimal treatment regimen +> from three possible regimens (for example, different dosages or different combinations of agents) and +> to assess the efficacy of that regimen with respect to a primary binary response endpoint measured at month 6. +> +> In Stage 1, one of four experimental regimens will be selected, or the trial will stop for futility. +> In this stage, a minimum of 200 and a maximum of 400 will be randomized 1:1:1:1 to one of the three experimental arms or one placebo arm. +> Interim analyses will be conducted after 200, 300 and 400 subjects have been enrolled to select the best experimental regimen and to potentially stop +> the trial for futility. +> If an experimental regimen is dropped for futility at an interim analysis, +> the next 100 subjects to be randomized will be allocated equally among the remaining arms in the study. +> At each of these three analysis time points (N = 200, 300, 400), +> the probabilities of being the best regimen (PrBest) and predictive probability of success (PPS) +> are calculated for each experimental regimen using a Bayesian approach, +> and the trial will either stop for futility, +> continue to the next interim analysis, +> or proceed to Stage 2 depending on the results of these PrBest and PPS calculations. +> +> In Stage 2, a minimum of 200 and a maximum of 400 additional subjects will be randomized 1:1 to the chosen regimen or placebo. +> The two groups (pooling both Stage 1 and Stage 2 subjects) will be compared for efficacy and futility assessment at an interim analysis +> after 200 subjects have been enrolled in Stage 2, +> and for efficacy at a final analysis after 400 subjects have been enrolled in Stage 2 and fully followed-up for response. +> The study may be stopped for futility or efficacy based on PPS at the interim analysis. +> If the study continues to the final analysis, +> the posterior distribution of the difference in response rates between placebo and the chosen experimental arm +> will be evaluated against a pre-specified decision criterion. +> +> - Scenario 1: interim analyses are based on available data on the primary endpoint (measured at month 6) +> - Scenario 2: interim analyses are based on available data on a secondary endpoint (measured at month 3) + + +This notebook breaks down and discusses the components of the trial. + + +## Model + + +The notation is as follows: + + +- $y \in \mathbb{N}^d$: Binomial responses. +- $p \in [0,1]^d$: probability parameter to the Binomial distribution. +- $n \in \mathbb{N}^d$: size parameter to the Binomial distribution. +- $q \in [0,1]^d$: base probability value to offset $p$. +- $\theta \in \R^d$: logit parameter that determines $p$. +- $\mu \in \mathbb{R}$: shared mean parameter among $\theta_i$. +- $\sigma^2 \in \mathbb{R}_+$: shared variance parameter among $\theta_i$. +- $\mu_0, \sigma_0^2, \alpha_0, \beta_0 \in \mathbb{R}$: hyper-parameters. + + +The Bayesian model is described below: +\begin{align*} +y_i | p_i &\sim \mathrm{Binom}(n_i, p_i) \quad i = 1,\ldots, d \\ +p_i &= {\sf expit}(\theta_i + \mathrm{logit}(q_i) ) \quad i = 1,\ldots, d \\ +\theta | \mu, \sigma^2 &\sim \mathcal{N}(\mu \mathbb{1}, \sigma^2 I) \\ +\mu &\sim \mathcal{N}(\mu_0, \sigma_0^2) \\ +\sigma^2 &\sim \Gamma^{-1}(\alpha_0, \beta_0) \\ +\end{align*} + +We note in passing that the model can be collapsed along $\mu$ to get: +\begin{align*} +y_i | p_i &\sim \mathrm{Binom}(n_i, p_i) \quad i = 1,\ldots, d \\ +p_i &= {\sf expit}(\theta_i + \mathrm{logit}(q_i) ) \quad i = 1,\ldots, d \\ +\theta | \sigma^2 &\sim \mathcal{N}(\mu_0 \mathbb{1}, \sigma^2 I + \sigma_0^2 \mathbb{1} \mathbb{1}^\top) \\ +\sigma^2 &\sim \Gamma^{-1}(\alpha_0, \beta_0) \\ +\end{align*} + + + +## Probability of Best Arm + + +The first quantity of interest is probability of best (treatment) arm. +Concretely, letting $i = 1$ denote the control arm, we wish to compute for each $1 < i \leq d$: +\begin{align*} +\mathbb{P}(p_i > \max\limits_{j \neq i} p_j | y, n) +&= +\int \mathbb{P}(p_i > \max\limits_{j \neq i} p_j | y, n, \sigma^2) p(\sigma^2 | y, n) \, d\sigma^2 +\\&= +\int \mathbb{P}(\theta_i + c_i > \max\limits_{j \neq i} (\theta_j + c_j) | y, n, \sigma^2) p(\sigma^2 | y, n) \, d\sigma^2 +\end{align*} +where $c = \mathrm{logit}(q)$. +We can approximate this quantity by estimating the two integrand terms separately. +By approximating $\theta_i | y, n$ as normal, the first integrand term can be estimated by Monte Carlo. +The second term can be estimated by INLA. + +```python +def pr_normal_best(mean, cov, key, n_sims): + ''' + Estimates P[X_i > max_{j != i} X_j] where X ~ N(mean, cov) via sampling. + ''' + out_shape = (n_sims, *mean.shape[:-1]) + sims = jax.random.multivariate_normal(key, mean, cov, shape=out_shape) + order = jnp.arange(1, mean.shape[-1]) + compute_pr_best_all = jax.vmap(lambda i: jnp.mean(jnp.argmax(sims, axis=-1) == i, axis=0)) + return compute_pr_best_all(order) +``` + +```python +d = 4 +mean = jnp.array([2, 2, 2, 5]) +cov = jnp.eye(d) +key = jax.random.PRNGKey(0) +n_sims = 100000 +jax.jit(pr_normal_best, static_argnums=(3,))(mean, cov, key, n_sims=n_sims) +``` + +Next, we perform INLA to estimate $p(\sigma^2 | y, n)$ on a grid of values for $\sigma^2$. + +```python +sig2_rule = quad.log_gauss_rule(15, 1e-6, 1e3) +sig2_rule_ops = berry.optimized(sig2_rule.pts, n_arms=4).config( + opt_tol=1e-3 +) +``` + +```python +def posterior_sigma_sq(data, sig2_rule, sig2_rule_ops): + n_arms, _ = data.shape + sig2 = sig2_rule.pts + n_sig2 = sig2.shape[0] + p_pinned = dict(sig2=sig2, theta=None) + + f = sig2_rule_ops.laplace_logpost + logpost, x_max, hess, iters = f( + np.zeros((n_sig2, n_arms)), p_pinned, data + ) + post = inla.exp_and_normalize( + logpost, sig2_rule.wts, axis=-1) + + return post, x_max, hess, iters +``` + +```python +dtype = jnp.float64 +N = 1 +data = berry.figure2_data(N).astype(dtype)[0] +n_arms, _ = data.shape +posterior_sigma_sq_jit = jax.jit(lambda data: posterior_sigma_sq(data, sig2_rule, sig2_rule_ops)) +post, _, hess, _ = posterior_sigma_sq_jit(data) +``` + +Putting the two pieces together, we have the following function to compute the probability of best treatment arm. + +```python +def pr_best(data, sig2_rule, sig2_rule_ops, key, n_sims): + n_arms, _ = data.shape + post, x_max, hess, _ = posterior_sigma_sq(data, sig2_rule, sig2_rule_ops) + mean = x_max + hess_fn = jax.vmap(lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1])) + prec = -hess_fn(hess) # (n_sigs, n_arms, n_arms) + cov = jnp.linalg.inv(prec) + pr_normal_best_out = pr_normal_best(mean, cov, key=key, n_sims=n_sims) + return jnp.matmul(pr_normal_best_out, post * sig2_rule.wts) +``` + +```python +n_sims = 13 +out = pr_best(data, sig2_rule, sig2_rule_ops, key, n_sims) +out +``` + +## Phase III Final Analysis + +\begin{align*} +\mathbb{P}(\theta_i - \theta_0 < t | y, n) < 0.1 +\end{align*} + +\begin{align*} +\mathbb{P}(\theta_i - \theta_0 < t | y, n) +&= +\mathbb{P}(q_1^\top \theta < t | y,n) += +\int \mathbb{P}(q_1^\top \theta < t | y, n, \sigma^2) p(\sigma^2 | y, n) \, d\sigma^2 +\\&= +\int \mathbb{P}(q_1^\top \theta < t | y, n, \sigma^2) p(\sigma^2 | y, n) \, d\sigma^2 +\\ +q_1^\top \theta | y, n, \sigma^2 &\sim \mathcal{N}(q_1^\top \theta^*, -q_1^\top (H\log p(\theta^*, y, \sigma^2))^{-1} q_1) +\end{align*} + +```python +posterior_difference_threshold = 0.2 +rejection_threshold = 0.1 +``` + +```python +def posterior_difference(data, arm, sig2_rule, sig2_rule_ops, thresh): + n_arms, _ = data.shape + post, x_max, hess, _ = posterior_sigma_sq(data, sig2_rule, sig2_rule_ops) + hess_fn = jax.vmap(lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1])) + prec = -hess_fn(hess) # (n_sigs, n_arms, n_arms) + order = jnp.arange(0, n_arms) + q1 = jnp.where(order == 0, -1, 0) + q1 = jnp.where(order == arm, 1, q1) + loc = x_max @ q1 + scale = jnp.linalg.solve(prec, q1[None,:]) @ q1 + normal_term = jax.scipy.stats.norm.cdf(thresh, loc=loc, scale=scale) + post_weighted = sig2_rule.wts * post + out = normal_term @ post_weighted + return out + +posterior_difference(data, 1, sig2_rule, sig2_rule_ops, posterior_difference_threshold) +``` + +## Posterior Probability of Success + +The next quantity we need to compute is the posterior probability of success (PPS). +For convenience of implementation, we will take this to mean the following: +let $y, n$ denote the currently observed data +and $A_i = \{ \text{Phase III rejects using treatment arm i} \}$. +Then, we wish to compute +\begin{align*} +\mathbb{P}(A_i | y, n) +\end{align*} +Expanding the quantity, +\begin{align*} +\mathbb{P}(A_i | y, n) &= +\int \mathbb{P}(A_i | y, n, \theta_i, \theta_0) p(\theta_0, \theta_i | y, n) \, d\theta_i d\theta_0 \\&= +\int \mathbb{P}(A_i | y, n, \theta_i, \theta_0) p(\theta_0, \theta_i | y, n) \, d\theta_i d\theta_0 +\end{align*} + +Once we have an estimate for $p(\theta_0, \theta_i | y, n)$, +we can use 2-D quadrature to numerically integrate the integrand. +Similar to computing the probability of best arm, +\begin{align*} +p(\theta_0, \theta_i | y, n) +&= +\int p(\theta_0, \theta_i | y, n, \sigma^2) p(\sigma^2 | y, n) \, d\sigma^2 +\end{align*} +We will use the Gaussian approximation for $p(\theta_0, \theta_i | y, n, \sigma^2)$ +and use INLA to estimate $p(\sigma^2 | y, n)$. + +\begin{align*} +p(\theta | y, n, \sigma^2) +\approx +\mathcal{N}(\theta^*, -(H\log p(\theta^*, y, \sigma^2))^{-1}) +\\ +\implies +p(\theta_0, \theta_i | y, n, \sigma^2) +\approx +\mathcal{N}(\theta^*_{[0,i]}, -(H\log p(\theta^*, y, \sigma^2))^{-1}_{[0,i], [0,i]}) +\end{align*} + +```python +# input parameters +n_Ai_sims = 1000 +p = jnp.full(n_arms, 0.5) +n_stage_2 = 100 +pps_threshold_lower = 0.1 +pps_threshold_upper = 0.9 +posterior_difference_threshold = 0.1 +rejection_threshold = 0.1 + +subset = jnp.array([0, 1]) +non_futile_idx = np.zeros(n_arms) +non_futile_idx[subset] = 1 +non_futile_idx = jnp.array(non_futile_idx) + +# create a dense grid of sig2 values +n_sig2 = 100 +sig2_grid = 10**jnp.linspace(-6, 3, n_sig2) +dsig2_grid = jnp.diff(sig2_grid) +sig2_grid_ops = berry.optimized(sig2_grid, n_arms=data.shape[0]).config( + opt_tol=1e-3 +) + +_, key = jax.random.split(key) +``` + +```python +def pr_Ai( + data, p, key, best_arm, non_futile_idx, + sig2_rule, sig2_rule_ops, + sig2_grid, sig2_grid_ops, dsig2_grid, +): + n_arms, _ = data.shape + + # compute p(sig2 | y, n), mode, hessian + p_pinned = dict(sig2=sig2_grid, theta=None) + logpost, x_max, hess, _ = jax.jit(sig2_grid_ops.laplace_logpost)( + np.zeros((len(sig2_grid), n_arms)), p_pinned, data + ) + max_logpost = jnp.max(logpost) + max_post = jnp.exp(max_logpost) + post = jnp.exp(logpost - max_logpost) * max_post + + # sample sigma^2 | y, n + dFx = post[:-1] * dsig2_grid + Fx = jnp.cumsum(dFx) + Fx /= Fx[-1] + _, key = jax.random.split(key) + unifs = jax.random.uniform(key=key, shape=(n_Ai_sims,)) + i_star = jnp.searchsorted(Fx, unifs) + + # sample theta | y, n, sigma^2 + mean = x_max[i_star+1] + hess_fn = jax.vmap( + lambda h: jnp.diag(h[0]) + jnp.full(shape=(n_arms, n_arms), fill_value=h[1]) + ) + prec = -hess_fn(hess) + cov = jnp.linalg.inv(prec)[i_star+1] + _, key = jax.random.split(key) + theta = jax.random.multivariate_normal( + key=key, mean=mean, cov=cov, + ) + p_samples = jax.scipy.special.expit(theta) + + # estimate P(A_i | y, n, theta_0, theta_i) + + def simulate_Ai(data, best_arm, diff_thresh, rej_thresh, non_futile_idx, key, p): + # add n_stage_2 number of patients to each + # of the control and selected treatment arms. + n_new = jnp.where(non_futile_idx, n_stage_2, 0) + y_new = dist.Binomial(total_count=n_new, probs=p).sample(key) + + # pool outcomes for each arm + data = data + jnp.stack((y_new, n_new), axis=-1) + + return posterior_difference(data, best_arm, sig2_rule, sig2_rule_ops, diff_thresh) < rej_thresh + + simulate_Ai_vmapped = jax.vmap( + simulate_Ai, in_axes=(None, None, None, None, None, 0, 0) + ) + keys = jax.random.split(key, num=p_samples.shape[0]) + Ai_indicators = simulate_Ai_vmapped( + data, + best_arm, + posterior_difference_threshold, + rejection_threshold, + non_futile_idx, + keys, + p_samples, + ) + out = jnp.mean(Ai_indicators) + return out + +``` + +```python +%%time +jax.jit(lambda data, p, key, best_arm, non_futile_idx: + pr_Ai(data, p, key, best_arm, non_futile_idx, + sig2_rule, sig2_rule_ops, sig2_grid, sig2_grid_ops, dsig2_grid), + static_argnums=(3,))(data, p, key, 1, non_futile_idx) +``` + +```python +# Sampling based on pdf values and linearly interpolating +n_sims = 1000 +n_unifs = 1000 +key = jax.random.PRNGKey(2) + +#x = jnp.linspace(-3, 3, num=n_sims) +#px_orig = 0.5*jax.scipy.stats.norm.pdf(x, -1, 0.5) + 0.5*jax.scipy.stats.norm.pdf(x, 1, 0.5) + +#x = jnp.linspace(0, 10, num=n_sims) +#px_orig = jax.scipy.stats.gamma.pdf(x, 10) + +x = jnp.linspace(0, 1, num=n_sims) +px_orig = jax.scipy.stats.beta.pdf(x, 4, 2) + +px = 2 * px_orig +dx = jnp.diff(x) +dFx = px[:-1] * dx +Fx = jnp.cumsum(dFx) +Fx /= Fx[-1] +_, key = jax.random.split(key) +unifs = jax.random.uniform(key=key, shape=(n_unifs,)) +i_star = jnp.searchsorted(Fx, unifs) + +# point mass approx +#samples = x[i_star+1] + +# constant approx +#samples = x[i_star+1] - (Fx[i_star] - unifs) / px[i_star] + +# linear approx +a = 0.5 * (px[i_star+1] - px[i_star]) / dx[i_star] +b = px[i_star] +c = Fx[i_star] - unifs - px[i_star] * dx[i_star] - a * dx[i_star]**2 +discr = jnp.sqrt(jnp.maximum(b**2 - 4*a*c, 0)) +quad_solve = jnp.where(jnp.abs(a) < 1e-8, -c/b, (-b + discr) / (2*a)) +samples = x[i_star] + quad_solve + +#plt.plot(x[1:], Fx) +#plt.plot(x[1:], jax.scipy.stats.norm.cdf(x[1:])) +plt.hist(x[i_star+1], density=True, alpha=0.5) +plt.hist(samples, density = True, alpha=0.5) +plt.plot(x, px_orig) +plt.show() +``` + +## Design Implementation + +```python +%%time +params = { + "n_arms" : 4, + "n_stage_1" : 50, + "n_stage_2" : 100, + "n_stage_1_interims" : 2, + "n_stage_1_add_per_interim" : 100, + "n_stage_2_add_per_interim" : 100, + "stage_1_futility_threshold" : 0.15, + "stage_1_efficacy_threshold" : 0.7, + "stage_2_futility_threshold" : 0.2, + "stage_2_efficacy_threshold" : 0.95, + "inter_stage_futility_threshold" : 0.6, + "posterior_difference_threshold" : 0, + "rejection_threshold" : 0.05, + "key" : jax.random.PRNGKey(0), + "n_pr_sims" : 100, + "n_sig2_sims" : 20, + "batch_size" : int(2**20), + "cache_tables" : False, +} +lei_obj = lewis.Lewis45(**params) +``` + +```python +batch_size = 2**12 +key = jax.random.PRNGKey(0) +n_points = 20 +n_pr_sims = 100 +n_sig2_sim = 20 +``` + +```python +%%time +lei_obj.pd_table = lei_obj.posterior_difference_table__( + batch_size=batch_size, + n_points=n_points, +) +lei_obj.pd_table +``` + +```python +%%time +lei_obj.pr_best_pps_1_table = lei_obj.pr_best_pps_1_table__( + key, + n_pr_sims, + batch_size=batch_size, + n_points=n_points, +) +lei_obj.pr_best_pps_1_table +``` + +```python +%%time +_, key = jax.random.split(key) +lei_obj.pps_2_table = lei_obj.pps_2_table__( + key, + n_pr_sims, + batch_size=batch_size, + n_points=n_points, +) +lei_obj.pps_2_table +``` + +```python +n_arms = params['n_arms'] +size = 52 +lower = np.full(n_arms, -1) +upper = np.full(n_arms, 1) +thetas, radii = lewgrid.make_cartesian_grid_range( + size=size, + lower=lower, + upper=upper, +) +ns = np.concatenate( + [np.ones(n_arms-1)[:, None], -np.eye(n_arms-1)], + axis=-1, +) +null_hypos = [ + grid.HyperPlane(n, 0) + for n in ns +] +gr = grid.build_grid( + thetas=thetas, + radii=radii, + null_hypos=null_hypos, +) +gr = grid.prune(gr) +``` + +```python +theta_tiles = gr.thetas[gr.grid_pt_idx] +null_truths = gr.null_truth.astype(bool) +grid_batch_size = int(2**12) +n_sim_batches = 500 +sim_batch_size = 50 + +p_tiles = jax.scipy.special.expit(theta_tiles) +``` + +```python +class LeiSimulator: + def __init__( + self, + lei_obj, + p_tiles, + null_truths, + grid_batch_size, + reduce_func=None, + ): + self.lei_obj = lei_obj + self.unifs_shape = self.lei_obj.unifs_shape() + self.unifs_order = np.arange(0, self.unifs_shape[0]) + self.p_tiles = p_tiles + self.null_truths = null_truths + self.grid_batch_size = grid_batch_size + + self.reduce_func = ( + lambda x: np.sum(x, axis=0) if not reduce_func else reduce_func + ) + + self.f_batch_sim_batch_grid_jit = jax.jit(self.f_batch_sim_batch_grid) + self.batch_all = batch.batch_all( + self.f_batch_sim_batch_grid_jit, + batch_size=self.grid_batch_size, + in_axes=(0, 0, None, None), + ) + + self.typeI_sum = None + self.typeI_score = None + + def f_batch_sim_batch_grid(self, p_batch, null_batch, unifs_batch, unifs_order): + return jax.vmap( + jax.vmap( + self.lei_obj.simulate, + in_axes=(0, 0, None, None), + ), + in_axes=(None, None, 0, None), + )(p_batch, null_batch, unifs_batch, unifs_order) + + def simulate_batch_sim(self, sim_batch_size, i, key): + start = time.perf_counter() + + unifs = jax.random.uniform(key=key, shape=(sim_batch_size,) + self.unifs_shape) + rejs_scores, n_padded = self.batch_all( + self.p_tiles, self.null_truths, unifs, self.unifs_order + ) + rejs, scores = tuple( + np.concatenate( + tuple(x[i] for x in rejs_scores), + axis=1, + ) + for i in range(2) + ) + rejs, scores = ( + (rejs[:, :-n_padded], scores[:, :-n_padded, :]) + if n_padded + else (rejs, scores) + ) + rejs_reduced = self.reduce_func(rejs) + scores_reduced = self.reduce_func(scores) + + end = time.perf_counter() + elapsed_time = (end-start) + print(f"Batch {i}: {elapsed_time:.03f}s") + return rejs_reduced, scores_reduced + + def simulate( + self, + key, + n_sim_batches, + sim_batch_size, + ): + keys = jax.random.split(key, num=n_sim_batches) + self.typeI_sum = np.zeros(self.p_tiles.shape[0]) + self.typeI_score = np.zeros(self.p_tiles.shape) + for i, key in enumerate(keys): + out = self.simulate_batch_sim(sim_batch_size, i, key) + self.typeI_sum += out[0] + self.typeI_score += out[1] + return self.typeI_sum, self.typeI_score + + +simulator = LeiSimulator( + lei_obj=lei_obj, + p_tiles=p_tiles, + null_truths=null_truths, + grid_batch_size=grid_batch_size, +) + +``` + +```python +%%time +key = jax.random.PRNGKey(3) +typeI_sum, typeI_score = simulator.simulate( + key=key, + n_sim_batches=n_sim_batches, + sim_batch_size=sim_batch_size, +) +``` + +```python +os.makedirs("output_lei4d2", exist_ok=True) +np.savetxt("output_lei4d2/typeI_sum.csv", typeI_sum, fmt="%s", delimiter=",") +np.savetxt("output_lei4d2/typeI_score.csv", typeI_score, fmt="%s", delimiter=",") +``` + +```python +sim_size = sim_batch_size * n_sim_batches + +plt.figure(figsize=(8,4), constrained_layout=True) +for i, t2_idx in enumerate([4, 8]): + t2 = np.unique(theta_tiles[:, 2])[t2_idx] + selection = (theta_tiles[:,2] == t2) + + plt.subplot(1,2,i+1) + plt.title(f'slice: $\\theta_2 \\approx$ {t2:.1f}') + plt.scatter(theta_tiles[selection,0], theta_tiles[selection,1], c=typeI_sum[selection]/sim_size, s=90) + cbar = plt.colorbar() + plt.xlabel(r'$\theta_0$') + plt.ylabel(r'$\theta_1$') + cbar.set_label('Simulated fraction of Type I errors') +plt.show() +``` + +```python +tile_radii = gr.radii[gr.grid_pt_idx] +sim_sizes = np.full(gr.n_tiles, sim_size) +n_arm_samples = ( + params['n_stage_1'] + + params['n_stage_1_add_per_interim'] // 2 * params['n_stage_1_interims'] + + params['n_stage_2'] + + params['n_stage_2_add_per_interim'] // 2 +) +total, d0, d0u, d1w, d1uw, d2uw = binomial.upper_bound( + theta_tiles, + tile_radii, + gr.vertices, + sim_sizes, + n_arm_samples, + typeI_sum, + typeI_score, +) +bound_components = np.array([ + d0, + d0u, + d1w, + d1uw, + d2uw, + total, +]).T +``` + +```python +t2_uniques = np.unique(theta_tiles[:, 2]) +t3_uniques = np.unique(theta_tiles[:, 3]) +t2 = t2_uniques[8] +t3 = t3_uniques[8] +selection = (theta_tiles[:, 2] == t2) & (theta_tiles[:, 3] == t3) + +np.savetxt('output_lei4d/P_lei.csv', theta_tiles[selection, :].T, fmt="%s", delimiter=",") +np.savetxt('output_lei4d/B_lei.csv', bound_components[selection, :], fmt="%s", delimiter=",") +``` + +```python +t2_uniques[8], t3_uniques[8] +``` + +# Sandbox diff --git a/research/stat/poisson_process.ipynb b/research/stat/poisson_process.ipynb new file mode 100644 index 00000000..81e0c0fd --- /dev/null +++ b/research/stat/poisson_process.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Poisson Process Fun Time" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import jax.numpy as jnp\n", + "import jax\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "metadata": {}, + "outputs": [], + "source": [ + "def h(p):\n", + " return p\n", + "\n", + "def method_1(lam, n_sims, t, seed):\n", + " key = jax.random.PRNGKey(seed)\n", + "\n", + " Ns = jax.random.poisson(key=key, lam=lam, shape=(n_sims,))\n", + " max_Ns = jnp.max(Ns)\n", + " order = jnp.arange(0, max_Ns)\n", + " \n", + " def stat(N, key): \n", + " p = jax.random.uniform(key=key, shape=(max_Ns,))\n", + " p_sub = jnp.where(order < N, p, jnp.nan)\n", + " return jnp.sum(h(p_sub) * (p_sub < t))\n", + " \n", + " keys = jax.random.split(key, num=n_sims)\n", + " \n", + " stat_vmapped = jax.vmap(stat, in_axes=(0,0))\n", + " stat_vmapped_jit = jax.jit(stat_vmapped)\n", + " out = stat_vmapped_jit(Ns, keys)\n", + " return out\n", + "\n", + "def method_2(lam, n_sims, t, seed, n_begin=10):\n", + " key = jax.random.PRNGKey(seed)\n", + "\n", + " # sample Exp(lam) until the running sum is >= 1, then take everything before that point.\n", + " # If X_1,..., X_n ~ Exp(lam) and T_i = sum_{j=1}^i X_j,\n", + " # then (T_1,..., T_{n-1}) | T_n = t ~ (U_{(1)}, ..., U_{(n-1)}) where each U_i ~ Unif(0, t)\n", + " # \n", + " # Sampling procedure:\n", + " # - Increase n until T_n >= 1\n", + " # - Sample (T_1,..., T_{n-1}) | T_n via formula above.\n", + " # - Sum over h(T_i) 1{T_i < t}\n", + "\n", + " def find_n_T_n(n_begin, key):\n", + " n = n_begin\n", + " T = 0\n", + " def body_fun(tup, key):\n", + " n, _ = tup\n", + " n = n + n_begin\n", + " _, key = jax.random.split(key)\n", + " return (n, jax.random.gamma(key=key, a=n) / lam)\n", + " out = jax.lax.while_loop(\n", + " lambda tup: tup[1] < 1, \n", + " lambda tup: body_fun(tup, key), \n", + " (n, T))\n", + " return jnp.array(out)\n", + "\n", + " keys = jax.random.split(key, num=n_sims)\n", + " NT = jax.jit(jax.vmap(find_n_T_n, in_axes=(None, 0)))(n_begin, keys)\n", + " \n", + " N_max = int(jnp.max(NT[:,0]))\n", + " order = jnp.arange(0, N_max)\n", + " def stat(nt, key):\n", + " n, t_n = nt\n", + " unifs = jax.random.uniform(key=key, shape=(N_max,))\n", + " unifs = jnp.where(order < (n-1), unifs, jnp.inf)\n", + " unifs_sorted = jnp.sort(unifs)\n", + " Ts = t_n * unifs_sorted\n", + " return jnp.sum(h(Ts) * (Ts < t))\n", + "\n", + " stat_vmapped = jax.vmap(stat, in_axes=(0,0))\n", + " stat_vmapped_jit = jax.jit(stat_vmapped)\n", + "\n", + " keys = jax.random.split(keys[-1], num=n_sims)\n", + " return stat_vmapped_jit(NT, keys)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 158, + "metadata": {}, + "outputs": [], + "source": [ + "lam = 100\n", + "n_sims = 100000\n", + "t = 0.2\n", + "seed = 69" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "metadata": {}, + "outputs": [], + "source": [ + "out_1 = method_1(lam=lam, n_sims=n_sims, t=t, seed=seed)\n", + "out_2 = method_2(lam=lam, n_sims=n_sims, t=t, seed=seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAY3klEQVR4nO3df4xV9Z3/8edLRMAfREUwlMGCDVJhdAcYkY2pFRVltRFt16807oqtZlpC+9Xtt11hE7tuEypNuupqW5NJ/YHdtoirhNGvPxZZ7JYGpINSFZB1Vlm8gS9M+a5Z7MYfo+/9436kl5k7M3fgcu7MnNcjmdxz3/d8zv0cDe/5zPt8zucoIjAzs3w4ptYdMDOz7Djpm5nliJO+mVmOOOmbmeWIk76ZWY4cW+sO9Oa0006LCRMm1LobZmYDyubNm38fEaM7x/t90p8wYQKtra217oaZ2YAi6T/KxV3eMTPLESd9M7MccdI3M8uRfl/TN7PB48MPP6RQKPDee+/VuiuDxvDhw6mrq2Po0KEV7e+kb2aZKRQKnHTSSUyYMAFJte7OgBcR7N+/n0KhwMSJEytq4/KOmWXmvffeY9SoUU74VSKJUaNG9ekvJyd9M8uUE3519fW/p5O+mVmOuKZvZjVz95p/q+rx/mrOWVU93mDkpG/9Sk9JwP+grda2bNnC7t27ueKKKwC44447OPHEE/n2t799WMfrrf1jjz3GHXfcwfbt29m0aRONjY2H3fdPOOlbTVR7hGeWhS1bttDa2now6R9t9fX1PPHEE3zta1+r2jFd0zezXNm5cyef/exnufnmm6mvr+f666/n+eef54ILLmDSpEls2rSJP/zhD3z1q1/lvPPOY9q0aaxevZoPPviA7373uzz66KM0NDTw6KOPArBt2zYuuugizjzzTO69996D33PXXXdRX19PfX0999xzz8H40qVLmTx5Mpdeeik7duzosa9nn302kydPrur5e6RvZrnT1tbGY489RnNzM+eddx6/+MUvWL9+PS0tLXz/+99nypQpXHzxxTz44IO88847zJw5k0svvZTvfe97tLa28qMf/Qgolmdef/111q1bx4EDB5g8eTILFy7klVde4aGHHuLFF18kIjj//PP5/Oc/z8cff8yKFSt4+eWX6ejoYPr06cyYMSPTc3fSN7PcmThxIueccw4AU6dO5ZJLLkES55xzDjt37qRQKNDS0sIPf/hDoHh/wa5du8oe68orr2TYsGEMGzaMMWPGsHfvXtavX88111zDCSecAMAXv/hFfv3rX/Pxxx9zzTXXcPzxxwNw1VVXZXC2h3LSN7PcGTZs2MHtY4455uD7Y445ho6ODoYMGcLjjz/epbTy4osv9nisIUOG0NHRQUR0+921vk/BSd/6hVm7mrvENp7RVIOeWJb664ysyy+/nPvuu4/77rsPSbz88stMmzaNk046iQMHDvTa/sILL+TGG29k8eLFRASrVq3iZz/7GRFxMN7R0cGTTz5Z1Yu0lXDSt6PKs3RsILr99tu59dZbOffcc4kIJkyYwFNPPcXs2bNZtmwZDQ0NLFmypNv206dP58Ybb2TmzJkA3HzzzUybNg2A6667joaGBj796U/zuc99rsd+rFq1im9+85u0t7dz5ZVX0tDQwHPPPXdE56ae/gzpDxobG8NPzhq4Oif9ciP67lQ60u+vo0Xravv27Zx99tm17sagU+6/q6TNEdFlYr9H+tZvdfcLwmUfs8PnpG9mVmOLFi3iN7/5zSGxW265ha985StV/65ek76kycCjJaEzge8Cj6T4BGAn8L8i4j9TmyXATcBHwP+OiOdSfAbwMDACeBq4Jfp7fcnM7Cj78Y9/nNl39XpHbkTsiIiGiGgAZgD/DawCFgNrI2ISsDa9R9IUYD4wFZgL/ETSkHS4+4EmYFL6mVvVszEzsx71dRmGS4B/j4j/AOYBy1N8OXB12p4HrIiI9yPiLaANmClpLDAyIjak0f0jJW3MzCwDfU3684Ffpu3TI2IPQHodk+LjgLdL2hRSbFza7hzvQlKTpFZJre3t7X3sopmZdafiC7mSjgOuArqfnJp2LROLHuJdgxHNQDMUp2xW2kczG2DW3Vnd483uLT1ZX2bv/BnwUkTsTe/3ShobEXtS6WZfiheA8SXt6oDdKV5XJm7WJ12mcq4bVXz1P3g7yrJeT/873/kOTz75JMcddxyf+cxneOihhzj55JMPt/tA38o7X+aPpR2AFmBB2l4ArC6Jz5c0TNJEihdsN6US0AFJs1RcfOKGkjZmZv3eli1bePrppzP7vjlz5vDaa6/xyiuvcNZZZ3HnnUf+l1FFSV/S8cAc4ImS8DJgjqQ30mfLACJiK7AS2AY8CyyKiI9Sm4XATyle3P134JkjPgMzsz4YSOvpX3bZZRx7bLEgM2vWLAqFQo/7V6Ki8k5E/DcwqlNsP8XZPOX2XwosLRNvBer73k0zs+oZiOvpP/jgg1x33XVHfO6+I9cGvA1v7gdgY8eh6/x4TR7rzkBbT3/p0qUce+yxXH/99Ud66k76dpSkWRmzdu2vcUfMuhpI6+kvX76cp556irVr11ZlLX4nfTOrnX4646q/rKf/7LPP8oMf/IBf/epXB/86OFJ+MLqZWSe33347H374Ieeeey719fXcfvvtAMyePZtt27YdciG3nNL19M8///yD6+lPnz794Hr6X/rSl3pdT/8b3/gGBw4cYM6cOTQ0NPD1r3/9iM/N6+lbVRzJuvnV0nnJZdf0+x+vp3909GU9fY/0zcxyxDV9M7Ma61fr6ZuZVVNEVGUWymByJOvp97VE7/KOmWVm+PDh7N+/v8+JysqLCPbv38/w4cMrbuORvpllpq6ujkKhgJdMr57hw4dTV1fX+46Jk76ZZWbo0KFMnDix1t3INZd3zMxyxEnfzCxHXN6xQcMPVzHrnZO+HRkvrGY2oLi8Y2aWI076ZmY54qRvZpYjlT4j92RJ/yTpdUnbJf2ppFMlrZH0Rno9pWT/JZLaJO2QdHlJfIakV9Nn98r3YpuZZarSC7n/ADwbEX8u6TjgeOBvgLURsUzSYmAxcJukKcB8YCrwKeB5SWelh6PfDzQBG4Gngbn44eh2lPgximZd9TrSlzQSuBB4ACAiPoiId4B5wPK023Lg6rQ9D1gREe9HxFtAGzBT0lhgZERsiOLCG4+UtDEzswxUUt45E2gHHpL0sqSfSjoBOD0i9gCk1zFp/3HA2yXtCyk2Lm13jnchqUlSq6RWr9FhZlY9lST9Y4HpwP0RMQ34A8VSTnfK1emjh3jXYERzRDRGROPo0aMr6KKZmVWikqRfAAoR8clj4P+J4i+BvalkQ3rdV7L/+JL2dcDuFK8rEzczs4z0mvQj4v8Bb0uanEKXANuAFmBBii0AVqftFmC+pGGSJgKTgE2pBHRA0qw0a+eGkjZmZpaBSmfvfBP4eZq58ybwFYq/MFZKugnYBVwLEBFbJa2k+IuhA1iUZu4ALAQeBkZQnLXjmTtmZhmqKOlHxBagy1PVKY76y+2/FFhaJt4K1Pehf2ZmVkVecM365O41h85590JrZgOLl2EwM8sRJ30zsxxx0jczyxHX9G3Q8xO1zP7II30zsxzxSN8q48cimg0KHumbmeWIk76ZWY446ZuZ5YiTvplZjvhCruWOH6NoeeaRvplZjjjpm5nliJO+mVmOOOmbmeWIk76ZWY446ZuZ5YiTvplZjlSU9CXtlPSqpC2SWlPsVElrJL2RXk8p2X+JpDZJOyRdXhKfkY7TJuleSar+KZmZWXf6cnPW7Ij4fcn7xcDaiFgmaXF6f5ukKcB8YCrwKeB5SWdFxEfA/UATsBF4GpgLPFOF87Aq87NwzQanIynvzAOWp+3lwNUl8RUR8X5EvAW0ATMljQVGRsSGiAjgkZI2ZmaWgUqTfgD/LGmzpKYUOz0i9gCk1zEpPg54u6RtIcXGpe3O8S4kNUlqldTa3t5eYRfNzKw3lZZ3LoiI3ZLGAGskvd7DvuXq9NFDvGswohloBmhsbCy7j5mZ9V1FI/2I2J1e9wGrgJnA3lSyIb3uS7sXgPElzeuA3SleVyZuZmYZ6TXpSzpB0kmfbAOXAa8BLcCCtNsCYHXabgHmSxomaSIwCdiUSkAHJM1Ks3ZuKGljZmYZqKS8czqwKs2uPBb4RUQ8K+m3wEpJNwG7gGsBImKrpJXANqADWJRm7gAsBB4GRlCcteOZO2ZmGeo16UfEm8CflInvBy7pps1SYGmZeCtQ3/dumlXfrF3NhwbWjSq+zl6SfWfMMuKHqJglfriK5YGXYTAzyxEnfTOzHHHSNzPLESd9M7MccdI3M8sRJ30zsxzxlE071Lo7AS+lbDZYeaRvZpYjTvpmZjnipG9mliNO+mZmOeKkb2aWI569Y9aJV9+0wcwjfTOzHHHSNzPLESd9M7MccdI3M8uRipO+pCGSXpb0VHp/qqQ1kt5Ir6eU7LtEUpukHZIuL4nPkPRq+uze9IB0MzPLSF9G+rcA20veLwbWRsQkYG16j6QpwHxgKjAX+ImkIanN/UATMCn9zD2i3puZWZ9UNGVTUh1wJcWHnX8rhecBF6Xt5cALwG0pviIi3gfektQGzJS0ExgZERvSMR8BrgaeqcJ52GG6e82hz4P1Qmtmg1ulI/17gL8GPi6JnR4RewDS65gUHwe8XbJfIcXGpe3OcTMzy0ivSV/SF4B9EbG5wmOWq9NHD/Fy39kkqVVSa3t7e4Vfa2ZmvamkvHMBcJWkK4DhwEhJ/wjslTQ2IvZIGgvsS/sXgPEl7euA3SleVybeRUQ0A80AjY2NZX8xmGVlw5vFktfGjkNLYX8156xadMfsiPQ60o+IJRFRFxETKF6g/ZeI+AugBViQdlsArE7bLcB8ScMkTaR4wXZTKgEdkDQrzdq5oaSNmZll4EjW3lkGrJR0E7ALuBYgIrZKWglsAzqARRHxUWqzEHgYGEHxAq4v4pqZZahPST8iXqA4S4eI2A9c0s1+SynO9OkcbwXq+9pJMzOrDt+Ra2aWI076ZmY54qRvZpYjTvpmZjniJ2fl1bo7AS+7YJY3HumbmeWIk76ZWY446ZuZ5Yhr+mYVmrWr+dDAulEwe0ltOmN2mDzSNzPLESd9M7MccdI3M8sRJ30zsxxx0jczyxEnfTOzHHHSNzPLEc/TNztMG97c3+W5ueBn51r/5qSfE3evOTQ5eaE1s3xyecfMLEd6TfqShkvaJOl3krZK+rsUP1XSGklvpNdTStoskdQmaYeky0viMyS9mj67V5KOzmmZmVk5lYz03wcujog/ARqAuZJmAYuBtRExCVib3iNpCjAfmArMBX4iaUg61v1AEzAp/cyt3qmYmVlvek36UfRuejs0/QQwD1ie4suBq9P2PGBFRLwfEW8BbcBMSWOBkRGxISICeKSkjZmZZaCimr6kIZK2APuANRHxInB6ROwBSK9j0u7jgLdLmhdSbFza7hwv931Nkloltba3t/fhdMzMrCcVJf2I+CgiGoA6iqP2+h52L1enjx7i5b6vOSIaI6Jx9OjRlXTRzMwq0KfZOxHxDvACxVr83lSyIb3uS7sVgPElzeqA3SleVyZuZmYZqWT2zmhJJ6ftEcClwOtAC7Ag7bYAWJ22W4D5koZJmkjxgu2mVAI6IGlWmrVzQ0kbMzPLQCU3Z40FlqcZOMcAKyPiKUkbgJWSbgJ2AdcCRMRWSSuBbUAHsCgiPkrHWgg8DIwAnkk/ZmaWERUn0vRfjY2N0draWutuDFzr7gSKSwZYNv70TD9G0WpP0uaIaOwc9x25ZmY54qRvZpYjTvpmZjnipG9mliNO+mZmOeKkb2aWI076ZmY54idnmVWZH6No/ZlH+mZmOeKkb2aWI076ZmY54qRvZpYjTvpmZjnipG9mliNO+mZmOeJ5+mZHwaxdzV2D67zOvtWeR/pmZjnikf4gc/eaQ+8EnbXLT8wysz/ySN/MLEd6TfqSxktaJ2m7pK2SbknxUyWtkfRGej2lpM0SSW2Sdki6vCQ+Q9Kr6bN7JenonJaZmZVTSXmnA/g/EfGSpJOAzZLWADcCayNimaTFwGLgNklTgPnAVOBTwPOSzoqIj4D7gSZgI/A0MBd4ptonlVvr7nQ5x8x61OtIPyL2RMRLafsAsB0YB8wDlqfdlgNXp+15wIqIeD8i3gLagJmSxgIjI2JDRATwSEkbMzPLQJ9q+pImANOAF4HTI2IPFH8xAGPSbuOAt0uaFVJsXNruHC/3PU2SWiW1tre396WLZmbWg4qTvqQTgceBWyPiv3ratUwseoh3DUY0R0RjRDSOHj260i6amVkvKpqyKWkoxYT/84h4IoX3ShobEXtS6WZfiheA8SXN64DdKV5XJm6WC364ivUHlczeEfAAsD0i7ir5qAVYkLYXAKtL4vMlDZM0EZgEbEoloAOSZqVj3lDSxszMMlDJSP8C4C+BVyVtSbG/AZYBKyXdBOwCrgWIiK2SVgLbKM78WZRm7gAsBB4GRlCcteOZO2ZmGeo16UfEesrX4wEu6abNUmBpmXgrUN+XDpqZWfX4jlwzsxzx2jtmGfLqm1ZrHumbmeWIk76ZWY446ZuZ5YiTvplZjvhC7gDV+WEp4AemmFnvnPTNaszLM1iWXN4xM8sRj/QHIj8sxcwOk0f6ZmY54pG+WT/gO3UtKx7pm5nliJO+mVmOOOmbmeWIk76ZWY446ZuZ5YiTvplZjlTyYPQHJe2T9FpJ7FRJayS9kV5PKflsiaQ2STskXV4SnyHp1fTZvenh6GZmlqFKRvoPA3M7xRYDayNiErA2vUfSFGA+MDW1+YmkIanN/UATMCn9dD6mmZkdZZU8GP1fJU3oFJ4HXJS2lwMvALel+IqIeB94S1IbMFPSTmBkRGwAkPQIcDXwzBGfgdkg1d1CbODF2OzwHe4duadHxB6AiNgjaUyKjwM2luxXSLEP03bneFmSmij+VcAZZ5xxmF00G/jK3qkLwA8z7YcNHtVehqFcnT56iJcVEc1AM0BjY2O3++WB1803s2o63KS/V9LYNMofC+xL8QIwvmS/OmB3iteViVtPvJqmmVXZ4U7ZbAEWpO0FwOqS+HxJwyRNpHjBdlMqBR2QNCvN2rmhpI2ZmWWk15G+pF9SvGh7mqQC8LfAMmClpJuAXcC1ABGxVdJKYBvQASyKiI/SoRZSnAk0guIFXF/ENTPLWCWzd77czUeXdLP/UmBpmXgrUN+n3plZWeWu9YBn9VjvvJ6+2QDkWT12uLwMg5lZjjjpm5nliJO+mVmOuKbfT/gmLDPLgpN+f+CbsMwsI076ZoOIp3Jab1zTNzPLEY/0zQYRz9+33nikb2aWI076ZmY54vKOWQ74Aq99wknfLAfK1fo3ntFUg55YrTnpZ2jDA9+udRfMLOdc0zczyxGP9M1yatauZjY80DW+8Ywm1/oHMY/0zcxyxCP9o2HdnbXugdlh818Ag5uT/lGw4U0vnmZm/VPmSV/SXOAfgCHATyNiWdZ9MLO+818Ag0OmSV/SEODHwBygAPxWUktEbMuyH9XiKZhmPf8y6I5/SdRO1iP9mUBbRLwJIGkFMA/oP0m/TD3e5Rqzvut+8TfK/pLojv+SqK6sk/444O2S9wXg/M47SWoCPhkmvCtpR8nHpwG/P2o97N987vmU13NP5/33fKvWPcleNf6ff7pcMOukrzKx6BKIaAbKDhMktUZEY7U7NhD43H3ueZLX84aje+5Zz9MvAONL3tcBuzPug5lZbmWd9H8LTJI0UdJxwHygJeM+mJnlVqblnYjokPQN4DmKUzYfjIitfTxM91eHBj+fez7l9dzzet5wFM9dEV1K6mZmNkh57R0zsxxx0jczy5EBlfQlzZW0Q1KbpMW17k9WJD0oaZ+k12rdlyxJGi9pnaTtkrZKuqXWfcqKpOGSNkn6XTr3v6t1n7ImaYiklyU9Veu+ZEnSTkmvStoiqbXqxx8oNf20hMO/UbKEA/DlgbqEQ19IuhB4F3gkIupr3Z+sSBoLjI2IlySdBGwGrs7J/3MBJ0TEu5KGAuuBWyJiY427lhlJ3wIagZER8YVa9ycrknYCjRFxVG7IG0gj/YNLOETEB8AnSzgMehHxr8D/r3U/shYReyLipbR9ANhO8a7uQS+K3k1vh6afgTFCqwJJdcCVwE9r3ZfBZiAl/XJLOOQiARhImgBMA16scVcyk8obW4B9wJqIyM25A/cAfw18XON+1EIA/yxpc1qSpqoGUtKvaAkHG3wknQg8DtwaEf9V6/5kJSI+iogGineuz5SUi9KepC8A+yJic637UiMXRMR04M+ARam8WzUDKel7CYccSvXsx4GfR8QTte5PLUTEO8ALwNza9iQzFwBXpdr2CuBiSf9Y2y5lJyJ2p9d9wCqKpe2qGUhJ30s45Ey6mPkAsD0i7qp1f7IkabSkk9P2COBS4PWadiojEbEkIuoiYgLFf+f/EhF/UeNuZULSCWnSApJOAC4Dqjprb8Ak/YjoAD5ZwmE7sPIwlnAYkCT9EtgATJZUkHRTrfuUkQuAv6Q40tuSfq6odacyMhZYJ+kVigOeNRGRq6mLOXU6sF7S74BNwP+NiGer+QUDZsqmmZkduQEz0jczsyPnpG9mliNO+mZmOeKkb2aWI076ZmY54qRvZpYjTvpmZjnyP/C4iJ6NTk+sAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.hist(out_1, bins=50, alpha=0.5, label='method_1')\n", + "plt.hist(out_2, bins=50, alpha=0.5, label='method_2')\n", + "plt.legend()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.5 ('confirm')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "d8e1ca1b3fede25e3995e2b26ea544fa1b75b9a17984e6284a43c1dc286640dd" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/research/stat/poisson_process.md b/research/stat/poisson_process.md new file mode 100644 index 00000000..223b8793 --- /dev/null +++ b/research/stat/poisson_process.md @@ -0,0 +1,111 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3.10.5 ('confirm') + language: python + name: python3 +--- + +# Poisson Process Fun Time + +```python +import numpy as np +import jax.numpy as jnp +import jax +import matplotlib.pyplot as plt +``` + +```python +def h(p): + return p + +def method_1(lam, n_sims, t, seed): + key = jax.random.PRNGKey(seed) + + Ns = jax.random.poisson(key=key, lam=lam, shape=(n_sims,)) + max_Ns = jnp.max(Ns) + order = jnp.arange(0, max_Ns) + + def stat(N, key): + p = jax.random.uniform(key=key, shape=(max_Ns,)) + p_sub = jnp.where(order < N, p, jnp.nan) + return jnp.sum(h(p_sub) * (p_sub < t)) + + keys = jax.random.split(key, num=n_sims) + + stat_vmapped = jax.vmap(stat, in_axes=(0,0)) + stat_vmapped_jit = jax.jit(stat_vmapped) + out = stat_vmapped_jit(Ns, keys) + return out + +def method_2(lam, n_sims, t, seed, n_begin=10): + key = jax.random.PRNGKey(seed) + + # sample Exp(lam) until the running sum is >= 1, then take everything before that point. + # If X_1,..., X_n ~ Exp(lam) and T_i = sum_{j=1}^i X_j, + # then (T_1,..., T_{n-1}) | T_n = t ~ (U_{(1)}, ..., U_{(n-1)}) where each U_i ~ Unif(0, t) + # + # Sampling procedure: + # - Increase n until T_n >= 1 + # - Sample (T_1,..., T_{n-1}) | T_n via formula above. + # - Sum over h(T_i) 1{T_i < t} + + def find_n_T_n(n_begin, key): + n = n_begin + T = 0 + def body_fun(tup, key): + n, _ = tup + n = n + n_begin + _, key = jax.random.split(key) + return (n, jax.random.gamma(key=key, a=n) / lam) + out = jax.lax.while_loop( + lambda tup: tup[1] < 1, + lambda tup: body_fun(tup, key), + (n, T)) + return jnp.array(out) + + keys = jax.random.split(key, num=n_sims) + NT = jax.jit(jax.vmap(find_n_T_n, in_axes=(None, 0)))(n_begin, keys) + + N_max = int(jnp.max(NT[:,0])) + order = jnp.arange(0, N_max) + def stat(nt, key): + n, t_n = nt + unifs = jax.random.uniform(key=key, shape=(N_max,)) + unifs = jnp.where(order < (n-1), unifs, jnp.inf) + unifs_sorted = jnp.sort(unifs) + Ts = t_n * unifs_sorted + return jnp.sum(h(Ts) * (Ts < t)) + + stat_vmapped = jax.vmap(stat, in_axes=(0,0)) + stat_vmapped_jit = jax.jit(stat_vmapped) + + keys = jax.random.split(keys[-1], num=n_sims) + return stat_vmapped_jit(NT, keys) + +``` + +```python +lam = 100 +n_sims = 100000 +t = 0.2 +seed = 69 +``` + +```python +out_1 = method_1(lam=lam, n_sims=n_sims, t=t, seed=seed) +out_2 = method_2(lam=lam, n_sims=n_sims, t=t, seed=seed) +``` + +```python +plt.hist(out_1, bins=50, alpha=0.5, label='method_1') +plt.hist(out_2, bins=50, alpha=0.5, label='method_2') +plt.legend() +plt.show() +```