From 043d5fac3c7e1b877cea64ed9bb3562cffcf54c1 Mon Sep 17 00:00:00 2001 From: QueensGambit Date: Sun, 25 Nov 2018 17:01:40 +0100 Subject: [PATCH 1/3] updates for CrazyAra 0.3.0 - higher NPS, multiple netpredictiors, search tree pruning option, oscilating cpuct option, q-value-factor --- .gitignore | 1 + CrazyAra-log-bug.txt | 196 -------- .../src/domain/abstract_cls/_GameState.py | 16 +- .../src/domain/agent/NeuralNetAPI.py | 8 +- .../src/domain/agent/player/MCTSAgent.py | 445 ++++++++++-------- .../src/domain/agent/player/RawNetAgent.py | 28 +- .../src/domain/agent/player/_Agent.py | 84 ++-- .../agent/player/util/NetPredService.py | 82 +++- .../src/domain/agent/player/util/Node.py | 77 ++- .../src/domain/crazyhouse/GameState.py | 51 +- .../domain/crazyhouse/input_representation.py | 48 +- .../crazyhouse/output_representation.py | 2 +- DeepCrazyhouse/src/domain/util.py | 5 + .../runtime/{Colorer.py => ColorLogger.py} | 40 +- .../src/samples/MCTS_eval_demo.ipynb | 230 ++++++++- .../src/tests/FullRoundTripTests.py | 7 +- crazyara.py | 289 +++++++++--- setup.py | 16 - 18 files changed, 918 insertions(+), 707 deletions(-) delete mode 100644 CrazyAra-log-bug.txt rename DeepCrazyhouse/src/runtime/{Colorer.py => ColorLogger.py} (78%) delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 36299fad..1a974345 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,4 @@ main_config.py # avoid pushing log-files generated by uci-communication CrazyAra-log.txt +score-log.txt diff --git a/CrazyAra-log-bug.txt b/CrazyAra-log-bug.txt deleted file mode 100644 index 66acb680..00000000 --- a/CrazyAra-log-bug.txt +++ /dev/null @@ -1,196 +0,0 @@ -> uci -< id name CrazyAra 0.2.0 -< id author Johannes Czech, Moritz Willig, Alena Beyer et al. -< option name UCI_Variant type combo default crazyhouse var crazyhouse -< option name context type combo default cpu var cpu var gpu -< option name use_raw_network type check default false -< option name threads type spin default 16 min 1 max 4096 -< option name batch_size type spin default 8 min 1 max 4096 -< option name playouts_empty_pockets type spin default 8192 min 56 max 8192 -< option name playouts_filled_pockets type spin default 8192 min 56 max 8192 -< option name centi_cpuct type spin default 300 min 1 max 500 -< option name centi_dirichlet_epsilon type spin default 10 min 0 max 100 -< option name centi_dirichlet_alpha type spin default 20 min 0 max 100 -< option name max_search_depth type spin default 40 min 1 max 100 -< option name centi_temperature type spin default 0 min 0 max 100 -< option name centi_clip_quantil type spin default 0 min 0 max 100 -< option name virtual_loss type spin default 3 min 0 max 10 -< option name centi_q_value_weight type spin default 70 min 0 max 100 -< option name threshold_time_for_raw_net_ms type spin default 100 min 1 max 300000 -< option name move_overhead_ms type spin default 300 min 0 max 60000 -< option name moves_left type spin default 40 min 10 max 320 -< option name extend_time_on_bad_position type check default true -< option name max_move_num_to_reduce_movetime type spin default 0 min 0 max 120 -< option name check_mate_in_one type check default false -< option name enable_timeout type check default false -< option name verbose type check default false -< uciok -> setoption name batch_size value 8 -< info string Updated option batch_size to 8 -> setoption name centi_clip_quantil value 0 -< info string Updated option centi_clip_quantil to 0 -> setoption name centi_cpuct value 300 -< info string Updated option centi_cpuct to 300 -> setoption name centi_dirichlet_alpha value 20 -< info string Updated option centi_dirichlet_alpha to 20 -> setoption name centi_dirichlet_epsilon value 10 -< info string Updated option centi_dirichlet_epsilon to 10 -> setoption name centi_q_value_weight value 70 -< info string Updated option centi_q_value_weight to 70 -> setoption name centi_temperature value 0 -< info string Updated option centi_temperature to 0 -> setoption name check_mate_in_one value false -< info string Updated option check_mate_in_one to false -> setoption name context value gpu -< info string Updated option context to gpu -> setoption name enable_timeout value false -< info string Updated option enable_timeout to false -> setoption name extend_time_on_bad_position value true -< info string Updated option extend_time_on_bad_position to true -> setoption name max_move_num_to_reduce_movetime value 0 -< info string Updated option max_move_num_to_reduce_movetime to 0 -> setoption name max_search_depth value 40 -< info string Updated option max_search_depth to 40 -> setoption name move_overhead_ms value 300 -< info string Updated option move_overhead_ms to 300 -> setoption name moves_left value 40 -< info string Updated option moves_left to 40 -> setoption name playouts_empty_pockets value 8192 -< info string Updated option playouts_empty_pockets to 8192 -> setoption name playouts_filled_pockets value 8192 -< info string Updated option playouts_filled_pockets to 8192 -> setoption name threads value 16 -< info string Updated option threads to 16 -> setoption name threshold_time_for_raw_net_ms value 100 -< info string Updated option threshold_time_for_raw_net_ms to 100 -> setoption name use_raw_network value false -< info string Updated option use_raw_network to false -> setoption name verbose value true -< info string Updated option verbose to true -> setoption name virtual_loss value 3 -< info string Updated option virtual_loss to 3 -> isready -< readyok -> setoption name UCI_Variant value crazyhouse -< info string Updated option UCI_Variant to crazyhouse -> ucinewgame -> position startpos -> isready -< readyok -> go wtime 300000 btime 300000 movestogo 40 -< info string Time for this move is 7200ms -< bestmove e2e4 -> position startpos moves e2e4 e7e5 -> isready -< readyok -> go wtime 292680 btime 288064 movestogo 39 -< info string Time for this move is 7204ms -< bestmove g1f3 -> position startpos moves e2e4 e7e5 g1f3 b8c6 -> isready -< readyok -> go wtime 285363 btime 286437 movestogo 38 -< info string Time for this move is 7209ms -< bestmove f1c4 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 -> isready -< readyok -> go wtime 278047 btime 262495 movestogo 37 -< info string Time for this move is 7214ms -< bestmove e1g1 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 -> isready -< readyok -> go wtime 270797 btime 245488 movestogo 36 -< info string Time for this move is 7222ms -< bestmove b1c3 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 -> isready -< readyok -> go wtime 263465 btime 230545 movestogo 35 -< info string Time for this move is 7227ms -< bestmove d2d3 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 -> isready -< readyok -> go wtime 256190 btime 228919 movestogo 34 -< info string Time for this move is 7235ms -< bestmove c1g5 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 -> isready -< readyok -> go wtime 248913 btime 191137 movestogo 33 -< info string Time for this move is 7242ms -< bestmove g5f6 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 -> isready -< readyok -> go wtime 241565 btime 189984 movestogo 32 -< info string Time for this move is 7248ms -< bestmove c3d5 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 -> isready -< readyok -> go wtime 234288 btime 183547 movestogo 31 -< info string Time for this move is 7257ms -< bestmove c2c3 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 c2c3 c8g4 -> isready -< readyok -> go wtime 226947 btime 150603 movestogo 30 -< info string Time for this move is 7264ms -< bestmove h2h3 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 c2c3 c8g4 h2h3 g4e6 -> isready -< readyok -> go wtime 219609 btime 143635 movestogo 29 -< info string Time for this move is 7272ms -< bestmove b2b4 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 c2c3 c8g4 h2h3 g4e6 b2b4 e6d5 -> isready -< readyok -> go wtime 212223 btime 142247 movestogo 28 -< info string Time for this move is 7279ms -< bestmove c4d5 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 c2c3 c8g4 h2h3 g4e6 b2b4 e6d5 c4d5 c5f2 -> isready -< readyok -> go wtime 204863 btime 138200 movestogo 27 -< info string Time for this move is 7287ms -< bestmove f1f2 -> position startpos moves e2e4 e7e5 g1f3 b8c6 f1c4 f8c5 e1g1 g8f6 b1c3 e8g8 d2d3 d7d6 c1g5 h7h6 g5f6 d8f6 c3d5 f6d8 c2c3 c8g4 h2h3 g4e6 b2b4 e6d5 c4d5 c5f2 f1f2 P@e3 -> isready -< readyok -> go wtime 197514 btime 130432 movestogo 26 -< info string Time for this move is 7296ms -< Traceback (most recent call last): - File "./crazyara.py", line 439, in main - perform_action(cmd_list) - File "./crazyara.py", line 246, in perform_action - value, selected_move, confidence, _ = mcts_agent.perform_action(gamestate) - File "/media/queensgambit/Volume/Deep_Learning/projects/CrazyAra/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py", line 461, in perform_action - value, selected_move, confidence, selected_child_idx = super().perform_action(state) - File "/media/queensgambit/Volume/Deep_Learning/projects/CrazyAra/DeepCrazyhouse/src/domain/agent/player/_Agent.py", line 32, in perform_action - value, legal_moves, self.p_vec_small = self.evaluate_board_state(state) - File "/media/queensgambit/Volume/Deep_Learning/projects/CrazyAra/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py", line 256, in evaluate_board_state - p_vec_small, p_vec_small.shape, state_in)) -Exception: Legal move list [Move.from_uci('d5f7'), Move.from_uci('d5e6'), Move.from_uci('d5c6'), Move.from_uci('d5c4'), Move.from_uci('d5b3'), Move.from_uci('f3g5'), Move.from_uci('f3e5'), Move.from_uci('f3h4'), Move.from_uci('f3d4'), Move.from_uci('f3h2'), Move.from_uci('f3d2'), Move.from_uci('f3e1'), Move.from_uci('f2e2'), Move.from_uci('f2d2'), Move.from_uci('f2c2'), Move.from_uci('f2b2'), Move.from_uci('f2f1'), Move.from_uci('g1h2'), Move.from_uci('g1h1'), Move.from_uci('g1f1'), Move.from_uci('d1a4'), Move.from_uci('d1b3'), Move.from_uci('d1e2'), Move.from_uci('d1d2'), Move.from_uci('d1c2'), Move.from_uci('d1f1'), Move.from_uci('d1e1'), Move.from_uci('d1c1'), Move.from_uci('d1b1'), Move.from_uci('a1c1'), Move.from_uci('a1b1'), Move.from_uci('b4b5'), Move.from_uci('h3h4'), Move.from_uci('d3d4'), Move.from_uci('c3c4'), Move.from_uci('g2g3'), Move.from_uci('a2a3'), Move.from_uci('g2g4'), Move.from_uci('a2a4'), Move.from_uci('N@b1'), Move.from_uci('B@b1'), Move.from_uci('N@c1'), Move.from_uci('B@c1'), Move.from_uci('N@e1'), Move.from_uci('B@e1'), Move.from_uci('N@f1'), Move.from_uci('B@f1'), Move.from_uci('N@h1'), Move.from_uci('B@h1'), Move.from_uci('N@b2'), Move.from_uci('B@b2'), Move.from_uci('N@c2'), Move.from_uci('B@c2'), Move.from_uci('N@d2'), Move.from_uci('B@d2'), Move.from_uci('N@e2'), Move.from_uci('B@e2'), Move.from_uci('N@h2'), Move.from_uci('B@h2'), Move.from_uci('N@a3'), Move.from_uci('B@a3'), Move.from_uci('N@b3'), Move.from_uci('B@b3'), Move.from_uci('N@g3'), Move.from_uci('B@g3'), Move.from_uci('N@a4'), Move.from_uci('B@a4'), Move.from_uci('N@c4'), Move.from_uci('B@c4'), Move.from_uci('N@d4'), Move.from_uci('B@d4'), Move.from_uci('N@f4'), Move.from_uci('B@f4'), Move.from_uci('N@g4'), Move.from_uci('B@g4'), Move.from_uci('N@h4'), Move.from_uci('B@h4'), Move.from_uci('N@a5'), Move.from_uci('B@a5'), Move.from_uci('N@b5'), Move.from_uci('B@b5'), Move.from_uci('N@c5'), Move.from_uci('B@c5'), Move.from_uci('N@f5'), Move.from_uci('B@f5'), Move.from_uci('N@g5'), Move.from_uci('B@g5'), Move.from_uci('N@h5'), Move.from_uci('B@h5'), Move.from_uci('N@a6'), Move.from_uci('B@a6'), Move.from_uci('N@b6'), Move.from_uci('B@b6'), Move.from_uci('N@e6'), Move.from_uci('B@e6'), Move.from_uci('N@f6'), Move.from_uci('B@f6'), Move.from_uci('N@g6'), Move.from_uci('B@g6'), Move.from_uci('N@d7'), Move.from_uci('B@d7'), Move.from_uci('N@e7'), Move.from_uci('B@e7'), Move.from_uci('N@h7'), Move.from_uci('B@h7'), Move.from_uci('N@b8'), Move.from_uci('B@b8'), Move.from_uci('N@c8'), Move.from_uci('B@c8'), Move.from_uci('N@e8'), Move.from_uci('B@e8'), Move.from_uci('N@h8'), Move.from_uci('B@h8')] with length 113 is uncompatible to policy vector [0.01185716 0.00592858 0.00790477 0.00691667 0.00691667 0.00592858 - 0.00592858 0.00592858 0.00592858 0.00592858 0.00592858 0.00592858 - 0.01284525 0.00988096 0.01383335 0.01185716 0.01284525 0.00790477 - 0.00790477 0.00691667 0.00988096 0.11762996 0.01580954 0.00988096 - 0.00988096 0.01086906 0.00790477 0.00691667 0.00691667 0.00691667 - 0.00691667 0.00691667 0.00691667 0.00592858 0.00790477 0.00691667 - 0.00691667 0.00691667 0.00592858 0.00592858 0.00691667 0.00889287 - 0.00691667 0.01185716 0.00790477 0.00691667 0.00790477 0.00691667 - 0.00691667 0.00691667 0.00691667 0.00691667 0.00691667 0.00988096 - 0.00691667 0.00691667 0.00691667 0.00889287 0.00592858 0.00691667 - 0.00691667 0.01086906 0.00790477 0.01086906 0.00691667 0.00691667 - 0.00790477 0.00988096 0.00592858 0.00592858 0.00592858 0.00592858 - 0.00988096 0.00790477 0.00691667 0.01482145 0.00691667 0.00691667 - 0.00592858 0.00691667 0.00691667 0.00691667 0.01086906 0.00988096 - 0.00790477 0.00691667 0.00889287 0.01086906 0.00691667 0.00691667 - 0.00691667 0.00790477 0.00790477 0.01185716 0.00592858 0.00691667 - 0.01086906 0.01482145 0.00691667 0.00691667 0.00691667 0.00691667 - 0.00691667 0.00790477 0.00691667 0.00790477 0.00691667 0.00691667 - 0.00691667 0.00790477 0.00691667 0.00691667] with shape (112,) for board state r2q1rk1/ppp2pp1/2np3p/3Bp3/1P2P3/2PPpN1P/P4RP1/R2Q2K1[BBNbn] w - - 1 15 - diff --git a/DeepCrazyhouse/src/domain/abstract_cls/_GameState.py b/DeepCrazyhouse/src/domain/abstract_cls/_GameState.py index 4f8904ab..2bd40b64 100644 --- a/DeepCrazyhouse/src/domain/abstract_cls/_GameState.py +++ b/DeepCrazyhouse/src/domain/abstract_cls/_GameState.py @@ -17,7 +17,7 @@ def __init__(self, board): self.board = board self._fen_dic = {} - def apply_move(self, move: chess.Move, remember_state=False): + def apply_move(self, move: chess.Move): #, remember_state=False): self.board.push(move) def get_state_planes(self): @@ -52,5 +52,19 @@ def get_board_fen(self): return self.board.fen() #return self.board.fen().rsplit(' ', 1)[0] + def get_transposition_key(self): + """ + Returns an identifier key for the current board state excluding move counters. + Calling ._transposition_key() is faster than .fen() + :return: + """ + return self.board._transposition_key() + def new_game(self): raise NotImplementedError + + def get_halfmove_counter(self): + return self.board.halfmove_clock + + def get_fullmove_number(self): + return self.board.fullmove_number diff --git a/DeepCrazyhouse/src/domain/agent/NeuralNetAPI.py b/DeepCrazyhouse/src/domain/agent/NeuralNetAPI.py index c47112aa..7c08539e 100644 --- a/DeepCrazyhouse/src/domain/agent/NeuralNetAPI.py +++ b/DeepCrazyhouse/src/domain/agent/NeuralNetAPI.py @@ -1,6 +1,5 @@ import logging import numpy as np -import DeepCrazyhouse.src.runtime.Colorer import time import json import glob @@ -79,6 +78,13 @@ def __init__(self, ctx='cpu', batch_size=1): grad_req='null', force_rebind=True) self.executor.copy_params_from(arg_params, aux_params) + #self.executors = [] + #for i in range(batch_size): + # executor = sym.simple_bind(ctx=self.ctx, data=(1, NB_CHANNELS_FULL, BOARD_HEIGHT, BOARD_WIDTH), + # grad_req='null', force_rebind=True) + # executor.copy_params_from(arg_params, aux_params) + # self.executors.append(executor) + def get_executor(self): """ Returns the executor object used for inference diff --git a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py index 2a7600b8..6bee3a2d 100644 --- a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py +++ b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py @@ -4,14 +4,23 @@ @project: crazy_ara_refactor @author: queensgambit -Please describe what the content of this file is about +The MCTSAgent runs playouts/simulations in the search tree and updates the node statistics. +The final move is chosen according to the visit count of each direct child node. +One playout is defined as expanding one new node in the tree. In the case of chess this means evaluating a new board position. + +If the evaluation for one move takes too long on your hardware you can decrease the value for: + nb_playouts_empty_pockets and nb_playouts_filled_pockets. + +For more details and the mathematical equations please take a look at src/domain/agent/README.md as well as the +official DeepMind-papers. """ import numpy as np + from DeepCrazyhouse.src.domain.crazyhouse.output_representation import get_probs_of_move_list, value_to_centipawn from DeepCrazyhouse.src.domain.agent.NeuralNetAPI import NeuralNetAPI from copy import deepcopy -from multiprocessing import Barrier, Pipe +from multiprocessing import Pipe import logging from DeepCrazyhouse.src.domain.agent.player.util.NetPredService import NetPredService from DeepCrazyhouse.src.domain.agent.player.util.Node import Node @@ -19,9 +28,11 @@ from time import time from DeepCrazyhouse.src.domain.agent.player._Agent import _Agent from DeepCrazyhouse.src.domain.crazyhouse.GameState import GameState +import collections +from DeepCrazyhouse.src.domain.crazyhouse.constants import NB_CHANNELS_FULL, BOARD_WIDTH, BOARD_HEIGHT, NB_LABELS import cProfile, pstats, io -from numba import jit +DTYPE = np.float def profile(fnc): @@ -48,22 +59,13 @@ def inner(*args, **kwargs): class MCTSAgent(_Agent): - def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_pockets=256, + def __init__(self, nets: [NeuralNetAPI], threads=16, batch_size=8, playouts_empty_pockets=256, playouts_filled_pockets=512, cpuct=1, dirichlet_epsilon=.25, - dirichlet_alpha=0.2, max_search_depth=15, temperature=0., clip_quantil=0., + dirichlet_alpha=0.2, max_search_depth=15, temperature=0., temperature_moves=4, q_value_weight=0., virtual_loss=3, verbose=True, min_movetime=100, check_mate_in_one=False, - enable_timeout=False): + use_pruning=True, use_oscillating_cpuct=True): """ Constructor of the MCTSAgent. - The MCTSAgent runs playouts/simulations in the search tree and updates the node statistics. - The final move is chosen according to the visit count of each direct child node. - One playout is defined as expanding one new node in the tree. In the case of chess this means evaluating a new board position. - - If the evaluation for one move takes too long on your hardware you can decrease the value for: - nb_playouts_empty_pockets and nb_playouts_filled_pockets. - - For more details and the mathematical equations please take a look at src/domain/agent/README.md as well as the - official DeepMind-papers. :param net: NeuralNetAPI handle which is used to communicate with the neural network :param threads: Number of threads to evaluate the nodes in parallel @@ -90,6 +92,8 @@ def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_p If 0. -> Deterministic policy. The move is chosen with the highest probability If 1. -> Pure random sampling policy. The move is sampled from the posterior without any scaling being applied. + :param temperature_moves: Number of fullmoves in which the temperature parameter will be applied. + Otherwise the temperature will be set to 0 for deterministic play. :param clip_quantil: A quantil clipping parameter with range [0., 1.]. All cummulated low percentages for moves are set to 0. This makes sure that very unlikely moves (blunders) are clipped after the exponential scaling. @@ -107,7 +111,7 @@ def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_p NetPredService. """ - super().__init__(temperature, clip_quantil, verbose) + super().__init__(temperature, temperature_moves, verbose) # the root node contains all references to its child nodes self.root_node = None @@ -119,10 +123,15 @@ def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_p self.node_lookup = {} # get the network reference - self.net = net + self.nets = nets self.virtual_loss = virtual_loss self.cpuct_init = cpuct + + if cpuct < 0.01 or cpuct > 10: + raise Exception('You might have confused centi-cpuct with cpuct.' + 'The requested cpuct is beyond reasonable range: cpuct should be around > 0.01 and < 10.') + self.cpuct = cpuct self.max_search_depth = max_search_depth self.threads = threads @@ -147,8 +156,6 @@ def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_p self.my_pipe_endings.append(ending1) pip_endings_external.append(ending2) - self.net_pred_service = NetPredService(pip_endings_external, self.net, batch_size, enable_timeout) - self.nb_playouts_empty_pockets = playouts_empty_pockets self.nb_playouts_filled_pockets = playouts_filled_pockets @@ -166,7 +173,29 @@ def __init__(self, net: NeuralNetAPI, threads=16, batch_size=8, playouts_empty_p # number of nodes before the evaluate_board_state() call are stored here to measure the nps correctly self.total_nodes_pre_search = None - def evaluate_board_state(self, state_in: GameState): + # allocate shared memory for communicating with the network prediction service + self.batch_state_planes = np.zeros((self.threads, NB_CHANNELS_FULL, BOARD_HEIGHT, BOARD_WIDTH), DTYPE) + self.batch_value_results = np.zeros(self.threads, DTYPE) + self.batch_policy_results = np.zeros((self.threads, NB_LABELS), DTYPE) + + # initialize the NetworkPredictionService and give the pointers to the shared memory + self.net_pred_services = [] + nb_pipes = self.threads // len(nets) + + # create multiple gpu-access points + for i, net in enumerate(nets): + net_pred_service = NetPredService(pip_endings_external[i*nb_pipes:(i+1)*nb_pipes], net, batch_size, self.batch_state_planes, + self.batch_value_results, self.batch_policy_results) + self.net_pred_services.append(net_pred_service) + + self.transposition_table = collections.Counter() + self.send_batches = False + self.root_node_prior_policy = None + + self.use_pruning = use_pruning + self.use_oscillating_cpuct = use_oscillating_cpuct + + def evaluate_board_state(self, state: GameState): """ Analyzes the current board state. This is the main method which get called by the uci interface or analysis request. @@ -178,30 +207,32 @@ def evaluate_board_state(self, state_in: GameState): # store the time at which the search started self.t_start_eval = time() - # create a deepcopy of the state in order not to change the given input parameter - state = deepcopy(state_in) - # check if the net prediction service has already been started - if self.net_pred_service.running is False: + if self.net_pred_services[0].running is False: # start the prediction daemon thread - self.net_pred_service.start() + for net_pred_service in self.net_pred_services: + net_pred_service.start() # receive a list of all possible legal move in the current board position - legal_moves = list(state.get_legal_moves()) + legal_moves = state.get_legal_moves() # consistency check if len(legal_moves) == 0: raise Exception('The given board state has no legal move available') # check first if the the current tree can be reused - board_fen = state.get_board_fen() - if board_fen in self.node_lookup: - self.root_node = self.node_lookup[board_fen] + key = (state.get_transposition_key(), state.get_halfmove_counter) + + if self.use_pruning is False and key in self.node_lookup: + self.root_node = self.node_lookup[key] logging.debug('Reuse the search tree. Number of nodes in search tree: %d', self.root_node.nb_total_expanded_child_nodes) self.total_nodes_pre_search = deepcopy(self.root_node.n_sum) + + # reset potential good nodes for the root + self.root_node.q[self.root_node.q < 1.1] = 0 + else: - logging.debug("The given board position wasn't found in the search tree.") logging.debug("Starting a brand new search tree...") self.root_node = None self.total_nodes_pre_search = 0 @@ -230,11 +261,12 @@ def evaluate_board_state(self, state_in: GameState): # receive the policy vector based on the MCTS search p_vec_small = self.root_node.get_mcts_policy(self.q_value_weight) - # store the current root in the lookup table - self.node_lookup[state.get_board_fen()] = self.root_node + if self.use_pruning is False: + # store the current root in the lookup table + self.node_lookup[key] = self.root_node # select the q-value according to the mcts best child value - best_child_idx = self.root_node.get_mcts_policy(self.q_value_weight).argmax() + best_child_idx = p_vec_small.argmax() value = self.root_node.q[best_child_idx] lst_best_moves, _ = self.get_calculated_line() @@ -243,21 +275,25 @@ def evaluate_board_state(self, state_in: GameState): # show the best calculated line node_searched = int(self.root_node.n_sum - self.total_nodes_pre_search) # In uci the depth is given using half-moves notation also called plies - logging.debug('Update info') time_e = time() - self.t_start_eval - print('info score cp %d depth %d nodes %d time %d nps %d pv%s' % ( value_to_centipawn(value), - max_depth_reached, - node_searched, time_e*1000, - node_searched/max(1,time_e), str_moves)) if len(legal_moves) != len(p_vec_small): - raise Exception('Legal move list %s with length %s is uncompatible to policy vector %s' - ' with shape %s for board state %s' % (legal_moves, len(legal_moves), - p_vec_small, p_vec_small.shape, state_in)) - - return value, legal_moves, p_vec_small + raise Exception('Legal move list %s with length %s is uncompatible to policy vector %s' + ' with shape %s for board state %s and nodes legal move list: %s' % + (legal_moves, len(legal_moves), + p_vec_small, p_vec_small.shape, state, + self.root_node.legal_moves)) + + # define the remaining return variables + cp = value_to_centipawn(value) + depth = max_depth_reached + nodes = node_searched + time_elapsed_s = time_e*1000 + nps = node_searched/time_e + pv = str_moves + + return value, legal_moves, p_vec_small, cp, depth, nodes, time_elapsed_s, nps, pv - @jit def _expand_root_node_multiple_moves(self, state, legal_moves): """ Checks if the current root node can be found in the look-up table. @@ -273,7 +309,7 @@ def _expand_root_node_multiple_moves(self, state, legal_moves): # start a brand new tree state_planes = state.get_state_planes() - [value, policy_vec] = self.net.predict_single(state_planes) + [value, policy_vec] = self.nets[0].predict_single(state_planes) # extract a sparse policy vector with normalized probabilities p_vec_small = get_probs_of_move_list(policy_vec, legal_moves, state.is_white_to_move()) @@ -284,9 +320,8 @@ def _expand_root_node_multiple_moves(self, state, legal_moves): str_legal_moves = '' # create a new root node - self.root_node = Node(value, p_vec_small, legal_moves, str_legal_moves, is_leaf) + self.root_node = Node(value, p_vec_small, legal_moves, str_legal_moves, is_leaf, clip_low_visit=False) - @jit def _expand_root_node_single_move(self, state, legal_moves): """ Expands the current root in the case if there's only a single move available. @@ -322,18 +357,19 @@ def _expand_root_node_single_move(self, state, legal_moves): p_vec_small_child = None # check if you can claim a draw - its assumed that the draw is always claimed - elif state.is_draw() is True: + elif self.can_claim_threefold_repetition(state.get_transposition_key(), [0]) or\ + state.get_pythonchess_board().can_claim_fifty_moves() is True: value = 0 is_leaf = True legal_moves_child = [] p_vec_small_child = None else: - legal_moves_child = list(state_child.get_legal_moves()) + legal_moves_child = state_child.get_legal_moves() # start a brand new prediction for the child state_planes = state_child.get_state_planes() - [value, policy_vec] = self.net.predict_single(state_planes) + [value, policy_vec] = self.nets[0].predict_single(state_planes) # extract a sparse policy vector with normalized probabilities p_vec_small_child = get_probs_of_move_list(policy_vec, legal_moves_child, @@ -357,11 +393,21 @@ def _run_mcts_search(self, state): # clear the look up table self.node_lookup = {} + # safe the prior policy of the root node + self.root_node_prior_policy = deepcopy(self.root_node.p) + # apply dirichlet noise to the prior probabilities in order to ensure # that every move can possibly be visited self.root_node.apply_dirichlet_noise_to_prior_policy(epsilon=self.dirichlet_epsilon, alpha=self.dirichlet_alpha) + # iterate through all children and add dirichlet if there exists any + for child_node in self.root_node.child_nodes: + if child_node is not None: + # test of adding dirichlet noise to a new node + child_node.apply_dirichlet_noise_to_prior_policy(epsilon=self.dirichlet_epsilon * .1, + alpha=self.dirichlet_alpha) + # store what depth has been reached at maximum in the current search tree # default is 1, in case only 1 move is available max_depth_reached = 1 @@ -373,6 +419,7 @@ def _run_mcts_search(self, state): nb_playouts = self.nb_playouts_empty_pockets else: nb_playouts = self.nb_playouts_filled_pockets + self.temperature_current = 0 t_elapsed = 0 cur_playouts = 0 @@ -380,61 +427,48 @@ def _run_mcts_search(self, state): cpuct_init = self.cpuct + decline = True + while max_depth_reached < self.max_search_depth and \ cur_playouts < nb_playouts and \ t_elapsed * 1000 < self.movetime_ms: # and np.abs(self.root_node.q.mean()) < 0.99: - # Test about decreasing CPUCT value - self.cpuct -= 0.005 #2 #5 #1 #np.random.randint(1,5) #0.005 - if self.cpuct < 1.3: # 5: - self.cpuct = 1.3 # 5 + if self.use_oscillating_cpuct is True: + # Test about decreasing CPUCT value + if decline is True: + self.cpuct -= 0.01 + else: + self.cpuct += 0.01 + if self.cpuct < cpuct_init * .5: + decline = False + elif self.cpuct > cpuct_init: + decline = True # start searching with ThreadPoolExecutor(max_workers=self.threads) as executor: for i in range(self.threads): # calculate the thread id based on the current playout futures.append(executor.submit(self._run_single_playout, state=state, - parent_node=self.root_node, pipe_id=i, depth=1, mv_list=[])) + parent_node=self.root_node, pipe_id=i, depth=1, chosen_nodes=[])) cur_playouts += self.threads time_show_info = time() - old_time - # store the mean of all value predictions in this variable - # mean_value = 0 - for i, f in enumerate(futures): - cur_value, cur_depth, mv_list = f.result() - - # sum up all values - # mean_value += cur_value + cur_value, cur_depth, chosen_nodes = f.result() if cur_depth > max_depth_reached: max_depth_reached = cur_depth # Print the explored line of the last line for every x seconds if verbose is true if self.verbose and time_show_info > 0.5 and i == len(futures) - 1: + mv_list = self._create_mv_list(chosen_nodes) str_moves = self._mv_list_to_str(mv_list) - #logging.debug('Update: %d' % cur_depth) print('info score cp %d depth %d nodes %d pv%s' % ( value_to_centipawn(cur_value), cur_depth, self.root_node.n_sum, str_moves)) - #print('info cpuct: %.2f' % self.cpuct) + logging.debug('Update info') old_time = time() - """ - # Show only current best line - # Print every second if verbose is true - if self.verbose and time_show_info > 1: - # select the q-value according to the mcts best child value - best_child_idx = self.root_node.get_mcts_policy(self.q_value_weight).argmax() - cur_value = self.root_node.q[best_child_idx] - - lst_best_moves, _ = self.get_calculated_line() - str_moves = self._mv_list_to_str(lst_best_moves) - print('info score cp %d depth %d nodes %d pv%s' % ( - value_to_centipawn(cur_value), len(lst_best_moves), self.root_node.n_sum, str_moves)) - old_time = time() - """ - # update the current search time t_elapsed = time() - self.t_start_eval if self.verbose and time_show_info > 1: @@ -445,7 +479,7 @@ def _run_mcts_search(self, state): return max_depth_reached - def perform_action(self, state: GameState, verbose=True): + def perform_action(self, state_in: GameState, verbose=True): """ Return a value, best move with according to the mcts search. This method is used when using the mcts agent as a player. @@ -458,50 +492,28 @@ def perform_action(self, state: GameState, verbose=True): selected_child_idx - Child index which correspond to the selected child """ - value, selected_move, confidence, selected_child_idx = super().perform_action(state) - - # apply the selected mve on the current board state in order to create a lookup table for future board states - state.apply_move(selected_move) - - # select the q value for the child which leads to the best calculated line - value = self.root_node.q[selected_child_idx] - - # select the next node - node = self.root_node.child_nodes[selected_child_idx] - - # store the reference links for all possible child future child to the node lookup table - for idx, mv in enumerate(state.get_legal_moves()): - state_future = deepcopy(state) - state_future.apply_move(mv) - - # store the current child node with it's board fen as the hash-key if the child node has already been expanded - if node is not None and idx < node.nb_direct_child_nodes and node.child_nodes[idx] is not None: - self.node_lookup[state_future.get_board_fen()] = node.child_nodes[idx] + # create a deepcopy of the state in order not to change the given input parameter + state = deepcopy(state_in) - return value, selected_move, confidence, selected_child_idx + return super().perform_action(state) - #@profile - def _run_single_playout(self, state: GameState, parent_node: Node, pipe_id=0, depth=1, mv_list=[]): + def _run_single_playout(self, state: GameState, parent_node: Node, pipe_id=0, depth=1, chosen_nodes=[]): """ This function works recursively until a leaf or terminal node is reached. It ends by backpropagating the value of the new expanded node or by propagating the value of a terminal state. - :param state: Current game-state for the evaluation. This state differs between the treads + :param state_: Current game-state for the evaluation. This state differs between the treads :param parent_node: Current parent-node of the selected node. In the first expansion this is the root node. :param depth: Current depth for the evaluation. Depth is increased by 1 for every recusive call - :param mv_list: List of moves which have been taken in the current path. For each selected child node this list + :param chosen_nodes: List of moves which have been taken in the current path. For each selected child node this list is expanded by one move recursively. + :param chosen_nodes: List of all nodes that this thread has explored with respect to the root node :return: -value: The inverse value prediction of the current board state. The flipping by -1 each turn is needed because the point of view changes each half-move depth: Current depth reach by this evaluation mv_list: List of moves which have been selected """ - # create a deepcopy of the state for all future recursive calls if it's the first of function - # call of _run_single_playout() - if depth == 1: - state = deepcopy(state) - # select a legal move on the chess board node, move, child_idx = self._select_node(parent_node) @@ -513,124 +525,153 @@ def _run_single_playout(self, state: GameState, parent_node: Node, pipe_id=0, de # the effect of virtual loss will be undone if the playout is over parent_node.apply_virtual_loss_to_child(child_idx, self.virtual_loss) + if depth == 1: + state = GameState(deepcopy(state.get_pythonchess_board())) + # apply the selected move on the board state.apply_move(move) # append the selected move to the move list - mv_list.append(move) + # append the chosen child idx to the chosen_nodes list + chosen_nodes.append(child_idx) if node is None: - # get the board-fen which is used as an identifier for the board positions in the look-up table - board_fen = state.get_board_fen() + # get the transposition-key which is used as an identifier for the board positions in the look-up table + transposition_key = state.get_transposition_key() # check if the addressed fen exist in the look-up table - if board_fen in self.node_lookup: - # get the node from the look-up list - node = self.node_lookup[board_fen] - - with parent_node.lock: - # setup a new connection from the parent to the child - parent_node.child_nodes[child_idx] = node - - # get the prior value from the leaf node which has already been expanded - value = node.v - # receive a free available pipe - my_pipe = self.my_pipe_endings[pipe_id] - my_pipe.send(state.get_state_planes()) - #this pipe waits for the predictions of the network inference service - [_, _] = my_pipe.recv() - - # get the value from the leaf node (the current function is called recursively) - #value, depth, mv_list = self._run_single_playout(state, node, pipe_id, depth+1, mv_list) + # note: It's important to use also the halfmove-counter here, otherwise the system can create an infinite + # feed-back-loop + key = (transposition_key, state.get_halfmove_counter()) - else: - # expand and evaluate the new board state (the node wasn't found in the look-up table) - # its value will be backpropagated through the tree and flipped after every layer + # expand and evaluate the new board state (the node wasn't found in the look-up table) + # its value will be backpropagated through the tree and flipped after every layer + # receive a free available pipe + my_pipe = self.my_pipe_endings[pipe_id] - # receive a free available pipe - my_pipe = self.my_pipe_endings[pipe_id] #.pop() - #logging.debug('thread %d request' % pipe_id) + if self.send_batches is True: my_pipe.send(state.get_state_planes()) # this pipe waits for the predictions of the network inference service [value, policy_vec] = my_pipe.recv() + else: + state_planes = state.get_state_planes() + self.batch_state_planes[pipe_id] = state_planes + + my_pipe.send(pipe_id) + + result_channel = my_pipe.recv() - # initialize is_leaf by default to false - is_leaf = False + value = np.array(self.batch_value_results[result_channel]) + policy_vec = np.array(self.batch_policy_results[result_channel]) + + # initialize is_leaf by default to false + is_leaf = False + + + # check if the current player has won the game + # (we don't need to check for is_lost() because the game is already over + # if the current player checkmated his opponent) + is_won = False + is_check = False - # check if the current player has won the game - # (we don't need to check for is_lost() because the game is already over - # if the current player checkmated his opponent) + if state.is_check() is True: + is_check = True if state.is_won() is True: + is_won = True + + if is_won is True: value = -1 is_leaf = True legal_moves = [] p_vec_small = None + # establish a mate in one connection in order to stop exploring different alternatives + parent_node.mate_child_idx = child_idx - # check if you can claim a draw - its assumed that the draw is always claimed - elif False: #state.is_draw() is True: TODO: Create more performant implementation - value = 0 - is_leaf = True - legal_moves = [] - p_vec_small = None - else: - # get the current legal move of its board state - legal_moves = list(state.get_legal_moves()) - if len(legal_moves) < 1: - raise Exception('No legal move is available for state: %s' % state) - - # extract a sparse policy vector with normalized probabilities - try: - p_vec_small = get_probs_of_move_list(policy_vec, legal_moves, - is_white_to_move=state.is_white_to_move(), normalize=True) - - except KeyError: - raise Exception('Key Error for state: %s' % state) - - # convert all legal moves to a string if the option check_mate_in_one was enabled - if self.check_mate_in_one is True: - str_legal_moves = str(state.get_legal_moves()) - else: - str_legal_moves = '' + # get the value from the leaf node (the current function is called recursively) + # check if you can claim a draw - its assumed that the draw is always claimed + elif self.can_claim_threefold_repetition(transposition_key, chosen_nodes) or \ + state.get_pythonchess_board().can_claim_fifty_moves() is True: + value = 0 + is_leaf = True + legal_moves = [] + p_vec_small = None + else: + # get the current legal move of its board state + legal_moves = state.get_legal_moves() - # create a new node - new_node = Node(value, p_vec_small, legal_moves, str_legal_moves, is_leaf) + if len(legal_moves) < 1: + raise Exception('No legal move is available for state: %s' % state) - #if is_leaf is False: - #if depth == 2 and pipe_id == 0: - # # test of adding dirichlet noise to a new node - # new_node.apply_dirichlet_noise_to_prior_policy(epsilon=self.dirichlet_epsilon/3, alpha=self.dirichlet_alpha) + # extract a sparse policy vector with normalized probabilities + try: + p_vec_small = get_probs_of_move_list(policy_vec, legal_moves, + is_white_to_move=state.is_white_to_move(), normalize=True) - # include a reference to the new node in the look-up table - self.node_lookup[board_fen] = new_node + except KeyError: + raise Exception('Key Error for state: %s' % state) + + # convert all legal moves to a string if the option check_mate_in_one was enabled + if self.check_mate_in_one is True: + str_legal_moves = str(state.get_legal_moves()) + else: + str_legal_moves = '' + + # clip the visit nodes for all nodes in the search tree except the director opp. move + clip_low_visit = self.use_pruning and depth != 1 + + # create a new node + new_node = Node(value, p_vec_small, legal_moves, str_legal_moves, is_leaf, transposition_key, clip_low_visit) - with parent_node.lock: - # add the new node to its parent - parent_node.child_nodes[child_idx] = new_node + if depth == 1: - # check if the new node has a mate_in_one connection (if yes overwrite the network prediction) - if new_node.mate_child_idx is not None: - value = 1 + # disable uncertain moves from being visited by giving them a very bad score + if is_leaf is False: + if self.root_node_prior_policy[child_idx] < 1e-3 and value*-1 < self.root_node.v: + with parent_node.lock: + value = 99 + + if value < 0: # and state.are_pocket_empty(): #and pipe_id == 0: + # test of adding dirichlet noise to a new node + new_node.apply_dirichlet_noise_to_prior_policy(epsilon=self.dirichlet_epsilon*.02, + alpha=self.dirichlet_alpha) + + if self.use_pruning is False: + # include a reference to the new node in the look-up table + self.node_lookup[key] = new_node + + with parent_node.lock: + # add the new node to its parent + parent_node.child_nodes[child_idx] = new_node # check if we have reached a leaf node elif node.is_leaf is True: value = node.v - # receive a free available pipe - my_pipe = self.my_pipe_endings[pipe_id] #.pop() - #logging.debug('thread %d request' % pipe_id) - my_pipe.send(state.get_state_planes()) - # this pipe waits for the predictions of the network inference service - [_, _] = my_pipe.recv() else: # get the value from the leaf node (the current function is called recursively) - value, depth, mv_list = self._run_single_playout(state, node, pipe_id, depth+1, mv_list) + value, depth, chosen_nodes = self._run_single_playout(state, node, pipe_id, depth + 1, chosen_nodes) # revert the virtual loss and apply the predicted value by the network to the node parent_node.revert_virtual_loss_and_update(child_idx, self.virtual_loss, -value) # we invert the value prediction for the parent of the above node layer because the player's turn is flipped every turn - return -value, depth, mv_list + return -value, depth, chosen_nodes + + def can_claim_threefold_repetition(self, transposition_key, chosen_nodes): + + search_occurrence_counter = 0 + + node = self.root_node.child_nodes[chosen_nodes[0]] + + for node_idx in chosen_nodes[1:-1]: + if node.transposition_key == transposition_key: + search_occurrence_counter += 1 + node = node.child_nodes[node_idx] + if node is None: + break + + return self.transposition_table[transposition_key] + search_occurrence_counter >= 2 def _select_node(self, parent_node: Node): """ @@ -660,13 +701,14 @@ def _select_node(self, parent_node: Node): return node, move, child_idx - def _select_node_based_on_mcts_policy(self, parent_node: Node): + def _select_node_based_on_mcts_policy(self, parent_node: Node, clip_bad_visit_nodes=False): """ Selects the next node based on the mcts policy which is used to predict the final best move. :param parent_node: Node from which to select the next child. :return: """ + child_idx = parent_node.get_mcts_policy(self.q_value_weight).argmax() nb_visits = parent_node.n[child_idx] @@ -701,7 +743,8 @@ def get_calculated_line(self): while node is not None and node.is_leaf is False: # go deep through the tree by always selecting the best move for both players - node, move, nb_visits = self._select_node_based_on_mcts_policy(node) + clip_bad_visit_nodes = node == self.root_node # and mv_number > 1 + node, move, nb_visits = self._select_node_based_on_mcts_policy(node, clip_bad_visit_nodes=clip_bad_visit_nodes) lst_best_moves.append(move) lst_nb_visits.append(nb_visits) return lst_best_moves, lst_nb_visits @@ -712,11 +755,29 @@ def _mv_list_to_str(self, lst_moves): :param lst_moves: List chess.Moves objects :return: String representing each move in the list """ - str_moves = "" - for mv in lst_moves: + str_moves = lst_moves[0].uci() + + for mv in lst_moves[1:]: str_moves += " " + mv.uci() + return str_moves + def _create_mv_list(self, lst_chosen_nodes: [int]): + """ + Creates a movement list given the child node indices from the root node onwards. + :param lst_chosen_nodes: List of chosen nodes + :return: mv_list - List of python chess moves + """ + #str_moves = "" + mv_list = [] + node = self.root_node #.child_nodes[lst_chosen_nodes[0]] + + for child_idx in lst_chosen_nodes: + mv = node.legal_moves[child_idx] + node = node.child_nodes[child_idx] + mv_list.append(mv) + return mv_list + def update_movetime(self, time_ms_per_move): """ Update move time allocation. diff --git a/DeepCrazyhouse/src/domain/agent/player/RawNetAgent.py b/DeepCrazyhouse/src/domain/agent/player/RawNetAgent.py index 4df18b25..b5e585b0 100644 --- a/DeepCrazyhouse/src/domain/agent/player/RawNetAgent.py +++ b/DeepCrazyhouse/src/domain/agent/player/RawNetAgent.py @@ -13,32 +13,38 @@ from DeepCrazyhouse.src.domain.agent.NeuralNetAPI import NeuralNetAPI from DeepCrazyhouse.src.domain.crazyhouse.output_representation import get_probs_of_move_list, value_to_centipawn from time import time - +import sys class RawNetAgent(_Agent): - def __init__(self, net: NeuralNetAPI, temperature=0., clip_quantil=0., verbose=True): - super().__init__(temperature, clip_quantil, verbose) + def __init__(self, net: NeuralNetAPI, temperature=0., temperature_moves=4, verbose=True): + super().__init__(temperature, temperature_moves, verbose) self._net = net - def evaluate_board_state(self, state: _GameState, verbose=True): + def evaluate_board_state(self, state: _GameState): """ :param state: :return: """ + t_start_eval = time() pred_value, pred_policy = self._net.predict_single(state.get_state_planes()) legal_moves = list(state.get_legal_moves()) + p_vec_small = get_probs_of_move_list(pred_policy, legal_moves, state.is_white_to_move()) - if verbose is True: - # use the move with the highest probability as the best move for logging - instinct_move = legal_moves[p_vec_small.argmax()] + # use the move with the highest probability as the best move for logging + instinct_move = legal_moves[p_vec_small.argmax()] - # show the best calculated line - print('info score cp %d depth %d nodes %d time %d pv %s' % ( - value_to_centipawn(pred_value), 1, 1, (time() - t_start_eval) * 1000, instinct_move.uci())) + # define the remaining return variables + time_e = (time() - t_start_eval) + cp = value_to_centipawn(pred_value) + depth = 1 + nodes = 1 + time_elapsed_s = time_e * 1000 + nps = nodes/time_e + pv = instinct_move.uci() - return pred_value, legal_moves, p_vec_small + return pred_value, legal_moves, p_vec_small, cp, depth, nodes, time_elapsed_s, nps, pv diff --git a/DeepCrazyhouse/src/domain/agent/player/_Agent.py b/DeepCrazyhouse/src/domain/agent/player/_Agent.py index ff8674f4..4e894a20 100644 --- a/DeepCrazyhouse/src/domain/agent/player/_Agent.py +++ b/DeepCrazyhouse/src/domain/agent/player/_Agent.py @@ -17,10 +17,11 @@ class _Agent: The greedy agent always performs the first legal move with the highest move probability """ - def __init__(self, temperature=0., clip_quantil=0., verbose=True): + def __init__(self, temperature=0, temperature_moves=4, verbose=True): self.temperature = temperature - self.p_vec_small = None - self.clip_quantil = clip_quantil + self.temperature_current = temperature + self.temperature_moves = temperature_moves + #self.p_vec_small = None self.verbose = verbose def evaluate_board_state(self, state: _GameState): @@ -29,76 +30,47 @@ def evaluate_board_state(self, state: _GameState): def perform_action(self, state: _GameState): # the first step is to call you policy agent to evaluate the given position - value, legal_moves, self.p_vec_small = self.evaluate_board_state(state) + value, legal_moves, p_vec_small, cp, depth, nodes, time_elapsed_s, nps, pv = self.evaluate_board_state(state) - if len(legal_moves) != len(self.p_vec_small): - raise Exception('Legal move list %s is uncompatible to policy vector %s' % (legal_moves, self.p_vec_small)) + if len(legal_moves) != len(p_vec_small): + raise Exception('Legal move list %s is uncompatible to policy vector %s' % (legal_moves, p_vec_small)) + + if state.get_fullmove_number() <= self.temperature_moves: + self.temperature_current = self.temperature + else: + self.temperature_current = 0 if len(legal_moves) == 1: selected_move = legal_moves[0] confidence = 1. idx = 0 else: - if self.temperature <= 0.01: - idx = self.p_vec_small.argmax() + if self.temperature_current <= 0.01: + idx = p_vec_small.argmax() else: - self._apply_temperature_to_policy() - self._apply_quantil_clipping() - idx = np.random.choice(range(len(legal_moves)), p=self.p_vec_small) + p_vec_small = self._apply_temperature_to_policy(p_vec_small) + idx = np.random.choice(range(len(legal_moves)), p=p_vec_small) selected_move = legal_moves[idx] - confidence = self.p_vec_small[idx] + confidence = p_vec_small[idx] - return value, selected_move, confidence, idx + return value, selected_move, confidence, idx, cp, depth, nodes, time_elapsed_s, nps, pv - def _apply_quantil_clipping(self): - """ - - :param p_vec_small: - :param clip_quantil: - :return: - """ - - if self.clip_quantil > 0: - # remove the lower percentage values in order to avoid strange blunders for moves with low confidence - p_vec_small_clipped = deepcopy(self.p_vec_small) - - # get the sorted indices in ascending order - idx_order = np.argsort(self.p_vec_small) - # create a quantil tank which measures how much quantil power is left - quantil_tank = self.clip_quantil - - # iterate over the indices (ascending) and apply the quantil clipping to it - for idx in idx_order: - if quantil_tank >= p_vec_small_clipped[idx]: - # remove the prob from the quantil tank - quantil_tank -= p_vec_small_clipped[idx] - # clip the index to 0 - p_vec_small_clipped[idx] = 0 - else: - # the target prob is greate than the current quantil tank - p_vec_small_clipped[idx] -= quantil_tank - # stop the for loop - break - - # renormalize the policy - p_vec_small_clipped /= p_vec_small_clipped.sum() - - # apply the changes - self.p_vec_small = p_vec_small_clipped - - def _apply_temperature_to_policy(self): + def _apply_temperature_to_policy(self, p_vec_small): """ :return: """ # treat very small temperature value as a deterministic policy - if self.temperature <= 0.01: - p_vec_one_hot = np.zeros_like(self.p_vec_small) - p_vec_one_hot[np.argmax(self.p_vec_small)] = 1. - self.p_vec_small = p_vec_one_hot + if self.temperature_current <= 0.01: + p_vec_one_hot = np.zeros_like(p_vec_small) + p_vec_one_hot[np.argmax(p_vec_small)] = 1. + p_vec_small = p_vec_one_hot else: # apply exponential scaling - self.p_vec_small = np.power(self.p_vec_small, 1/self.temperature) + p_vec_small = p_vec_small ** (1/self.temperature_current) # renormalize the values to probabilities again - self.p_vec_small /= self.p_vec_small.sum() + p_vec_small /= p_vec_small.sum() + + return p_vec_small + diff --git a/DeepCrazyhouse/src/domain/agent/player/util/NetPredService.py b/DeepCrazyhouse/src/domain/agent/player/util/NetPredService.py index 63db223e..3535c9ea 100644 --- a/DeepCrazyhouse/src/domain/agent/player/util/NetPredService.py +++ b/DeepCrazyhouse/src/domain/agent/player/util/NetPredService.py @@ -15,17 +15,25 @@ import numpy as np from DeepCrazyhouse.src.domain.crazyhouse.output_representation import NB_LABELS, LABELS from time import time +import cython class NetPredService: - def __init__(self, pipe_endings: [connection], net: NeuralNetAPI, batch_size, enable_timeout=False): + def __init__(self, pipe_endings: [connection], net: NeuralNetAPI, batch_size, batch_state_planes: np.ndarray, + batch_value_results: np.ndarray, batch_policy_results: np.ndarray): """ :param pipe_endings: List of pip endings which are for communicating with the thread workers. :param net: Neural Network API object which provides the reference for the neural network. :param batch_size: Constant batch_size used for inference. - :param enable_timeout: Decides wether to enable a timout if a batch didn't occur under 1 second. + :param batch_state_planes: Shared numpy memory in which all threads set their state plane request for the + prediction service. Each threads has it's own channel. + :param batch_value_results: Shared numpy memory in which the value results of all threads are stored. + Each threads has it's own channel. + :param batch_policy_results: Shared numpy memory in which the policy results of all threads are stored. + Each threads has it's own channel. + #:param enable_timeout: Decides wether to enable a timout if a batch didn't occur under 1 second. """ self.net = net self.my_pipe_endings = pipe_endings @@ -34,14 +42,23 @@ def __init__(self, pipe_endings: [connection], net: NeuralNetAPI, batch_size, en self.thread_inference = Thread(target=self._provide_inference, args=(pipe_endings,), daemon=True) self.batch_size = batch_size - self.time_start = None - self.timeout_second = 1 - #self.enable_timeout = enable_timeout + self.batch_state_planes = batch_state_planes + self.batch_value_results = batch_value_results + self.batch_policy_results = batch_policy_results + + #@cython.boundscheck(False) + #@cython.wraparound(False) def _provide_inference(self, pipe_endings): print('provide inference...') - #use_random = False + #use_random = True + + #cdef double[:, :, :, ::1] batch_state_planes_view = self.batch_state_planes + #cdef double[::1] batch_value_results_view = self.batch_value_results + #cdef double[:, ::1] batch_policy_results = self.batch_policy_results + + send_batches = False #True while self.running is True: @@ -49,26 +66,41 @@ def _provide_inference(self, pipe_endings): if filled_pipes: - if True or len(filled_pipes) >= self.batch_size: + if True or len(filled_pipes) >= self.batch_size: # 1 + + if send_batches is True: + planes_batch = [] + pipes_pred_output = [] - planes_batch = [] - pipes_pred_output = [] + for pipe in filled_pipes: #[:self.batch_size]: + while pipe.poll(): + planes_batch.append(pipe.recv()) + pipes_pred_output.append(pipe) - for pipe in filled_pipes[:self.batch_size]: - while pipe.poll(): - planes_batch.append(pipe.recv()) - pipes_pred_output.append(pipe) + # logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes))) + state_planes_mxnet = mx.nd.array(planes_batch, ctx=self.net.get_ctx()) + else: + planes_ids = [] + pipes_pred_output = [] - #logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes))) - planes_batch = mx.nd.array(planes_batch, ctx=self.net.get_ctx()) + for pipe in filled_pipes: #[:self.batch_size]: + while pipe.poll(): + planes_ids.append(pipe.recv()) + pipes_pred_output.append(pipe) - #pred = self.net.get_executor().forward(is_train=False, data=planes_batch) - pred = self.net.get_net()(planes_batch) + #logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes))) + state_planes_mxnet = mx.nd.array(self.batch_state_planes[planes_ids], ctx=self.net.get_ctx()) + + #pred = self.net.get_executor().forward(is_train=False, data=state_planes_mxnet) + pred = self.net.get_net()(state_planes_mxnet) + #print('pred: %.3f' % (time()-t_s)*1000) + #t_s = time() value_preds = pred[0].asnumpy() # for the policy prediction we still have to apply the softmax activation # because it's not done by the neural net + #policy_preds = pred[1].softmax().asnumpy() policy_preds = pred[1].softmax().asnumpy() #if use_random is True: @@ -77,10 +109,20 @@ def _provide_inference(self, pipe_endings): # send the predictions back to the according workers for i, pipe in enumerate(pipes_pred_output): - pipe.send([value_preds[i], policy_preds[i]]) - # reset the timer - self.time_start = time() + if send_batches is True: + pipe.send([value_preds[i], policy_preds[i]]) + else: + # get the according channel index for setting the result + channel_idx = planes_ids[i] + + # set the value result + self.batch_value_results[channel_idx] = value_preds[i] + self.batch_policy_results[channel_idx] = policy_preds[i] + # give the thread the signal that the result has been set by sending back his channel_idx + pipe.send(channel_idx) + + #print('send back res: %.3f' % (time()-t_s)*1000) def start(self): print('start inference thread...') diff --git a/DeepCrazyhouse/src/domain/agent/player/util/Node.py b/DeepCrazyhouse/src/domain/agent/player/util/Node.py index 5b15527f..7fc55da9 100644 --- a/DeepCrazyhouse/src/domain/agent/player/util/Node.py +++ b/DeepCrazyhouse/src/domain/agent/player/util/Node.py @@ -7,17 +7,16 @@ Helper class which stores the statistics of all nodes and in the search tree. """ -from numba import jit from threading import Lock import chess import numpy as np -import logging from copy import deepcopy class Node: - def __init__(self, value, p_vec_small: np.ndarray, legal_moves: [chess.Move], str_legal_moves: str, is_leaf=False): + def __init__(self, value, p_vec_small: np.ndarray, legal_moves: [chess.Move], str_legal_moves: str, is_leaf=False, + transposition_key=None, clip_low_visit=True): # lock object for this node to protect its member variables self.lock = Lock() @@ -29,55 +28,48 @@ def __init__(self, value, p_vec_small: np.ndarray, legal_moves: [chess.Move], st self.nb_direct_child_nodes = 0 else: # specify the number of direct child nodes from this node - self.nb_direct_child_nodes = np.array(len(p_vec_small)) #, np.uint32) + self.nb_direct_child_nodes = np.array(len(p_vec_small)) # prior probability selecting each child, which is estimated by the neural network - self.p = p_vec_small #np.zeros(self.nb_direct_child_nodes, np.float32) + self.p = p_vec_small # possible legal moves from this node on which represents the edges self.legal_moves = legal_moves # stores the number of all direct children and all grand children which have already been expanded - self.nb_total_expanded_child_nodes = np.array(0) #, np.uint32) + self.nb_total_expanded_child_nodes = np.array(0) # visit count of all its child nodes - self.n = np.zeros(self.nb_direct_child_nodes) #, np.int32) + self.n = np.zeros(self.nb_direct_child_nodes) # total action value estimated by MCTS for each child node - self.w = np.zeros(self.nb_direct_child_nodes) #, np.float32) + self.w = np.zeros(self.nb_direct_child_nodes) # q: combined action value which is calculated by the averaging over all action values # u: exploration metric for each child node # (the q and u values are stacked into 1 list in order to speed-up the argmax() operation - self.q = np.zeros(self.nb_direct_child_nodes) #, np.float32) - #self.q_u = np.stack((q, u)) + self.q = np.zeros(self.nb_direct_child_nodes) - #np.concatenate((q, u)) + if is_leaf is False and clip_low_visit is True: + self.q[p_vec_small < 1e-3] = -9999 # number of total visits to this node # we initialize with 1 because if the node was created it must have been visited - self.n_sum = np.array(1) #, #np.int32) + self.n_sum = 1 # check if there's a possible mate on the board if yes create a quick link to the mate move mate_mv_idx_str = str_legal_moves.find('#') - #logging.debug('legal_moves: %s' % str(str_legal_moves)) - #logging.debug('mate_mv_idx_str: %d' % mate_mv_idx_str) if mate_mv_idx_str != -1: # -1 means that no mate move has been found # find the according index of the move in the legal_moves generator list # here we count the ',' which represent the move index mate_mv_idx = str_legal_moves[:mate_mv_idx_str].count(',') # quick reference path to a child node which leads to mate - self.mate_child_idx = mate_mv_idx #legal_moves[mate_mv_idx] - # overwrite the number of direct child nodes to 1 - #self.nb_direct_child_nodes = np.array(1) #, np.uint32) - #logging.debug('set mate in one connection') + self.mate_child_idx = mate_mv_idx else: # no direct mate move is possible so set the reference to None self.mate_child_idx = None # stores the number of all possible expandable child nodes - self.nb_expandable_child_nodes = np.array(self.nb_direct_child_nodes) #, np.uint32) - - #assert self.nb_direct_child_nodes > 0 + self.nb_expandable_child_nodes = np.array(self.nb_direct_child_nodes) # list of all child nodes which are described by each board position # the children are ordered in the same way as the legal_move generator output @@ -86,18 +78,10 @@ def __init__(self, value, p_vec_small: np.ndarray, legal_moves: [chess.Move], st # determine if the node is a leaf node this avoids checking for state.is_draw() or .state.is_won() self.is_leaf = is_leaf - ''' TODO: Delete - def update_u_for_child(self, child_idx, cpuct): - """ - Updates the u parameter via the formula given in the AlphaZero paper for a given child index - :param child_idx: Child index to update - :param cpuct: cpuct constant to apply (cpuct manages the amount of exploration) - :return: - """ - self.q_u[child_idx] = cpuct * self.p[child_idx] * (np.sqrt(self.n_sum) / (1 + self.n[child_idx])) - ''' + # store a unique identifier for the board state excluding the move counter for this node + self.transposition_key = transposition_key - def get_mcts_policy(self, q_value_weight=.65): + def get_mcts_policy(self, q_value_weight=.65, clip_low_visit_nodes=True): """ Calculates the finetuned policies based on the MCTS search. These policies should be better than the initial policy predicted by a the raw network. @@ -110,30 +94,30 @@ def get_mcts_policy(self, q_value_weight=.65): """ assert 0 <= q_value_weight <= 1. - clip_low_visit_nodes = True - if clip_low_visit_nodes is True: + if clip_low_visit_nodes is True and q_value_weight > 0: visit = deepcopy(self.n) value = deepcopy((self.q + 1)) if visit.max() > 0: - visit = self.n / self.n.sum() max_visits = visit.max() # mask out nodes that haven't been visited much - thresh_idces = visit < max_visits * 0.5 #0.33 #0.5 #.33 + thresh_idces = visit < max_visits * 0.33 # normalize to sum of 1 - value /= value.sum() value[thresh_idces] = 0 + #visit[thresh_idces] = 0 + # renormalize ot 1 + visit /= visit.sum() + value /= value.sum() policy = ((1-q_value_weight) * visit + q_value_weight * value) return policy / sum(policy) else: return visit - elif q_value_weight > 0: # disable the q values if there's at least one child which wasn't explored if None in self.child_nodes: @@ -145,9 +129,9 @@ def get_mcts_policy(self, q_value_weight=.65): return policy else: if max(self.n) == 1: - policy = (self.n + 0.05 * self.p)#/ self.n_sum + policy = (self.n + 0.05 * self.p) else: - policy = (self.n - 0.05 * self.p) #/ self.n_sum + policy = (self.n - 0.05 * self.p) return policy / sum(policy) @@ -161,8 +145,9 @@ def apply_dirichlet_noise_to_prior_policy(self, epsilon=0.25, alpha=0.15): """ if self.is_leaf is False: - dirichlet_noise = np.random.dirichlet([alpha] * self.nb_direct_child_nodes) - self.p = (1 - epsilon) * self.p + epsilon * dirichlet_noise + with self.lock: + dirichlet_noise = np.random.dirichlet([alpha] * self.nb_direct_child_nodes) + self.p = (1 - epsilon) * self.p + epsilon * dirichlet_noise def apply_virtual_loss_to_child(self, child_idx, virtual_loss): @@ -177,7 +162,6 @@ def apply_virtual_loss_to_child(self, child_idx, virtual_loss): # make it look like if one has lost X games from this node forward where X is the virtual loss value self.w[child_idx] -= virtual_loss self.q[child_idx] = self.w[child_idx] / self.n[child_idx] - #parent_node.update_u_for_child(child_idx, self.cpuct) def revert_virtual_loss_and_update(self, child_idx, virtual_loss, value): # revert the virtual loss effect and apply the backpropagated value of its child node @@ -186,7 +170,6 @@ def revert_virtual_loss_and_update(self, child_idx, virtual_loss, value): self.n[child_idx] -= virtual_loss - 1 self.w[child_idx] += virtual_loss + value - self.q[child_idx] = self.w[child_idx] / self.n[child_idx] - #parent_node.update_u_for_child(child_idx, self.cpuct) - self.nb_total_expanded_child_nodes += 1 - self.nb_expandable_child_nodes += self.nb_direct_child_nodes + self.q[child_idx] = self.w[child_idx] / self.n[child_idx] + self.nb_total_expanded_child_nodes += 1 + self.nb_expandable_child_nodes += self.nb_direct_child_nodes diff --git a/DeepCrazyhouse/src/domain/crazyhouse/GameState.py b/DeepCrazyhouse/src/domain/crazyhouse/GameState.py index 5052d991..3867c58d 100644 --- a/DeepCrazyhouse/src/domain/crazyhouse/GameState.py +++ b/DeepCrazyhouse/src/domain/crazyhouse/GameState.py @@ -3,6 +3,8 @@ from DeepCrazyhouse.src.domain.crazyhouse.input_representation import board_to_planes from DeepCrazyhouse.src.domain.abstract_cls._GameState import _GameState import numpy as np +import collections + class GameState(_GameState): @@ -12,12 +14,12 @@ def __init__(self, board=CrazyhouseBoard()): self._fen_dic = {} self._board_occ = 0 - def apply_move(self, move: chess.Move, remember_state=False): + def apply_move(self, move: chess.Move): #, remember_state=False): # apply the move on the board self.board.push(move) - if remember_state is True: - self._remember_board_state() + #if remember_state is True: + # self._remember_board_state() def get_state_planes(self): return board_to_planes(self.board, board_occ=self._board_occ, normalize=True) @@ -28,8 +30,15 @@ def get_pythonchess_board(self): def is_draw(self): # check if you can claim a draw - its assumed that the draw is always claimed - return self.board.can_claim_fifty_moves() #can_claim_draw() - #return self.board.can_claim_threefold_repetition() + return self.can_claim_threefold_repetition() or self.board.can_claim_fifty_moves() + #return self.board.can_claim_draw() + + def can_claim_threefold_repetition(self): + """ + Custom implementation for threefold-repetition check which uses the board_occ variable. + :return: True if claim is legal else False + """ + return self._board_occ >= 2 def is_won(self): # only a is_won() and no is_lost() function is needed because the game is over @@ -37,7 +46,11 @@ def is_won(self): return self.board.is_checkmate() def get_legal_moves(self): - return self.board.legal_moves + #return list(self.board.legal_moves) + legal_moves = [] + for mv in self.board.generate_legal_moves(): + legal_moves.append(mv) + return legal_moves def is_white_to_move(self): return self.board.turn @@ -52,19 +65,19 @@ def new_game(self): def set_fen(self, fen, remember_state=True): self.board.set_fen(fen) - if remember_state is True: - self._remember_board_state() - - def _remember_board_state(self): - fen = self.board.board_fen() - if fen in self._fen_dic: - # create a new entry in the dictionary if there exists one - self._fen_dic[fen] += 1 - else: - # create a new entry in the dictionary if there exists one - self._fen_dic[fen] = 1 - # receive the number of occurrence given the fen list - self._board_occ = self._fen_dic[fen] - 1 + #if remember_state is True: + # self._remember_board_state() + + #def _remember_board_state(self): + # calculate the transposition key + # transposition_key = self.get_transposition_key() + # update the number of board occurrences + #self._board_occ = self._transposition_table[transposition_key] + # increase the counter for this transposition key + # self._transposition_table.update((transposition_key,)) + + def is_check(self): + return self.board.is_check() def are_pocket_empty(self): """ diff --git a/DeepCrazyhouse/src/domain/crazyhouse/input_representation.py b/DeepCrazyhouse/src/domain/crazyhouse/input_representation.py index 4a677374..48f8cd6a 100644 --- a/DeepCrazyhouse/src/domain/crazyhouse/input_representation.py +++ b/DeepCrazyhouse/src/domain/crazyhouse/input_representation.py @@ -69,6 +69,8 @@ def board_to_planes(board, board_occ=0, normalize=True): :return: planes - the plane representation of the current board state """ + # TODO: Remove board.mirror() for black by addressing the according color channel + # (I) Define the Input Representation for one position planes_pos = np.zeros((NB_CHANNELS_POS, BOARD_HEIGHT, BOARD_WIDTH)) planes_const = np.zeros((NB_CHANNELS_CONST, BOARD_HEIGHT, BOARD_WIDTH)) @@ -82,14 +84,20 @@ def board_to_planes(board, board_occ=0, normalize=True): board = board.mirror() # Fill in the piece positions - board_piece_map = board.piece_map() - for pos in board_piece_map: - p_char = str(board_piece_map[pos]) - channel = P_MAP[p_char] - row, col = get_row_col(pos) - # set the bit at the right position - planes_pos[channel, row, col] = 1 + # Iterate over both color starting with WHITE + for z, color in enumerate(chess.COLORS): + # the PIECE_TYPE is an integer list in python-chess + for piece_type in chess.PIECE_TYPES: + # define the channel by the piecetype (the input representation uses the same ordering as python-chess) + # we add an offset for the black pieces + # note that we subtract 1 because in python chess the PAWN has index 1 and not 0 + channel = (piece_type-1) + z * len(chess.PIECE_TYPES) + # iterate over the piece mask and receive every position square of it + for pos in board.pieces(piece_type, color): + row, col = get_row_col(pos) + # set the bit at the right position + planes_pos[channel, row, col] = 1 # (II) Fill in the Repetition Data # a game to test out if everything is working correctly is: https://lichess.org/jkItXBWy#73 @@ -114,22 +122,15 @@ def board_to_planes(board, board_occ=0, normalize=True): planes_pos[ch + 5, :, :] = board.pockets[chess.BLACK].count(p_type) # (III) Fill in the promoted pieces + # iterate over all promoted pieces according to the mask and set the according bit + ch = CHANNEL_MAPPING_POS['promo'] + for pos in chess.SquareSet(board.promoted): + row, col = get_row_col(pos) - bb_pos = chess.BB_A1 - - # iterate over all board field and check if there is a positive result for the binary & operation - board_promoted = board.promoted - for pos in range(0, 64): - if board_promoted & bb_pos > 0: - ch = CHANNEL_MAPPING_POS['promo'] - row, col = get_row_col(pos) - - if board.piece_at(pos).color == chess.WHITE: - planes_pos[ch, row, col] = 1 - else: - planes_pos[ch + 1, row, col] = 1 - # for each new square the value is doubled - bb_pos *= 2 + if board.piece_at(pos).color == chess.WHITE: + planes_pos[ch, row, col] = 1 + else: + planes_pos[ch + 1, row, col] = 1 # (III.2) En Passant Square # mark the square where an en-passant capture is possible @@ -189,7 +190,8 @@ def board_to_planes(board, board_occ=0, normalize=True): board = board.mirror() if normalize is True: - planes = normalize_input_planes(planes) + planes *= MATRIX_NORMALIZER + #planes = normalize_input_planes(planes) # return the plane representation of the given board return planes diff --git a/DeepCrazyhouse/src/domain/crazyhouse/output_representation.py b/DeepCrazyhouse/src/domain/crazyhouse/output_representation.py index 47f9e3d2..8a9535b8 100644 --- a/DeepCrazyhouse/src/domain/crazyhouse/output_representation.py +++ b/DeepCrazyhouse/src/domain/crazyhouse/output_representation.py @@ -208,7 +208,7 @@ def value_to_centipawn(value): :return: """ - if np.absolute(value) == 1.: + if np.absolute(value) >= 1.: # return a constant if the given value is 1 (otherwise log will result in infinity) return np.sign(value) * 9999 else: diff --git a/DeepCrazyhouse/src/domain/util.py b/DeepCrazyhouse/src/domain/util.py index 713a9d88..e14e944f 100644 --- a/DeepCrazyhouse/src/domain/util.py +++ b/DeepCrazyhouse/src/domain/util.py @@ -147,6 +147,11 @@ def normalize_input_planes(x): return x +# use a constant matrix for normalization to allow broad cast operations +MATRIX_NORMALIZER = np.ones((NB_CHANNELS_FULL, BOARD_HEIGHT, BOARD_WIDTH)) +MATRIX_NORMALIZER = normalize_input_planes(MATRIX_NORMALIZER) + + def unnormalize_input_planes(x): """ Reverts normalization back to integer values. Works in place. diff --git a/DeepCrazyhouse/src/runtime/Colorer.py b/DeepCrazyhouse/src/runtime/ColorLogger.py similarity index 78% rename from DeepCrazyhouse/src/runtime/Colorer.py rename to DeepCrazyhouse/src/runtime/ColorLogger.py index 17e2cddb..83114adb 100644 --- a/DeepCrazyhouse/src/runtime/Colorer.py +++ b/DeepCrazyhouse/src/runtime/ColorLogger.py @@ -1,8 +1,5 @@ """ @file: Colorer.py -Created on 10.06.18 -@project: DeepCrazyhouse -@author: queensgambit Script which allows are colored logging output multiplattform. The script is based on this post and was slightly adjusted: @@ -127,26 +124,27 @@ def new(*args): return new -if platform.system() == 'Windows': - # Windows does not support ANSI escapes and we are using API calls to set the console color - logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) -else: - # all non-Windows platforms are supporting ANSI escapes so we use them - logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) +def enable_color_logging(debug_lvl=logging.DEBUG): + if platform.system() == 'Windows': + # Windows does not support ANSI escapes and we are using API calls to set the console color + logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) + else: + # all non-Windows platforms are supporting ANSI escapes so we use them + logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) -root = logging.getLogger() -root.setLevel(logging.DEBUG) + root = logging.getLogger() + root.setLevel(debug_lvl) -ch = logging.StreamHandler(sys.stdout) -ch.setLevel(logging.DEBUG) -# FORMAT = '[%(asctime)-s][%(name)-s][\033[1m%(levelname)-7s\033[0m] %(message)-s' -# FORMAT='%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(debug_lvl) + # FORMAT = '[%(asctime)-s][%(name)-s][\033[1m%(levelname)-7s\033[0m] %(message)-s' + # FORMAT='%(asctime)s %(name)-12s %(levelname)-8s %(message)s' -# FORMAT from https://github.com/xolox/python-coloredlogs -FORMAT = '%(asctime)s %(name)s[%(process)d] \033[1m%(levelname)s\033[0m %(message)s' + # FORMAT from https://github.com/xolox/python-coloredlogs + FORMAT = '%(asctime)s %(name)s[%(process)d] \033[1m%(levelname)s\033[0m %(message)s' -# FORMAT="%(asctime)s %(name)-12s %(levelname)-8s %(message)s" -formatter = logging.Formatter(FORMAT, "%Y-%m-%d %H:%M:%S") + # FORMAT="%(asctime)s %(name)-12s %(levelname)-8s %(message)s" + formatter = logging.Formatter(FORMAT, "%Y-%m-%d %H:%M:%S") -ch.setFormatter(formatter) -root.addHandler(ch) + ch.setFormatter(formatter) + root.addHandler(ch) diff --git a/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb b/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb index 9d10d1f0..e518ae55 100644 --- a/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb +++ b/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb @@ -54,7 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "batch_size = 8 #1 #8" + "batch_size = 4 #1 #8" ] }, { @@ -63,7 +63,9 @@ "metadata": {}, "outputs": [], "source": [ - "net = NeuralNetAPI(ctx='gpu', batch_size=batch_size)" + "nets = []\n", + "for i in range(2):\n", + " nets.append(NeuralNetAPI(ctx='gpu', batch_size=batch_size))" ] }, { @@ -72,7 +74,7 @@ "metadata": {}, "outputs": [], "source": [ - "raw_agent = RawNetAgent(net)" + "raw_agent = RawNetAgent(nets[0], temperature=0.1, temperature_moves=4)" ] }, { @@ -81,10 +83,10 @@ "metadata": {}, "outputs": [], "source": [ - "mcts_agent = MCTSAgent(net, threads=16, playouts_empty_pockets=4096*5, playouts_filled_pockets=4096*5,\n", - " cpuct=3, dirichlet_epsilon=.1, dirichlet_alpha=0.2, batch_size=batch_size, q_value_weight=0,#.5, #99,\n", - " max_search_depth=40, temperature=0., clip_quantil=0., virtual_loss=3, verbose=False,\n", - " min_movetime=20000, check_mate_in_one=False, enable_timeout=False)" + "mcts_agent = MCTSAgent(nets, threads=16, playouts_empty_pockets=4096*10, playouts_filled_pockets=4096*10,\n", + " cpuct=3, dirichlet_epsilon=.25, dirichlet_alpha=0.2, batch_size=batch_size, q_value_weight=.7,\n", + " max_search_depth=100, temperature=0.1, temperature_moves=4, virtual_loss=3, verbose=False,\n", + " min_movetime=8000, check_mate_in_one=False)" ] }, { @@ -95,11 +97,12 @@ "source": [ "board = chess.variant.CrazyhouseBoard()\n", "\n", - "#board.push_uci('e2e4')\n", + "board.push_uci('e2e4')\n", "#board.push_uci('e7e6')\n", "\n", - "fen = 'rnbqkb1r/ppp1pppp/5n2/3P4/8/8/PPPP1PPP/RNBQKBNR/P w KQkq - 1 3'\n", + "#fen = 'rnbqkb1r/ppp1pppp/5n2/3P4/8/8/PPPP1PPP/RNBQKBNR/P w KQkq - 1 3'\n", "#fen = 'rnb2rk1/p3bppp/2p5/3p2P1/4n3/8/PPPPBPPP/RNB1K1NR/QPPq w KQ - 0 11'\n", + "\n", "#fen = 'r1b1kbnr/ppp1pppp/2n5/3q4/3P4/8/PPP1NPPP/RNBQKB1R/Pp b KQkq - 1 4'\n", "#fen = 'r1b1k2r/ppp2ppp/2n5/3np3/3P4/2PBP3/PpPB1PPP/1Q2K1NR/QNrb b Kkq - 27 14'\n", "#fen = 'r1bb4/ppp2pkp/5npb/4p3/4P3/2N5/PPP1BPPP/3RK2R/QNRqpnp w K - 3 16'\n", @@ -110,10 +113,13 @@ "#fen = 'rn2N2k/pp5p/3pp1pN/3p4/5P2/3P1p2/PP3RPP/RN4K1/QQprbbpbb b - - 1 30'\n", "\n", "#fen = '3R1b2/1bP1kp2/3Npn1p/3p4/5p2/5N1b/PPP1QP1P/3R1RK1/QPpprnpbp b - - 0 29'\n", + "# TEST POSITIONS:\n", "#fen = 'rn2N2k/pp5p/3pp1pN/3p4/3q1P2/3P1p2/PP3PPP/RN3RK1[Qrbbpbb] b - - 3 30'\n", - "#fen = 'q6r/p2P1pkp/1p1b1n2/2p2B2/8/6n1/PPP2KPp/R4R2/PNNRPBPbqpp w - - 2 26'\n", "#fen ='2kr1b2/1bp2p1p/p3pP1p/1p5Q/5B2/3B1p2/PPP2PrP/R4R1K/QNpnnnp w - - 0 18'\n", - "#fen = 'q6r/p2P1pkp/1p1b1n2/2p2B2/8/6n1/PPP2KPp/R4R2/PNNRPBPbqpp w - - 50 26'\n", + "#fen = 'q6r/p2P1pkp/1p1b1n2/2p2B2/8/6n1/PPP2KPp/R4R2/PNNRPBPbqpp w - - 0 26'\n", + "#fen = 'q6r/p2P2kp/1p1bpn2/2p2B2/8/6n1/PPP2KPp/R4R2/PNRPBPnbqpp w - - 0 27'\n", + "#fen = 'q6N~/p2Pk2p/1p1bpn1P/2p2B2/8/6n1/PPP2KPp/R4R2/RNBPrnbqpp w - - 2 31'\n", + "\n", "#fen = 'r4r1k/ppp1q1bp/4Ppp1/1P6/2NN3P/2BPPb2/P2PNPpR/R3K3/PBQn b Q - 2 34'\n", "#fen = 'r1bq1b1r/ppp1kPpp/4Pn2/n2Pp3/2B4n/3P4/PPP2PPP/RNBQK2R/ w KQ - 0 10'\n", "\n", @@ -141,12 +147,8 @@ "#fen = 'r1bqk1r1/ppp2ppp/2n2n2/3p1N2/1bB1P3/2N4P/PPP2P1P/R1BQK1R1/PP w Qq - 0 10'\n", "#fen = 'r1bqk2r/ppppbppp/2n2n2/8/2BNP3/8/PPP2PPP/RNBQK2R/Pp w KQkq - 1 6'\n", "#fen = 'r1b1k1r1/pp1p1ppp/1q1B1nn1/1B3N2/4P3/2P1p3/P1P1QPpP/R3K1R1/BPPn w Qq - 34 18'\n", - "#fen = 'r1bqk3/ppp1bprp/3p1n2/5PP1/4P2n/2NQ4/PPP2PBP/R1B1K2R/Npp w KQq - 1 13'\n", - "#fen = 'r1bqk1nr/pppp1ppp/5b2/4nNP1/2B1P3/7p/PPP2PPP/RNBQK2R/ w KQkq - 1 8'\n", - "#fen = 'r1b1k1nr/ppp2ppp/5q2/3pnN2/4P3/7p/PPP2PPP/RNBQKB1R/Bp w KQkq - 1 10'\n", "#fen = 'r3k2r/ppp2ppp/5n2/3pnb2/4P3/7p/PPP2PPP/RNBQKB1R/Qbnp w KQkq - 22 12'\n", "#fen = 'rnbqkbnr/pppp1ppp/8/4p3/4P3/8/PPPP1PPP/RNBQKBNR/ w KQkq - 2 2'\n", - "#fen = 'r1bq1rk1/ppp2ppp/2np4/2bBp3/4P1n1/3P1N2/PPP1QPPP/R1B2RK1/N w - - 2 10'\n", "#fen = 'r1bq1rk1/pppp1ppp/2n2n2/2b1p3/2B1P3/2NP1N2/PPP2PPP/R1BQ1RK1/ w - - 12 7'\n", "#fen = 'r1bq1rk1/pppp1ppp/2n5/2bBp3/4P1n1/3P1N2/PPP2PPP/R1BQ1RK1/N w - - 2 9'\n", "#fen = 'r1bqk1r1/ppppbppp/5n2/1B2nN2/4P3/2N4p/PPP2PPP/R1BQK2R/P w KQq - 16 9'\n", @@ -155,6 +157,38 @@ "#fen = 'r1bk3r/ppppbpQp/4p3/8/4n3/4P2N/PPPP2PP/R1Bq1BKR/PNNp b - - 2 13'\n", "#fen = 'r1bqkbnr/ppp2ppp/3p4/8/3QP3/2N4p/PPP2PPP/R1B1KB1R/PNn w KQkq - 1 7'\n", "#fen = 'r1bq1rk1/ppp2pp1/2np1n1p/2b1p1B1/2B1P3/2NP1N2/PPP2PPP/R2Q1RK1/ w - - 14 8'\n", + "#fen = 'r4k2/ppp2pp1/2np1bpr/6P1/5P2/4B1N1/PPb1BPPP/R4RK1/Qpnnqp w - - 1 31'\n", + "#fen = 'r4k2/ppp2p2/2np1ppr/8/5P2/4B1N1/PPb1BPPP/R4RK1/BQppnnqp w - - 1 32'\n", + "#fen = 'r3kb1r/2p2pp1/ppN1p3/3qPnB1/5Q2/7P/PPP1BP1P/4RRK1/BNPpnp b kq - 0 16'\n", + "#fen = 'r3kb1r/2p2pp1/ppN1p1n1/3qPnB1/5Q2/7P/PPP1BP1P/4RRK1/BNPpp w kq - 0 17'\n", + "#fen = '3Q~1nkr/pp2np1p/3PbB1b/3p4/3P4/5BPp/P1P2PPp/3R1R1K/QNnrqp w - - 0 33'\n", + "#fen = '5bkr/pp2np1p/3PbB2/3p4/3P4/5BPp/P1P2PPp/3R1R1K/NQNpnrqp w - - 0 34'\n", + "#fen = 'r1bqkb1r/ppp1pppp/2n5/3n4/8/2N2N2/PPPP1PPP/R1BQKB1R/Pp w KQkq - 0 5'\n", + "#fen = 'r2B1rk1/ppp2ppp/2np4/2bBp3/4P1b1/3P1N2/PPP2PPP/R2n1RK1/NQq w - - 0 11'\n", + "#fen = 'r1bqkb1r/ppp1pppp/2n5/1B1n4/8/2N2N1p/PPPP1PPP/R1BQK1R1/P b Qkq - 0 6'\n", + "#fen = 'r1k2b1r/p2bp1pp/q2p4/B3N1Q1/bP6/K1P4p/P1P1nPpP/6R1/PNNPPr w - - 0 31'\n", + "#fen = 'rnb1kb1r/ppp1pppp/5n2/q7/8/2N2N2/PPPP1PPP/R1BQKB1R/Pp w KQkq - 0 5'\n", + "#fen = 'rnb2b1r/ppp1pkpp/5n2/2q5/3N2p1/2N5/PPPP1PPP/R1BQK2R/PPb w KQ - 0 8'\n", + "#fen = 'r1b2bnr/ppk1pppp/2np4/2Q5/1p1P4/3b4/PP2nPPP/R1B2RK1/QNPP w - - 1 19'\n", + "#fen = 'rnbqkbnr/pppp1ppp/8/4p3/4P3/8/PPPP1PPP/RNBQKBNR/ w KQkq - 2 2'\n", + "#fen = 'r1bq1rk1/ppp2pp1/2np1n1p/2b1p1B1/2B1P3/2NP1N2/PPP2PPP/R2Q1RK1/ w - - 14 8'\n", + "#fen = 'r1bqkb1r/pppppppp/2n2n2/8/4P3/2N5/PPPP1PPP/R1BQKBNR/ w KQkq - 4 3'\n", + "#fen = 'r1bqkb1r/ppp2ppp/2n1p3/3pP3/3P4/2PBB3/PnP2PPP/R2QK1NR/N w KQkq - 0 8'\n", + "#fen = 'r1bk3r/ppp2ppp/3ppb2/6N1/5B2/4P3/PPP2PPP/R2K1BNR/QNnpq b - - 0 13'\n", + "#fen = 'r1bqk2r/pppp1ppp/2n1p3/4N3/1b2Q3/4P3/PPPP1PPP/R1B1KBNR/n b KQkq - 0 6'\n", + "#fen = 'r1b2R2/ppq2p1k/2n2Ppp/4p2N/8/P1PB1N1P/2P2PP1/R2Q1RK1/ppnbbp w - - 0 23'\n", + "#fen = 'r1b3k1/ppq2pP1/3n1Ppp/4n2N/4B3/P1P4P/2P2PPp/R2Q1R1K/Brpnbp w - - 0 28'\n", + "#fen = 'r1b1kb1r/p1p1pppp/2N5/1B2N3/2pPn3/2PKB3/P1PP2p1/3q1rR1/QPPNP w - - 0 25'\n", + "# example to check for infinite feed-back-loop\n", + "#fen = 'rn1k4/p2p1qNp/b1pp4/8/1p6/5BP1/PPP1PPBP/R4RK1/BRqpnppn w - - 0 30'\n", + "fen = 'r1bqkb1r/pp2pppp/2p5/1B1pP3/4p3/2P1P3/P1P1N1PP/Q3K2R/NBNNr w Kkq - 0 13'\n", + "#fen = 'r1bqkb1r/ppp1pppp/2n5/3pP3/3Pn3/2N5/PPP2PPP/R1BQKBNR/ w KQkq - 8 5'\n", + "#fen = 'r3kb1r/pbp1pppp/2n5/1B1q4/8/2P2N2/P1PP1PpP/R1BQK1R1/PNpnp w Qkq - 1 10'\n", + "#fen = 'r3kb1r/pbp1pppp/2n5/1B1q4/8/2P2N2/P1PP1PpP/R1BQK1R1/PNpnp w Qkq - 1 10'\n", + "#fen = 'r1bqkbnr/pppp1ppp/2n5/4p3/2B1P3/5N2/PPPP1PPP/RNBQK2R/ b KQkq - 5 3'\n", + "#fen = 'r1bq1rk1/ppp2pp1/2np1n1p/2b1p1B1/2B1P3/2NP1N2/PPP2PPP/R2Q1RK1/ w - - 14 8'\n", + "#fen = 'r1bqkbnr/ppp2ppp/2n5/3pp3/8/3P1NP1/PPP1PPBP/RNBQK2R/ b KQkq - 3 4'\n", + "fen = 'r2q1rk1/ppp1bpp1/2n1bn1p/3pp3/8/P1NP1NPP/1PP1PPB1/R1BQ1RK1/ w - - 1 9'\n", "board.set_fen(fen)\n", "#board = board.mirror()\n", "\n", @@ -162,6 +196,28 @@ "board" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3 fold-test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "import chess.pgn\n", + "pgn = open(\"crazywa_crazyara_3fold_rep.pgn\")\n", + "state = GameState(chess.variant.CrazyhouseBoard())\n", + "first_game = chess.pgn.read_game(pgn)\n", + "# Iterate through all moves and play them on a board.\n", + "board = first_game.board()\n", + "for move in first_game.main_line():\n", + " state.apply_move(move, remember_state=True)\n", + " #board.push(move)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -242,6 +298,15 @@ "plot_moves_with_prob(legal_moves, p_vec_small, only_top_x=10)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "p_vec_small" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -250,16 +315,24 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ "t_s = time()\n", "value, legal_moves, p_vec_small = mcts_agent.evaluate_board_state(state)\n", "print('Elapsed time: %.4fs' % (time()-t_s))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(1):\n", + " print(mcts_agent.perform_action(state))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -278,6 +351,34 @@ "mcts_agent.root_node.legal_moves[mcts_agent.root_node.q.argmax()]" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mcts_agent.root_node.q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mcts_agent.root_node.q.argmax()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "board = chess.variant.CrazyhouseBoard('r3kb1r/p1p1pppp/2N5/4N3/2BPb3/2PKB1n1/P1PP2p1/5rR1/PQPPNPq w - - 56 29')\n", + "board.is_checkmate()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -391,7 +492,7 @@ "metadata": {}, "outputs": [], "source": [ - "c" + "mcts_agent.root_node.n.argmax()" ] }, { @@ -403,13 +504,43 @@ "thresh_idces" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "value_queues = np.zeros(mcts_agent.root_node.nb_direct_child_nodes)\n", + "for i, node in enumerate(mcts_agent.root_node.child_nodes):\n", + " value_queues[i] = np.sum(list(node.value_queue)) / mcts_agent.root_node.n[i]\n", + "print(value_queues)\n", + "max_visits = mcts_agent.root_node.n.max()\n", + "thresh_idces = mcts_agent.root_node.n < max_visits * 0.33 #5 #2 # 0.5 #.33 \n", + "value_queues[thresh_idces] = -1\n", + "print(value_queues)\n", + "#value_queues *= np.sqrt(mcts_agent.root_node.n)\n", + "q_combi = value_queues + mcts_agent.root_node.q" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "c.sum()" + "len(legal_moves)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "plot_moves_with_prob(legal_moves, value_queues, only_top_x=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "plot_moves_with_prob(legal_moves, q_combi, only_top_x=10)" ] }, { @@ -418,7 +549,7 @@ "metadata": {}, "outputs": [], "source": [ - "mcts_agent.root_node.n.argmax()" + "((mcts_agent.root_node.q.max() + 1) / 2) * .75" ] }, { @@ -427,7 +558,7 @@ "metadata": {}, "outputs": [], "source": [ - "mcts_agent.get_calculated_line()" + "mcts_agent.root_node.n.argmax()" ] }, { @@ -445,7 +576,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_moves_with_prob(legal_moves, mcts_agent.root_node.q, only_top_x=10)" + "plot_moves_with_prob(legal_moves, c, only_top_x=10)" ] }, { @@ -454,7 +585,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_moves_with_prob(legal_moves, c, only_top_x=10)" + "plot_moves_with_prob(legal_moves, mcts_agent.root_node.q, only_top_x=10)" ] }, { @@ -463,7 +594,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_moves_with_prob(legal_moves, p_vec_small, only_top_x=10)" + "p_vec_small = mcts_agent.root_node.get_mcts_policy(0.5)" ] }, { @@ -489,6 +620,53 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "chess.PIECE_TYPES" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "board.promoted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "chess.BISHOP" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mask = board.pieces(chess.BISHOP,chess.BLACK)\n", + "#mask.pop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for square in mask:\n", + " print(square)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -505,7 +683,7 @@ "outputs": [], "source": [ "%load_ext line_profiler\n", - "r = %lprun -r -f board_to_planes board_to_planes(board)\n", + "r = %lprun -r -f board_to_planes board_to_planes(board.mirror())\n", "r.print_stats()" ] }, diff --git a/DeepCrazyhouse/src/tests/FullRoundTripTests.py b/DeepCrazyhouse/src/tests/FullRoundTripTests.py index 199e4dff..ef282560 100644 --- a/DeepCrazyhouse/src/tests/FullRoundTripTests.py +++ b/DeepCrazyhouse/src/tests/FullRoundTripTests.py @@ -7,8 +7,8 @@ Loads the plane representation for the test dataset and iterates through all board positions and moves. """ -from src.domain.util import * -from src.domain.crazyhouse.input_representation import planes_to_board +from DeepCrazyhouse.src.domain.util import * +from DeepCrazyhouse.src.domain.crazyhouse.input_representation import planes_to_board from DeepCrazyhouse.src.domain.crazyhouse.output_representation import policy_to_move import chess import chess.pgn @@ -17,8 +17,11 @@ from DeepCrazyhouse.src.preprocessing.PGN2PlanesConverter import PGN2PlanesConverter from multiprocessing import Pool from copy import deepcopy +from DeepCrazyhouse.src.preprocessing.dataset_loader import load_pgn_dataset # import the Colorer to have a nicer logging printout +from DeepCrazyhouse.src.runtime.ColorLogger import enable_color_logging +enable_color_logging() def board_single_game(params_inp): diff --git a/crazyara.py b/crazyara.py index d8c04477..0eb53286 100644 --- a/crazyara.py +++ b/crazyara.py @@ -14,7 +14,11 @@ import chess.pgn import traceback - +import collections +import numpy as np +# import the Colorer to have a nicer logging printout +from DeepCrazyhouse.src.runtime.ColorLogger import enable_color_logging +enable_color_logging() # Constants MIN_SEARCH_TIME_MS = 100 @@ -24,9 +28,21 @@ MAX_BAD_POS_VALUE = -0.10 # When pos eval [-1.0 to 1.0] is equal or worst than this then extend time MOVES_LEFT_INCREMENT = 10 # Used to reduce the movetime in the opening +# this is the assumed "maximum" blitz game length for calculating a constant movetime +# after 80% of this game length a new time management starts which is based on movetime left +BLITZ_GAME_LENGTH = 50 +# use less time in the opening defined by "max_move_num_to_reduce_movetime" by using a portion of the constant move time +MV_TIME_OPENING_PORTION = 0.7 +# this variable is intended to increase the variance in the moves played by using a small different amount of time each +# move +RANDOM_MV_TIME_PORTION = 0.1 + +# enable this variable if you want to see debug messages in certain environments, like the lichess.org api +ENABLE_LICHESS_DEBUG_MSG = False + client = { 'name': 'CrazyAra', - 'version': '0.2.0', + 'version': '0.3.0', 'authors': 'Johannes Czech, Moritz Willig, Alena Beyer et al.' } @@ -55,6 +71,7 @@ " ASCII-Art: Joan G. Stark, Chappell, Burton \n" log_file_path = "CrazyAra-log.txt" +score_file_path = "score-log.txt" try: log_file = open(log_file_path, 'w') @@ -66,21 +83,39 @@ print(traceback_text) +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def print_if_debug(string): + if ENABLE_LICHESS_DEBUG_MSG is True: + eprint("[debug] " + string) + + def log_print(text: str): print(text) + print_if_debug(text) if log_file: log_file.write("< %s\n" % text) log_file.flush() +def write_score_to_file(score: str): + score_file = open(score_file_path, 'w') + score_file.seek(0) + score_file.write(score) + score_file.truncate() + score_file.close() + + def log(text: str): if log_file: log_file.write("> %s\n" % text) log_file.flush() -print(INTRO_PART1, end="") -print(INTRO_PART2, end="") +eprint(INTRO_PART1, end="") +eprint(INTRO_PART2, end="") # GLOBAL VARIABLES mcts_agent = None @@ -88,7 +123,11 @@ def log(text: str): gamestate = None setup_done = False bestmove_value = None +constant_move_time = None engine_played_move = 0 +score = None +# transposition table for future 3-fold-repetition check +transposition_table = collections.Counter() # SETTINGS s = { @@ -99,13 +138,15 @@ def log(text: str): "use_raw_network": False, "threads": 16, "batch_size": 8, + "neural_net_services": 2, "playouts_empty_pockets": 8192, "playouts_filled_pockets": 8192, "centi_cpuct": 300, - "centi_dirichlet_epsilon": 10, + "centi_dirichlet_epsilon": 25, "centi_dirichlet_alpha": 20, "max_search_depth": 40, - "centi_temperature": 0, + "centi_temperature": 10, + "temperature_moves": 0, "centi_clip_quantil": 0, "virtual_loss": 3, "centi_q_value_weight": 70, @@ -113,9 +154,10 @@ def log(text: str): "move_overhead_ms": 300, "moves_left": 40, "extend_time_on_bad_position": True, - "max_move_num_to_reduce_movetime": 0, + "max_move_num_to_reduce_movetime": 4, "check_mate_in_one": False, - "enable_timeout": False, + "use_pruning": True, + "use_oscillating_cpuct": True, "verbose": False } @@ -142,41 +184,57 @@ def setup_network(): # check for valid parameter setup and do auto-corrections if possible param_validity_check() - net = NeuralNetAPI(ctx=s['context'], batch_size=s['batch_size']) - rawnet_agent = RawNetAgent(net, temperature=s['centi_temperature'], clip_quantil=s['centi_clip_quantil']) + nets = [] + for i in range(s['neural_net_services']): + nets.append(NeuralNetAPI(ctx=s['context'], batch_size=s['batch_size'])) - mcts_agent = MCTSAgent(net, cpuct=s['centi_cpuct'] / 100, playouts_empty_pockets=s['playouts_empty_pockets'], + rawnet_agent = RawNetAgent(nets[0], temperature=s['centi_temperature'], temperature_moves=s['temperature_moves']) + + mcts_agent = MCTSAgent(nets, cpuct=s['centi_cpuct'] / 100, playouts_empty_pockets=s['playouts_empty_pockets'], playouts_filled_pockets=s['playouts_filled_pockets'], max_search_depth=s['max_search_depth'], dirichlet_alpha=s['centi_dirichlet_alpha'] / 100, q_value_weight=s['centi_q_value_weight'] / 100, dirichlet_epsilon=s['centi_dirichlet_epsilon'] / 100, virtual_loss=s['virtual_loss'], - threads=s['threads'], temperature=s['centi_temperature'] / 100, verbose=s['verbose'], - clip_quantil=s['centi_clip_quantil'] / 100, min_movetime=MIN_SEARCH_TIME_MS, + threads=s['threads'], temperature=s['centi_temperature'] / 100, + temperature_moves=s['temperature_moves'], verbose=s['verbose'], + min_movetime=MIN_SEARCH_TIME_MS, batch_size=s['batch_size'], check_mate_in_one=s['check_mate_in_one'], - enable_timeout=s['enable_timeout']) + use_pruning=s['use_pruning'], use_oscillating_cpuct=s['use_oscillating_cpuct']) gamestate = GameState() setup_done = True -def param_validity_check(): +def validity_with_threads(optname: str): """ - Handles some possible issues when giving an illegal batch_size and number of threads combination. + Checks for consistency with the number of threads with the given parameter + :param optname: Option name :return: """ - if s['batch_size'] > s['threads']: + + if s[optname] > s['threads']: log_print('info string The given batch_size %d is higher than the number of threads %d. ' 'The maximum legal batch_size is the same as the number of threads (here: %d) ' - % (s['batch_size'], s['threads'], s['threads'])) - s['batch_size'] = s['threads'] - log_print('info string The batch_size was reduced to %d' % s['batch_size']) + % (s[optname], s['threads'], s['threads'])) + s[optname] = s['threads'] + log_print('info string The batch_size was reduced to %d' % s[optname]) - if s['threads'] % s['batch_size'] != 0: + if s['threads'] % s[optname] != 0: log_print('info string You requested an illegal combination of threads %d and batch_size %d.' - ' The batch_size must be a divisor of the number of threads' % (s['threads'], s['batch_size'])) - divisor = s['threads'] // s['batch_size'] - s['batch_size'] = s['threads'] // divisor - log_print('info string The batch_size was changed to %d' % s['batch_size']) + ' The batch_size must be a divisor of the number of threads' % (s['threads'], s[optname])) + divisor = s['threads'] // s[optname] + s[optname] = s['threads'] // divisor + log_print('info string The batch_size was changed to %d' % s[optname]) + + +def param_validity_check(): + """ + Handles some possible issues when giving an illegal batch_size and number of threads combination. + :return: + """ + + validity_with_threads('batch_size') + validity_with_threads('neural_net_services') def perform_action(cmd_list): @@ -191,6 +249,8 @@ def perform_action(cmd_list): global rawnet_agent global bestmove_value global engine_played_move + global constant_move_time + global score movetime_ms = MIN_SEARCH_TIME_MS tc_type = None @@ -214,6 +274,9 @@ def perform_action(cmd_list): my_time = btime my_inc = binc + if constant_move_time is None: + constant_move_time = (my_time + BLITZ_GAME_LENGTH * my_inc) / BLITZ_GAME_LENGTH + # TC with period (traditional) like 40/60 or 40 moves in 60 sec repeating if 'movestogo' in cmd_list: tc_type = 'traditional' @@ -229,7 +292,14 @@ def perform_action(cmd_list): moves_left = s['moves_left'] moves_left = adjust_moves_left(moves_left, tc_type, bestmove_value) - movetime_ms = max(my_time/moves_left + INC_FACTOR*my_inc//INC_DIV - s['move_overhead_ms'], MIN_SEARCH_TIME_MS) + if tc_type == 'blitz' and engine_played_move < BLITZ_GAME_LENGTH * .8: + movetime_ms = constant_move_time + (np.random.rand()-0.5) * RANDOM_MV_TIME_PORTION * constant_move_time + + if engine_played_move < s['max_move_num_to_reduce_movetime']: + # avoid spending too much time in the opening + movetime_ms *= MV_TIME_OPENING_PORTION + else: + movetime_ms = max(my_time/moves_left + INC_FACTOR*my_inc//INC_DIV - s['move_overhead_ms'], MIN_SEARCH_TIME_MS) # movetime in UCI protocol, go movetime x, search exactly x mseconds # UCI protocol: http://wbec-ridderkerk.nl/html/UCIProtocol.html @@ -238,17 +308,32 @@ def perform_action(cmd_list): mcts_agent.update_movetime(movetime_ms) log_print('info string Time for this move is %dms' % movetime_ms) + log_print('info string Requested pos: %s' % gamestate) if s['use_raw_network'] or movetime_ms <= s['threshold_time_for_raw_net_ms']: log_print('info string Using raw network for fast mode...') - value, selected_move, confidence, _ = rawnet_agent.perform_action(gamestate) + value, selected_move, confidence, _, cp, depth, nodes, time_elapsed_s, nps, pv = rawnet_agent.perform_action(gamestate) else: - value, selected_move, confidence, _ = mcts_agent.perform_action(gamestate) + value, selected_move, confidence, _, cp, depth, nodes, time_elapsed_s, nps, pv = mcts_agent.perform_action(gamestate) + + score = "score cp %d depth %d nodes %d time %d nps %d pv %s" % (cp, depth, nodes, time_elapsed_s, nps, pv) + if ENABLE_LICHESS_DEBUG_MSG: + write_score_to_file(score) + # print out the search information + log_print('info %s' % score) # Save the bestmove value [-1.0 to 1.0] to modify the next movetime bestmove_value = float(value) engine_played_move += 1 + # apply CrazyAra's selected move the global gamestate + if gamestate.get_pythonchess_board().is_legal(selected_move): + # apply the last move CrazyAra played + gamestate.apply_move(selected_move) + mcts_agent.transposition_table.update((gamestate.get_transposition_key(),)) + else: + raise Exception('all_ok is false! - crazyara_last_move') + log_print('bestmove %s' % selected_move.uci()) @@ -261,25 +346,57 @@ def setup_gamestate(cmd_list): """ #artificial_max_game_len = 30 + global gamestate + global mcts_agent + position_type = cmd_list[1] - if position_type == "startpos": - gamestate.new_game() - else: - fen = " ".join(cmd_list[2:8]) - gamestate.set_fen(fen) if 'moves' in cmd_list: + # position startpos moves e2e4 g8f6 if position_type == 'startpos': mv_list = cmd_list[3:] else: # position fen rn2N2k/pp5p/3pp1pN/3p4/3q1P2/3P1p2/PP3PPP/RN3RK1/Qrbbpbb b - - 3 27 moves d4f2 f1f2 mv_list = cmd_list[9:] - for move in mv_list: - gamestate.apply_move(chess.Move.from_uci(move)) - #if len(mv_list)//2 > artificial_max_game_len: - # log_print('info string Setting fullmove_number to %d' % artificial_max_game_len) - # gamestate.get_pythonchess_board().fullmove_number = artificial_max_game_len + # try to apply opponent last move to the board state + mv_compatible = True + + if len(mv_list) > 0: + # the move the opponent just played is the last move in the list + opponent_last_move = chess.Move.from_uci(mv_list[-1]) + if gamestate.get_pythonchess_board().is_legal(opponent_last_move): + # apply the last move the opponent played + gamestate.apply_move(opponent_last_move) #, remember_state=True) + mcts_agent.transposition_table.update((gamestate.get_transposition_key(),)) + else: + log_print('info string all_ok is false! - opponent_last_move %s' % opponent_last_move) + mv_compatible = False + + if not mv_compatible: + log_print("info string The given last two moves couldn't be applied to the previous board-state.") + log_print("info string Rebuilding the game from scratch...") + + # create a new game state from scratch + if position_type == "startpos": + new_game() + else: + fen = " ".join(cmd_list[2:8]) + gamestate.set_fen(fen) + + for move in mv_list: + gamestate.apply_move(chess.Move.from_uci(move)) + mcts_agent.transposition_table.update((gamestate.get_transposition_key(),)) + else: + log_print("info string Move Compatible") + + +def new_game(): + global gamestate + global mcts_agent + + gamestate.new_game() + mcts_agent.transposition_table = collections.Counter() def set_options(cmd_list): @@ -292,41 +409,47 @@ def set_options(cmd_list): # SETTINGS global s - if cmd_list[1] != 'name' or cmd_list[3] != 'value': - log_print("info string The given setoption command wasn't understood") - log_print('info string An example call could be: "setoption name threads value 4"') - else: - option_name = cmd_list[2] + # make sure there exists enough items in the given command list like "setoption name nb_threads value 1" + if len(cmd_list) >= 5: + if cmd_list[1] != 'name' or cmd_list[3] != 'value': + log_print("info string The given setoption command wasn't understood") + log_print('info string An example call could be: "setoption name threads value 4"') + else: + option_name = cmd_list[2] - if option_name not in s: - raise Exception("The given option %s wasn't found in the settings list" % option_name) + if option_name not in s: + log_print("info string The given option %s wasn't found in the settings list" % option_name) + else: - if option_name in ['UCI_Variant', 'context', 'use_raw_network', - 'extend_time_on_bad_position', 'verbose', 'check_mate_in_one', 'enable_timeout']: + if option_name in ['UCI_Variant', 'context', 'use_raw_network', + 'extend_time_on_bad_position', 'verbose', 'check_mate_in_one', 'use_pruning', + 'use_oscillating_cpuct']: - value = cmd_list[4] - else: - value = int(cmd_list[4]) - - if option_name == 'use_raw_network': - s['use_raw_network'] = True if value == 'true' else False - elif option_name == 'extend_time_on_bad_position': - s['extend_time_on_bad_position'] = True if value == 'true' else False - elif option_name == 'verbose': - s['verbose'] = True if value == 'true' else False - elif option_name == 'check_mate_in_one': - s['check_mate_in_one'] = True if value == 'true' else False - elif option_name == 'enable_timeout': - s['enable_timeout'] = True if value == 'true' else False - else: - # by default all options are treated as integers - s[option_name] = value + value = cmd_list[4] + else: + value = int(cmd_list[4]) + + if option_name == 'use_raw_network': + s['use_raw_network'] = True if value == 'true' else False + elif option_name == 'extend_time_on_bad_position': + s['extend_time_on_bad_position'] = True if value == 'true' else False + elif option_name == 'verbose': + s['verbose'] = True if value == 'true' else False + elif option_name == 'check_mate_in_one': + s['check_mate_in_one'] = True if value == 'true' else False + elif option_name == 'use_pruning': + s['use_pruning'] = True if value == 'true' else False + elif option_name == 'use_oscillating_cpuct': + s['use_oscillating_cpuct'] = True if value == 'true' else False + else: + # by default all options are treated as integers + s[option_name] = value - # Guard threads limits - if option_name == 'threads': - s[option_name] = min(4096, max(1, s[option_name])) + # Guard threads limits + if option_name == 'threads': + s[option_name] = min(4096, max(1, s[option_name])) - log_print('info string Updated option %s to %s' % (option_name, value)) + log_print('info string Updated option %s to %s' % (option_name, value)) def adjust_moves_left(moves_left, tc_type, prev_bm_value): @@ -371,16 +494,18 @@ def uci_reply(): log_print('option name use_raw_network type check default %s' %\ ('false' if not s['use_raw_network'] else 'true')) log_print('option name threads type spin default %d min 1 max 4096' % s['threads']) - log_print('option name batch_size type spin default %d min 1 max 4096' % s['batch_size']) + log_print('option name batch_size type spin default %d min 1 max 4096' % s['batch_size']) + log_print('option name neural_net_services type spin default %d min 1 max 10' % s['neural_net_services']) log_print('option name playouts_empty_pockets type spin default %d min 56 max 8192' %\ s['playouts_empty_pockets']) log_print('option name playouts_filled_pockets type spin default %d min 56 max 8192' %\ s['playouts_filled_pockets']) log_print('option name centi_cpuct type spin default %d min 1 max 500' % s['centi_cpuct']) - log_print('option name centi_dirichlet_epsilon type spin default 10 min 0 max 100') - log_print('option name centi_dirichlet_alpha type spin default 20 min 0 max 100') - log_print('option name max_search_depth type spin default 40 min 1 max 100') - log_print('option name centi_temperature type spin default 0 min 0 max 100') + log_print('option name centi_dirichlet_epsilon type spin default %d min 0 max 100' % s['centi_dirichlet_epsilon']) + log_print('option name centi_dirichlet_alpha type spin default %d min 0 max 100' % s['centi_dirichlet_alpha']) + log_print('option name max_search_depth type spin default %d min 1 max 100' % s['max_search_depth']) + log_print('option name centi_temperature type spin default %d min 0 max 100' % s['centi_temperature']) + log_print('option name temperature_moves type spin default %d min 0 max 99999' % s['temperature_moves']) log_print('option name centi_clip_quantil type spin default 0 min 0 max 100') log_print('option name virtual_loss type spin default 3 min 0 max 10') log_print('option name centi_q_value_weight type spin default %d min 0 max 100' % s['centi_q_value_weight']) @@ -394,8 +519,10 @@ def uci_reply(): s['max_move_num_to_reduce_movetime']) log_print('option name check_mate_in_one type check default %s' %\ ('false' if not s['check_mate_in_one'] else 'true')) - log_print('option name enable_timeout type check default %s' %\ - ('false' if not s['check_mate_in_one'] else 'true')) + log_print('option name use_pruning type check default %s' %\ + ('false' if not s['use_pruning'] else 'true')) + log_print('option name use_oscillating_cpuct type check default %s' %\ + ('false' if not s['use_oscillating_cpuct'] else 'true')) log_print('option name verbose type check default %s' %\ ('false' if not s['verbose'] else 'true')) @@ -403,6 +530,14 @@ def uci_reply(): log_print('uciok') +def handle_uci(line): + #new_game() + print("id name " + client["name"]) # + client["version"]) + print("id author " + client["authors"]) + # print("option name OwnBook, type true") + # print("option variation crazyhouse") + print("uciok") + # main waiting loop for processing command line inputs def main(): global bestmove_value @@ -411,6 +546,9 @@ def main(): while True: line = input() + print_if_debug("waiting ...") + #line = sys.stdin.readline() + print_if_debug(line) # wait for an std-in input command if line: @@ -431,6 +569,7 @@ def main(): elif main_cmd == 'ucinewgame': bestmove_value = None engine_played_move = 0 + new_game() elif main_cmd == "position": setup_gamestate(cmd_list) elif main_cmd == "setoption": diff --git a/setup.py b/setup.py deleted file mode 100644 index 7ce3dc89..00000000 --- a/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -@file: setup.py -Created on 02.11.18 -@project: CrazyAra -@author: queensgambit - -Setup scripting for creating Cython binaries -""" - -from distutils.core import setup -from Cython.Build import cythonize - -setup( - ext_modules = cythonize(["DeepCrazyhouse/src/domain/agent/player/MCTSAgent.pyx", - "DeepCrazyhouse/src/domain/agent/player/Node.pyx"], annotate=True) -) From a33c53e1632196da02259480e2014c07f12db59e Mon Sep 17 00:00:00 2001 From: QueensGambit Date: Sat, 1 Dec 2018 15:25:23 +0100 Subject: [PATCH 2/3] analysis of training data, proper eval for only 1 legal move, enabled go search depth X --- .gitignore | 5 + .../src/domain/agent/player/MCTSAgent.py | 18 +- .../src/preprocessing/PGN2PlanesConverter.py | 31 ++ .../preprocessing/analyze_train_data.ipynb | 476 ++++++++++++++++++ .../preprocessing/convert_pgn_to_planes.ipynb | 2 +- .../plots/crazyara_training_data.pdf | Bin 0 -> 31270 bytes .../plots/crazyara_training_data.png | Bin 0 -> 78044 bytes .../src/samples/MCTS_eval_demo.ipynb | 8 +- README.md | 6 + crazyara.py | 36 +- .../training/crazyara_training_data.png | Bin 0 -> 78044 bytes 11 files changed, 559 insertions(+), 23 deletions(-) create mode 100644 DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb create mode 100644 DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.pdf create mode 100644 DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.png create mode 100644 etc/media/wiki/CrazyAra_v02/training/crazyara_training_data.png diff --git a/.gitignore b/.gitignore index 1a974345..c52dfd29 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,8 @@ main_config.py # avoid pushing log-files generated by uci-communication CrazyAra-log.txt score-log.txt + +# avoid pushing dataset files used for visualization +crazyara_lichess_dataset.pgn +crazyara_lichess_dataset_stats.csv + diff --git a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py index 6bee3a2d..a0e698b1 100644 --- a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py +++ b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py @@ -332,8 +332,10 @@ def _expand_root_node_single_move(self, state, legal_moves): :return: """ - # set value 0 as a dummy value - value = 0 + # request the value prediction for the current position + state_planes = state.get_state_planes() + [value, _] = self.nets[0].predict_single(state_planes) + # we can create the move probability vector without the NN this time p_vec_small = np.array([1], np.float32) # create a new root node @@ -382,6 +384,10 @@ def _expand_root_node_single_move(self, state, legal_moves): # connect the child to the root self.root_node.child_nodes[0] = child_node + # assign the value of the root node as the q-value for the child + # here we must invert the invert the value because it's the value prediction of the next state + self.root_node.q[0] = -value + def _run_mcts_search(self, state): """ Runs a new or continues the mcts on the current search tree. @@ -785,3 +791,11 @@ def update_movetime(self, time_ms_per_move): :return: """ self.movetime_ms = time_ms_per_move + + def set_max_search_depth(self, max_search_depth: int): + """ + Assigns a new maximum search depth for the next search + :param max_search_depth: Specifier of the search depth + :return: + """ + self.max_search_depth = max_search_depth diff --git a/DeepCrazyhouse/src/preprocessing/PGN2PlanesConverter.py b/DeepCrazyhouse/src/preprocessing/PGN2PlanesConverter.py index c579a6ef..0d726d96 100644 --- a/DeepCrazyhouse/src/preprocessing/PGN2PlanesConverter.py +++ b/DeepCrazyhouse/src/preprocessing/PGN2PlanesConverter.py @@ -261,6 +261,37 @@ def _filter_pgn_thread(self, queue, pgn): queue.put(batch_black_won) queue.put(batch_draw) + def filter_all_pgns(self): + """ + Filters out all games based on the given conditions in the constructor and returns all games in + :return: lst_all_pgn_sel: List of selected games in String-IO format + lst_nb_games_sel: Number of selected games for each pgn file + lst_batch_white_won: Number of white wins in each pgn file + lst_black_won: Number of black wins in each pgn file + lst_draw_won: Number of draws in each pgn file + """ + + total_games_exported = 0 + + lst_all_pgn_sel = [] + lst_nb_games_sel = [] + lst_batch_white_won = [] + lst_batch_black_won = [] + lst_batch_draw = [] + + pgns = os.listdir(self._import_dir) + for pgn_name in pgns: + self._pgn_name = pgn_name + all_pgn_sel, nb_games_sel, batch_white_won, batch_black_won, batch_draw = self.filter_pgn() + lst_all_pgn_sel.append(all_pgn_sel) + lst_nb_games_sel.append(nb_games_sel) + lst_batch_white_won.append(batch_white_won) + lst_batch_black_won.append(batch_black_won) + lst_batch_draw.append(batch_draw) + + return lst_all_pgn_sel, lst_nb_games_sel, lst_batch_white_won, lst_batch_black_won, lst_batch_draw + + def convert_all_pgns_to_planes(self): """ Master function which calls convert_pgn_to_planes() for all available pgns in the import directory diff --git a/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb b/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb new file mode 100644 index 00000000..a5550261 --- /dev/null +++ b/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb @@ -0,0 +1,476 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CrazyAra\n", + "\n", + "## Data Analysis of the Training Data\n", + "\n", + "* file: analyze_train_data.ipynb\n", + "* brief: Filterts out the used games of lichess crazyhouse dataset and does some analysis on it.\n", + "\n", + "* author: QueensGambit\n", + "* contact: johannes.czech@stud.tu-darmstadt.de\n", + "* version: 2018-11-28 initial version\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys, os\n", + "sys.path.insert(0,'../../../')\n", + "import os\n", + "import sys\n", + "from DeepCrazyhouse.src.preprocessing.PGN2PlanesConverter import PGN2PlanesConverter\n", + "from DeepCrazyhouse.src.runtime.ColorLogger import enable_color_logging\n", + "from DeepCrazyhouse.src.preprocessing.dataset_loader import load_pgn_dataset\n", + "import logging\n", + "from io import StringIO\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import chess.pgn\n", + "import pandas as pd\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "plt.style.use('seaborn-whitegrid')\n", + "enable_color_logging()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Settings\n", + "_same as_ `convert_pgn_to_planes.ipynb`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "min_elo_both = 2000\n", + "nb_games_per_file = 1000" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "s_idcs, x, yv, yp, pgn_dataset = load_pgn_dataset()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pgn_dataset.tree()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "converter = PGN2PlanesConverter(limit_nb_games_to_analyze=0, nb_games_per_file=nb_games_per_file,\n", + " max_nb_files=0, min_elo_both=min_elo_both, termination_conditions=[\"Normal\"], log_lvl=logging.DEBUG,\n", + " compression='lz4', clevel=5, dataset_type='train')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lst_all_pgn_sel, lst_nb_games_sel, lst_batch_white_won, lst_batch_black_won, lst_batch_draw = converter.filter_all_pgns()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sum(lst_nb_games_sel)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file = open('crazyara_lichess_dataset.pgn', mode='w')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for pgn_sel in lst_all_pgn_sel:\n", + " for pgn in pgn_sel:\n", + " file.writelines(pgn.readlines())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pgn = open('crazyara_lichess_dataset.pgn')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "columns=['Event','Site','Date','Round','White','Black','Result', 'WhiteElo', 'BlackElo', 'WhiteRatingDiff', 'BlackRatingDiff', 'Termination', 'TimeControl', 'UTCDate', 'UTCTime', 'Variant']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nb_games" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(lst_all_pgn_sel[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fill the pandas dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this list contains the full data of the pandas table\n", + "data = []\n", + "\n", + "# read the first game\n", + "game = chess.pgn.read_game(pgn)\n", + "\n", + "\n", + "# read in all games one by one\n", + "for offset, headers in chess.pgn.scan_headers(pgn):\n", + "#while game is not None:\n", + " row = []\n", + " # iterate over all collumns\n", + " for colname in columns:\n", + " # fill one row of data\n", + " try:\n", + " row.append(headers[colname])\n", + " except KeyError:\n", + " # add empty value if entry is missing\n", + " row.append([])\n", + " print(headers)\n", + " # add the row to the full table content\n", + " data.append(row)\n", + " # read in the next game\n", + " #game = chess.pgn.read_game_h(pgn)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pgn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(data, columns=columns)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Export the dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_csv('crazyara_lichess_dataset_stats.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame.from_csv('crazyara_lichess_dataset_stats.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['White'].value_counts()[:10][::-1].plot('barh')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_white = df['White'].value_counts().reset_index().rename(columns={'index': 'Name', 0: 'White'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_black = df['Black'].value_counts().reset_index().rename(columns={'index': 'Name', 0: 'Black'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "elo = np.concatenate((df['WhiteElo'].values, df['BlackElo'].values))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "elo.astype(np.float)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(elo[-5000:])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def example_plot(ax, fontsize=12):\n", + " ax.plot([1, 2])\n", + " ax.locator_params(nbins=3)\n", + " ax.set_xlabel('x-label', fontsize=fontsize)\n", + " ax.set_ylabel('y-label', fontsize=fontsize)\n", + " ax.set_title('Title', fontsize=fontsize)\n", + " \n", + "plt.close('all')\n", + "fig = plt.figure(figsize=(10*1.5,8*1.5))\n", + "\n", + "ax1 = plt.subplot(211)\n", + "ax2 = plt.subplot(425)\n", + "ax3 = plt.subplot(224)\n", + "ax4 = plt.subplot(427)\n", + "\n", + "top_x = 20\n", + "cum_perc = df_full.value_counts()[:top_x].sum() / len(df) * 100\n", + "\n", + "plt.suptitle(\"CrazyAra's Traing Data\\n569,537 Games total (%.2f\" % cum_perc + \"% \" + \"by %d players)\" % top_x, y=1.05, size=20)\n", + "\n", + "#ax = (df_full.value_counts()[:20][::-1] / len(df) * 100).plot('barh', title=\"CrazyAra's Traing Data\")\n", + "df_full = pd.concat([df['White'], df['Black']])\n", + "ax = (df_full.value_counts()[:top_x][::-1]).plot('barh', title=\"\\nTop %d Active Crazyhouse-Players with Matches >= 2,000 elo for both Players\\nfrom January 2016 to June 2018 (database.lichess.org/)\" % top_x, ax=ax1)\n", + "ax.set_xlabel(\"Number of Games\")\n", + "#ax.set_ylabel(\"Crazyhouse Players on lichess.org\")\n", + "\n", + "ax2.hist(elo[-5000000:])\n", + "ax2.axvline(x=elo.mean(), linewidth=2, color='lightblue')\n", + "ax2.text(elo.mean() + elo.mean()*.02,5000000 / 20, \"mean=%.2f\" % elo.mean(), fontsize=12)\n", + "ax2.set_title(\"Elo Rating\")\n", + "ax2.set_xlabel(\"Rating\")\n", + "\n", + "#example_plot(ax1)\n", + "#example_plot(ax2)\n", + "#example_plot(ax3)\n", + "\n", + "df['TimeControl'].value_counts()[:15][::-1].plot('barh', title='Time Control', ax=ax3)\n", + "ax3.set_xlabel(\"Number of Games\")\n", + "\n", + "df['Result'].value_counts()[::-1].plot('barh', ax=ax4)\n", + "ax4.set_title('Game Results')\n", + "ax4.set_xlabel(\"Number of Games\")\n", + "\n", + "plt.tight_layout()\n", + "\n", + "plt.savefig(\"plots/crazyara_training_data.png\", bbox_inches='tight')\n", + "plt.savefig(\"plots/crazyara_training_data.pdf\", bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_full = pd.concat([df['White'], df['Black']])\n", + "ax = (df_full.value_counts()[:20][::-1]).plot('barh', title=\"CrazyAra's Traing Data\")\n", + "ax.set_xlabel(\"Number of Games\")\n", + "ax.set_ylabel(\"Crazyhouse Players on lichess.org\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.array(df['WhiteElo'].values, np.int).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.array(df['WhiteElo'].values, np.int).std()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/DeepCrazyhouse/src/preprocessing/convert_pgn_to_planes.ipynb b/DeepCrazyhouse/src/preprocessing/convert_pgn_to_planes.ipynb index 09052187..70da7c4d 100644 --- a/DeepCrazyhouse/src/preprocessing/convert_pgn_to_planes.ipynb +++ b/DeepCrazyhouse/src/preprocessing/convert_pgn_to_planes.ipynb @@ -9,7 +9,7 @@ "\n", "## Conversion of PGN files to Image-Plane Representation\n", "\n", - "* file: load_pgn_parallel.ipynb\n", + "* file: convert_pgn_to_planes.ipynb\n", "* brief: Loads in a png-file from the lichess crazyhouse dataset and converts it to plane representation. The plane representations can later be used by a convolutional neural network.\n", "\n", "* author: QueensGambit\n", diff --git a/DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.pdf b/DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c4fe2fcc3c80432cb851c34e9ae95acf99bfffcc GIT binary patch literal 31270 zcma%ib980TwsqLCosOM!Y};0MY}*~%wr$(CI!VX2ZTm~#`|kK||6Y$b`mb~L*}JM% z&9!#bT(j08mK7AHqN8SnBraRQD{XF82$7nz}6Cv z>DMJBjjXMeo`Qirp4!Jag2EsD8#vkH(TG`mjLH8$p925$DTb$kN26e@qi`_HQujj z3Mk;w2pW7h(KC=2`55}+O3_xw(#~4P*1%HF`9CxN_wyeE(MTESo9OUcIeql|YcM)m z7J6z{W_l((Hd;1nMy6liKE{*%)!hC+=*b({SvlD1eZct(uaD{f8_r({{}-0OO#6@R z!Y1Y)aN*Gin}6UbXrO1M{{g?Gfu)hXF&^D7FUkLEXlHM0pko2)ls2ZZZ2Orl#(iID zW@O;|pkyxq;(Xv@P==b-%}?`_8&+N1hI?;|>4qiZNddXjTesH?LAf%?5cy|$`FIQe zenNWhSb!@Zao&n&=iQe1^iz+4`R6mwmiPX58jgAQz{>eAp-PEgjA~hkAA^+IoxWNy z;a@p&t&wZBtCVY&XShQl%s-4eesLR0mbX3OmnIp+QaABYU<~FV*wF0i-%9lz%>{>O z2zZ-JM>vjmpC=*86K#99JiA>d^ z)-?BcIJ;jBJ$6+|J6t$AFYR=AZ?fpYM9Vg`?02Yv=~~D3XnA_QK3mMhZn7WODmP`4 z1vVog);o=!yiP2wJegm;&TQr3vZ9jYbMM#3EjSBhLX~ow+5zPzcFzncUL!rX1K&+U z%&E6K!0*#-8)Hk81oD)zP0qCTEAQk{$lJ-S5%3R_U0Um^y!4dTYs3NKMcI!Ofd7h5{Hb-LGE=7Q%Wj7C2iGSv$%lNGpa9t2XVX8K*RXs001v(xEy4{rGjWz?~ABIurZ z1XF(z3SEzala_+A;n0Abmn0?Ydfmmohih$XlLtfTCwbu$6x}gZU_wrB9RMUzA|yM1 zL8$ZPhfWPoC%enLt)tsANHTpyUBM|${w^d*Z%#?Th|E!lK_($pY= zS>&q`@3$Vc%axUeUxBiL%O8N$C7ohwp^Cq4o8x(A6di&2ow;K*$`IL#pfiUAcuy&0)I4R7^{7z zFcflK$g@5otVsxD0Jv!qzHsocl4}%k^qigB*;y^-RJKDSG=z7>REG|t4Tw>`W+t#j zHY5oz<;>$Kn%BWVkL$#-<4|Wbei9>j=sSH(a3>%;FaL=vzp$TbT6E;%Ap#vYw$CH| zPG?+0G@$coo_}_Ezv!_`w(RmNhP}lSo+2(U4p~B{UlAQVj2d z!}!@h#9qdl>yc*=Y(=k{-TpkwcKk7Dp9BqxYjOv)nLdoe=bPwMc?l`7l?Hbumd?MN z+tPp44}upM#m(H|w8^H4YE83Y91S5`LoU_;slm8>F!7x-nw7r*6@egyIQSm&9=4!c zMGaXim^zh4#g7x&*2Im@tk%JsXw9q*PcMW?v0o`?Wbbo~ST@M|fuRDKHVIUdk~Oqw zn0c{ub^JliVvdDca68rX?g0)B@rnd1hvc#H(3b{84d!Ka9Ro-eSfk>rfwD@Wo_0UF z(T()lnh{71W;LR3D7lCW)OwfrvjIjQW+2KiYYXWy#v6P)$a%cFQ@wamMsmxQc>xcZ zAocz*5>*9yE2->3+FKtUAooJq5~K#%MP{g;mp?Y;nR4H6`N z3B$V%F;?Gb?~$Nyb1%^|^rud>f~lfE5H@xTF4fqO16hG$ReQ(_dr+ReBDbG3clI2a zQxBrDgVaYPu`c=5XEc>^&Agk$n(nu5&lT4V&v*TATZvNlfvZoMxNGKC z6)fv7N}X?2iyDsSGqYvvWLp3fTlo}tAH7Ec!Urme>wOIwA3jUMgn)3#n1 z!eO%+8fg2le$Ba+-&qwqEDE4n%&aFj6cUEROn*Lz&m9-`% zB;F7sC!84TXcURMm6;_xBWBN6&j_MNweYu_pEDCA zB@T-p-SvW1F{2>$JM18E^AiNuF|`AP!RDkhF(_8$cfF0h%*TRp+|0%Cr^wp*fMynm?`kEM)dub`0$MD(i!qJ zOPIu<15{FUM=vB$M3*2f@u9E)EmGhT3k{@@IomzS{4^Ux@`bS$MBkcphPVSf#HiUz zcSGcKSZ)egBuSS!__*cJuca@M@1uom*_;!|z(r)p7P#Gkx&`%R-2dFjL*5CO(3F`b z4C+JW(~-BooLP$#RTjQE8deJ5<9{oU1ma*M~p~YmqshQu=?&G@+ zg14%}-7|vc6amqLcf+6x*+Aig2y5&gVgZ(oY~$^@>5-#^k7cWpMBk@BTI{Dr;N+dv zOftfA=FwD(wwvt+C*ZjSCGzl=_nO#ESI z_Zc33*kDA5&FTQlcT4ZZrCCMm6?Ul)#CrdJ--0$9I=@wMB%9}DlIM0=`dr*?w8ODx zvWX-NHvaOeyB8r2Z4Ew?BxJI|H1P5ok?ZG7>lPf+t)5lu;aVk5+vc|9{kY@d$d>{x zb&rueu9|DL@>2Ef$wdMne2<88WG!#$AFQNEC87NYhUo?Fq+>4TM{Ig`e4+$m8aWE` z8lLL!*aHbM z)Gq^@fHO(%tcOT)j)&szD6-~(x|WZg7~ORg`bpRxa&2~jc@QddWW=qF3rQ5It#?LTiKZ3JnrD*&Ib~m}ma`t}rl1~@F)#pP~ z&+9tCjc6vV@7=_qEfu0&vGkEawC(}QkwsZ_a5mI?$(5o-%*UxnAe`iOYY?G;x)!j9 z--E{)r<(`*dsvRAyMZ#M3OHCDlqdfgAo7`87h@HVGi*L0a2aahosNKVlHV#v%cunPhp^B2 zkLvGo@*D`j@78mmUF6M3EZ@~+z(G9d0kFV1)VTd@CmM4TEMz>_A4V2|XwxeWq{|Ft zS)Y%~DP@mCxQG*os2bbR)}R_9nzqRv7vy5^{hA>3Y!4{Iet`dZt%JrIJCKpe9mDP! zz5YrMcY;q4 zgppCySUusT2n3KCNxu(eb56ZeR?>s4C%iR{P6eQEq{LU-xD9$S_E4iFh z=EdS|N=?2ZUmAp&5+F6mc)f*IY9Ydt_uOh#F*LN7WAt zw=5azAv^pe9|Yt+D4H|Vj#NxBdxYrdQS-o9fnx}kzWf_}CcIOnbw>^$^DxDH@q|C( zuFvVp4~HbOEiAXp^gdVrYn0*cfbSL|qKu#))>PxsiA+qZ8&d~Ft`M>#J5^qQ{3Ks4 zODp0>6OXf^6}F8@McW-aR$Y!RL^wy$EH93im)HC?rubYkrh(CLYdCRx;yG#e@Z?ag&nZf}gVKSm?Z^GrNHtiv?om-sMB##^oGtMGG7P#2?~zLkYL zP`9a@iQm9K9vJ&NZeON;IY`&A-bA`POq{ByhcEQ8p&DR5YHk2jzZjt}S#j&luBU>y zO;&ergR&{tmB&((`=qdnlddMwbT&a1y}EF)0eiT#{>#t>F#dXNSXlN zBkl=gkYi)4OQQ!54Sfh9>{G3>oe0^`U%3mm(Z>g?pUF@D>4e2jyHsgHWJ zZt>??tT2E2-uO0ueD-MRZgbo6yzSD-{l%9omA$E_hd5W@CiBSZ5i(#8_9HXvJZr`6 z>=CcOp;?u2V^FxaY+5&Ib2+pseFsCkuHOG8YW!2gFS=hM z)L#iTENs82G)$4r2*7mEz;<7~5g!yv2603jUkd}=Z1JLfK|4zVx*kdpnMl{zYy5k- zbjc-==O;2q)j|(2?_Q=ondUE=r5m2j?-=n91#50_G@*(CD|uBJAnB1SH^pRP@(4;d zCWp?XdZJ_3vK*ry1jWp^{N?iz!yL9=5O_kc17FO{xiu&=D+Zi3Qpred?Uc4jVA0sw zVg;MLQOxD!jCSs7f+1#M*U)!KpS4`W$I;9N`HcSn$S=k1Z|2TO%kr z_Uj>;FNlCYEgu(?uJl9dnhBH)y@>XwK!makF(QxTw0S zm~Vl*W){^g4H6{G+J^G&+&;6;qL}nNl>-mW&op49K5+AdwU{hp&C$| zZZDtjFyYxoe6yAxC!1suB9EBzim!kK`%{~NSjat7kxuYeKDpellHc^nkq%}S5-=QV zL9{M&^A)m%YUTY?H0bNoJMdR!)jyLL>uNQDk%3uApit)4eTgGO==dV%MdPAmO1@oE zOs!wlD7}nK^0T&n0?}iuwm_PF?dyBz%lY*)?mKqe(;vM1-*WE%B|*|NG5qFTCOI)n z5Payi>kt>8aI_6B?A@8`ce5WPg!R^&0ji@ZNIn`8)FNl$xY(dn$eF^|==8wTD((W~ z-3(L6nBK_WZ}8>@Dsa0AyW6D~d}u(KGB!z+tJ34>In*;JTdd*{Jpzl}sTaa{|d23vR2C%l(hI=B!{&%P8bp+4m8 z7dRW%mPFv6?Kn+sWjd5bIBm#DlvrMirGq7$)wVrmuoud0MBoX?--N2+5Qe1ZSS}P9 zw4o46@0}Ja#$*GY!aOye(&Uk7I5fd7!-=rRmUuHueR~BSkk+{U1B!nu%lThG+a*(s<*c6Tr>TCpgfP5-x1^#84l|6i{C$C@P-HUi^C3oUS-+(XF7_h^sX zM}Fc-98R&J%QHO3U(B)NHWfYP4`%)MiNpV879+#&UXfZU%@P?f^b!Fgc$qvy3cdkFI@6)3Y ziGv{f(6k~gGlr{HnLUSn)>$~%K3+$)kVh4;B5@rWmK4q0iw>;-qC*w~FkqMf1A_5r#X#K028EBDY( zwWY1!_|%~^3Qq+O>LOLA!{f!3aS_?kna>f?!P;>E#D{&#o4)L-CtY|zHhZ*#oIOw4 zytzo0%Rumk;yAz(GVRSSt4T^r7)ILKW`efbO1#6XYt{%O@SE5 z&J)*0u-7B3_HjLpeAV(M%g#vfr?;WG%y^$0M3H>^K)rxukJnqB*23JlII7pc*fOM#9$-qFOB*r|f*?X0 zLoEeZMf=qnRS?w>#?JsC9(d9Oz+w2a+#ihkbz=25M$yy$z6eUm@H$#c0)z_4)dReZCtJG zP}g8#5_E>WPe`+td6?;(;k1-KC?w@iAqmKnmT)g=^%TA59=yL|$=L;~PZygmC#YXoYek96X)gb=@RzD+uB_R;|$pdr-$2TXpQ==}{QtgOE+X+q)6FkQUh zAa*5LAQ47z_j+tK&oX0N2}s`=r)qeOHfPwuY3U!$k*SGsnx@9bmxeHCF~Vp8}BBw8TE z0^x!GM3S}#br2B5jx|O~Dna$=m=|=EDQho)sPFBLbv__~ZzDDSbl?Kl+@K<~sfpJ&ZNV;QnLojlNKyT1MVlI0p%(|L;VlS?sjgbZPZ-FZKSy{r|FGEy7}yYEqwq*Lqbu20clzQZg2x_y+3dNS~QgTsWkEyBS-7e92=L*bxhObD_>@OfrsVzwG{rS&K4y zT(w@BjdQ}@5p9S0Ou!Djj=x5Yl($VSrRm{3f+MK+E;tHk5Ft-{OtcgLfpfn*@i|og9))r@-n7nk32xv!Pr|Mp%<+`SY z8@Y~qE4=;40psd>8$}tAzEtRO^3R2_Az~cq@ba+i z8LXbSDp<@n7)ukwj8oa%5?jQ(5+UUo)=*^X)y}?HNTca=e5PMAaP|Vk zSQv_ntwwLY4<`3!g{v9YSat$uaBESSVs<-j^K`szx!y3FwuB4HI9k7w?qbP9Av%${ z+*qu@wz%S|_a8`ImaX&bxXPaX;Vt}I=0*RH4Mi$iBBl=yTHrpdM+D#n@aMEb?NP`3 z_lyJaS;`<3(1Di?z+?a4d_VzyvHwC9yg<4is$Lx4Wdybb$KmqDD6N%$3|%NBuhvIqc#cJ>fb0eB#>n z#`@SKSMbm83fsc5|-c`I!u9=PRwwSmgru}Yv?r&;G~ zZfU2%ySr6;S^MaHgQiJG#;eS6pnK zOtSfY8VJ$#hF!NxreGTXiYn)k5~I%LHxJJF1x`4gl@Pl^H%wg8&CSN9&Kgq1P0L2A zhGjsl`1B|Y9{xM_PZLs_5t-v>9Pgb}c!w57#lf#9L_rxEjhq(a1_9t&mi1mcH(yG0 z$smVvytjDJkl3xpMryN6`x}Bt0vNvLp*H3~Qu>uBut%gw3_*g6_RF*FK1N(*2*8;F zSh`=fzT6;&=>%?7$&;e*uJM?Fja;Zhuz;ytVQslUjTozjIO zN5XhglHv48T9UzN$}@QtTTKjRL;9d7BgE8UsLMw)FIzJg=c|6UJXbE`B{{60R$~eV z_{f^0v}RO;C7~9l52$gbKO)UkjLX0mKU?OmV%7Ur{-Yn`$<+N+#`VH@RCC5ZhYQqt zvV7V3Di_i`)O`yCOjtL+@J83RMavh(4TvRV89tfExAb{h6@Qu)3-y(XUCK2syE6Ks zkHj&BlQ1`&2#IrlQgkOsLDjTGzM~c36s^hB>QM{2rLmkXgF32CH!da4O1a;7zPUviplQ@GT&x|G}2O<-bfUzf(=7 zI5Asnx{vjW8sHnwr?i2=x@Rib-FOeGS`II~-&@*9z}E#k3&e5GyTMM~o1f>2 z^Itc%RTm*fs?ICfEN=5%Z`RFfi@p5VQO_EB2x3pUsD?V-&Kd7(SVP@qV4KdbRU^h$ zEe`qNY^M@IgANm2r5C%!aBK-r_QMO_BYCA|uVctz1^jHS7QCxBQ?#iYOQs-L%xnu; zvtz^hmq=-ggrFQ%|P&#R|kkO4ZKTtQcV3~ud|nX-W>mJ z<>VV)sIpwa<`3l*qi$-gOQ`vk=Nd5^Pf#{=5GuEaL6;EPm5vl)`b=V^jB%kQc=&!V zDI!;@(#?oFH`R^A!ZAN`oS+R3ocaCg>M?NRXS7-vQ17%+K0|Z3qpw)AWO`qADbL+M z?GB}>MIufT0GvDV}yp&emn!ij$*iT ztiP+na^+KzuFwvGAz%5_ptAF#dF=lDM7$Aq50<51TIyNIU8i9g$6h{gKfmr77qH^~ zhba8pgolxy`8P@-6)R;)NeAwCZwIM`7xP%Vx}Qu|n)qzBu5?ou*N={=M^=(l!nCK% z)=?pspywKH2fDMglWk9uLC^@~& zFXHlBrW$-T>8%4*I&&ol8NXlao0Gx*<=!FrwaW<%{5e2`N$p_&)C~#2@y*U6bnueX z+J+q3l`SwJwV7(HTEV<-Ym&eYnxGiW;{BJOrZfTvMIcuzJp}rPvs=Qamt-R_Gwgn^ z4QNR6F#f)0in7#f@HjD^Mo8MG5Zv3*ALmVN?$3R37VE>#adIj%N)f%!`2 zA%s7?A%k}U3TdUu{I%elxM-C7E?DX$)z%vQt?@GW(zJNUq229gHAki#KI*Sh974FM z;o(2y9FN_toz#~k=R-0!TUlmRA~Y8OXryVqWUH#Etc-QK0=dBvHPlDqhm0|!%WIG) zf$Wjo>L;i^y{C!9-5>Ascr=u3{lVpbo6j+@{VwA&$xFoO(Sa}TBe(FDK!bb!jIGeT zUGe0I(Yhy~8~=3UjROoU*#_4XR1se3B!s|*_QoYA<>$f1>;qg4m8ldJr1_Mu`I!z1 zVT7jCI${d-dufPr`SEDiOyk6!?DMI>{QjqSzlxC4obl2EUXiQ>_Feu{u&rx)8Q^y^ zOtcgQ7QU)rfx@8F8fEVcZ8G8&1=&SY@;Uz_pV%*OYl0Ax)OjvPw5z9h(wkKNXgyng zg*XThnstXeOxbP4ZIX45Qqxg>?&ek{eS>w3pv3F>b&|4}uE#C7(c+t|vLXd)R-l~B zaVerftoOtO3Qm1~{y6+G!PsqYGIdUV3-yY5rY%z`Vf$itO}Wz1_m7g@(Ss8y&T{J} z2=$|KG^Et9^`M;24nte{G_y7V#z#Bu(Rc+(()%Oiw?(HW&z+1ZxVP?W;yi^)=zs-T^7=HIDS0o+mM+yr3+}4B33ltNz!{xd}asPfNS?7w9fYJ4k zBunX^1h&6HwY_;v~218_oJTO?YNS_By58bZ*J zw?h;GgZvlG4u1Lmkv{PiiJ6@aH}gk4#?atBi!%wcX}Qli)MgUYf_5RWdPwQ3BOy#1 zf>~>1Vp;1I2gOPS0ruSZYg-2V5!)ks5Cq z=3LS4mODfHM08ix*UDjz(oIzMeOd1rK)09QAz-TSU`V=)C)b@N!$`yU^x)_$a=U5M z7Q4!Z*(}f~2h+BOs8wFMA-bTg>3qy5A0WWyae7<7jGa|UjJ;*d8nzSCNay~fSLAbT z;HAK{n81LfZdM47^O)k}$mjt!W=*EX0?F@+vr+4_QYPpZlt%a|Lf8-&;mc{gi=G2M ziSHZCH(f+hY21v*O)lsQzks86h8BgiCXoG@J+mj#sps0Z*}*Xwic~A^X#<0o>fQq$ z8t!|G?fS!9TW%k&3xamZr@u&8m9$^KYbClP^z(|x{(&l$SpLHUN0ux4{v;|zA27r- zJJcO2mC~6q54rYuV%+o78?X>!)$!#@ZbSB#h?Asw{Py#u5so)@bLs|-_( z?OEsP{cJcugIFl?JFkPx=WjY%gG;^ajjxx3wN$1{`{r&mj73*xF>U zFOnYdbD6U`N4~3-*X$l8>>vi@hQF^>z{{*lKmAFF+iw9Wax?CgvyszTA!NXO zUV1;edb+Q0zqtCX&|=&5`p394Wk&o9C5`Tu8MAy`zLsG~?)E+s+k2wsYgl!n+bS;M zqbkxPHOW${%JYUZ9X7QJ3Ecc82sIUaP6M4#qL7xGFuqL$Hdf_H!|s+5aymj#QUdti zPznk^P^@N9IGW^^u-eE6+pI*^4PkU&WAmlrX-2iBAhG?!koYsRRT6S7Xa_=^HYrojiS zYNQ9*EUjIfRod9w3SB?RQ1XlXJ0vrJs12!WKH>Rux(}OXPY4TvnrmgxNWk>SZ4nGrzB?eC4OV4WPmn3Cg-k^q*; zuk9pE@nK^GL4Dnk7sd+}Ss^Y)ygVqpq~?1(I6hCSlF0(o^4K4MoM;vndW3DkSL~XK z(vt6rT$nr+qAt!HGfl|@(%JH?ddHaFRp+#0l@Nvsi|fexqzdvfKsqOQ>54Kd*Sli_ z9+aG(0)!Y?0G4{6$7*{#!^OIAsV+~R* zk{kA}+;s9oOyQ0$#QxEQW&k$;I-)QC7UxHKUM)q5aYFw^u0{fFYuW%)H5-8BH*`!A zBUdok&(xP*00a*b!@si9928c>KfFADt6Z`CzSl9-)5?8hrPYeGjJ2|~Ow6G4Fv#-I z&0Yi~0jQA8+Mi^}0-;DC3;>Dkv`f06qX4ycg-K|Iz)?jJD<}fLzyN%mLqq%D)_mxp zf0rZE)BUce3(3)a|F!kHLt+C~0X}mc9>=YDmgAYcM~U)jOZ3?aWQEd<_}2N;V5fMa zK=F)$+%G)Cm3Y8OzaTZfisII2A;v6#=|nIfNZ5qOg3ARIP}X`RPD@?7g#>?_|N5c*pcfsyWa89^#mtm5;pb4zzf&8}a&RHScN+OyTQ_#0pi zEt!HM3{k7=QWfSM`&G@#M5O6{$ZWBd#aY}*eUCI*RxGhJbN1(-9riq!VQxJT4HOvt23N)o3 z(4hi*UZd2Go-uiPj0@#Hv;F?v2w#S+BPJVBGPm=KF|`QT8AZoR2trRwp}2xjS3XO1 zzL*BAa15AfVdufmW9rE?2!;ifzO=OS@B#k!t%><&Yk3yk{Bf4vD%+Ar=P$eI*+09p zehOXYqBp8!nP^JLX&_flxLpNgJ4ql}LV;HB0p<1X^OZB7UC)%f%dqO$(!Qr9#bHql z7F5G^%(BJRBsjc&vZH>&@Ivd@?hAj|?YJs!-7)rbEsLn7r#;Y^>X)ya(>RNtuS`(4 zth*Cqf&$a?vgScReOoWQ^6+>EZG24I`Gc$fR-~b0{k_~rC`t^m7Y`cb+`~V^i#g$* zH6H2loe73|8O^060E39W6q#;c)J)zrF%oA}2zkHpYqYadUEoMESh7e`|5e+$PqRWL zy~Iq3=PXM5db3L{oaRkY5eW_;c8dOI+bJGd{IiuJVD_6+T924)wrM>}pkfxYR zzq*z!vT0R?wQo;l-{s)YWpz{>=SD{hPmGG;1I29O>@Pr5Ue>%m*|C8Jexj5B{<>L@*g71J9zJFZ zt98u|GCmmI>|hpt5vK=2h;oA|$6jmHM@YtiqOR}j3opF^^42AN3m@Dw^OetHFjaV-I&4 zpWU7ip76nVk<1iKBD+8A3cCPNVnrz?Vb$vOcMgjudg)P2LBi*Tpa+*r_HLn)u-rw1 z^@c-xAafBaPKG6l(?O^%+$c!rNqacSD9N4ws0{k5g zWPwy%=*|il(tkJBKux0x4^V+DG0n#6$+yw{m_J4xmU|L;Vi$N;`UqPM8awqHHW3$Y z*g4|iAME+tOr3$2@%PL&M#8p__TydpI4p+e!B01mR~cLJObz6$pxh}i=3VVcRtOj! zBW(~@`4Op^LcS@%o%k~kYX%iX>t-rCMh>cI}gz@ZXj$vQ*&KR@d zmjmOWXGmnugAAECA+7tkhqbhC8=uFU>|KDRBHk~)ZOpb5rRzH}SfMDS>KIC*=cA=J zY~xwUdKkrw?yRP9*sfYQl+LoM=EQTrJ4k4JiiLSM)bl9F|dZ)Y%gA2YqeLd z^W+kLFZu)Ue=8|4|GtJX(EqA8p=AI=3@9jo+!HSNRdB-FksO@OPaOn^_+!&Yj9(dV z7%Pc_wih60VP*)Z1f-kZt9WQr92g+c6y@t5O!?b1m;U2F&A+J|Q4$d!rFI`jl>}(K z1h_yB8JM>vdvD9tVu^;{SIEA4x`y<5soP=1?1hx6SGiP)`b95@3oc7vQ03g#)Mk$` znmmoc+O$i00h6+rcPud>?4cp=3W8gJY~=+LD$GJD`rCJXVl^Qw7>L^BzZ9L9IM%x5 z))u&ptUlqhgW#`eVKU{o;IV!}&Tu_~9(SQE8=|;j;fOgSSibFvwsJL8Yn zjMW<;7OSPJVm?H3jB#q4&|1*SppQQ#MiQ=H^xTTbyn6)h?Ue)m12lh|NV5F{negsk z^>e`QlA8!#e!O?ZPp0?ps4=3i3#1Ya;TU0?R z-}2Mn^D4xsunK06V&RJ67%Rru%Yt9;CSC!x;XecY!NR{~>x{pXzdy#&f67g|^YAEv z@nF71_r`w$@M(u;|KH;>{VgwNWcz(3<5J{s@x_N;wu6lDTHoV5ohNU9R~>#ij2f!$ z2G_|So6}5wC@DjkJ66;D=G}LmrBZeuE}sxjWd?d?uwPc0s8cH;Jqxo~wM$**L#If6-!j~!D?~k;(OwiJ#j@)9Gy-UvP2kT}B)&;$9|tlo z{>{l04+#|2#b#Z6GvHh)1MUKrWkMZ9ln;DLH>xFu(LKx81mvUYWf{$HUN4LPt8Y8_ z7{jaNjR(Qi5^p;hT^sx<|0hVk^-x!eAk2GMG6ffeG-W>IQCa6sV>{hGK~}orwb`H_ z5(4cx6;n;i%`F3S`8gXs_X-YzM&#^K<)1v36)*U`m1fT{FqVy?tUL(KD!MRIxv)?= z;&H+Em|tEq^K;J)hMy&IUWcdNRpfzd!l-&LAN8NgdMz8TD9zL+@?GPLM{{^6r6r!++Q*GPCF8 zQbBZl#@aUBvKQ}pw3y}G7micE(6AF_)@Eg91FIvp8p-CR&zR>OsV|d<3v@!}4?**{ zsR{%A@BH;c>;L{y&$c{H&g%2H#4YpGB7W9c4sxg~?4s=p|C3Th{-yt`7|&I%45i4% z^g+fI&Nqr`#h0Xq)rX2UoKEU&|AT}J;*Lr|GKSAWNc%_#ZaCy}Kf`k6vwZ?Uq!+Q> zxEkCEg2DBZ&pSfHq?puZZ{r=!33Dfqj&hog8wFLI(HKCvl4Ow(8oOsPs{EUe-hhqJ zb%{Zg`9yyZkjJvaqrs43d}<~?NHpVR!wDwWkeVaFx(i!Ac{P}+_hK$uZ0VX427&Ef z|JB^L8EM+`vmc3JR}6v^8r{JvT`dNC&%LON{;8X5!(plB@$53BJu@BGM9(WmL!3*$ z?x*S=i1X1+1u-W!nE@)yYL%OVAz_qA55h$?$nCO)?CJ95R_GJ>I!InMkSAqdr{dcbmU!hON-eaYoh5Xst>$OV##Q;R-AKL_j)%jUl* zzn>2TODHfygUpF<63thneHN$e?*N*PVtZC?OjwSzBBs^!f4grzW@#DH8e(p@+-A-* z>FT5mj^1n0JT*}mL!FYHCh}7z2{&WfFcz|1GxO>1A>MS;2HUH1+YFR@^7!NXAZN zeSS(F`0254Ed7YR(#8ToI)8A(VW(QT;i%NII4`RyB-NN-74Pn5dSd*v<+*0ER022~ z39T`oQG;9NOM#JEbd3pVscKN-?s#6vns#9dVEbey6)s}>?fSc^Z;>SJ+{OrPbAMLX z(6WHE&Af~!_xO3I*Xo@4AENGWGdKpu-`NjS42LZU-$#ie8z`FfSKotm=ega{dzn}5 z_HJZGE&Q>MN&|z1`T9c>g~5(ja+bNe^*gt~-T23--MBHmN>Z*-gDokmldpTXewLkJ zR_u2TQZv^ZsUj&!EXs8NQfavS^)MUlP#T zPqHe+=xSKPTp}u2qn_cm+IXbt&4i`~hN6aQEssk7nFiss0{s1^FAMB6O^t!n4__({ zpyNOoegBh9Iw=`uXK6zZJtFc`&1GgcluOZt-68(6t6Rd46BXr3E|bRjGy^dGjfJ%? z7kOEz12S?F%zAt`M>dLr&>Dl%ra4IgSeqsC)gDVImq!sinnxLRp;{I%#2d!01Ok0r zG0pLD+*kxZvp@LgZ!;Cv->HvO_{cBnBXBO+4W74&bXV*`@T>);XzJPxr}G&>h`$tH zzsPKpE$d!G*9R~#3@(m!7-exEIHse;c@UUnI+9&^l}6Uo6THcibQ`_<4|e@+#==DR z`yL{tC{gqAx1Z&E@}{m*dLGZ848>(6_v^wj9A6iq(UU%ZOoXdMGhg|cq_zAq}b!c3*;6%oP@0Hb2*to~|{C-Tb;EEVbgx}H%Ef#A@*l#gh z6NPJGtOEOJut`WH;AWY%Wi&KB7kx-kv@btNM?Ao%E$Bjx;uUzN-*gmuTHE(<)$D4wZT z2@OQ$j)B{ZPF9`5BV^Co@CnMO%y#R_3zI@AA$s4l!TLvPgcqTN^9S7iI&1t#2_~1^ zh=l+hbgL^QHGJ38Bg6O?Tk-qOgu2i)qSPEgIRx(r@5Fn+%z#6GTr=@@R1YgdSX?A# zRK38*zUelg&@%fSzvrI0O5#?k1?o28?t*QDRd5x{f^O~#9-!s3ox)d%hdOjLOdB&L z_e1x!2<|h#ycp^%#9@|drg2_7vFheW-ATv_c$PW7Bq5n1f-75HY(FXbp%0mwE$<1j zr#c=uMs7!8fmW`_UeUW}Fd?z#!sOZI(M0u|SA){3${D1ZN60Q&uKn7VYpEIpWpR)JoB76bLPyR{HH zeb*b+19ZH@dNzE^-E>Iz+)zoGzBlh-*L1O4G1ZyxC3(0m#hZL_sDJeK^-D{Er#n=; z2jYLeF$vH+BE9<&b^EpaJ+A_Hy0%^OtY<1Ik4o{FOe3cS3Nv{n3tQ5h17g>e#WSz( zA&S&buiNr-oT4JqocKJY@na%61G{Uu%9vN*s*+XFp5o_(&hXETxh8+|-t-#wx7x5*aG1o%U zy~@7)Vm~3<(8zQ;e%*CX>H>+;uO|Qq1|f-jge%p9^%e5);oD ziwcDQwA@`JZ^9~dzpG$ooyOU1zPm~Vl>L=YitjOwNl(R{D9qZTyGmK$s)lWI=$C2S zRwwIexzpSXQ8y}`XMKG%XbQ5zY^mp;L|5h-SoxRGaz8SK(Kc^?E%IbiE^7~~BInPT zL|^@|am56AuP6L|#C`={jyJ*;d?o1N;=2PpVHt(SzlX46f|Un0E6R&xw}tIoVV%g`$#FTG5E`k$9SDzw4ZHUviYuq|V#uyteb zGTO4sA0ZOVQcLrdTIuo!opq!X4c2kNih9s2vQ78qaG>pBOW(31mej@F*am3eezK%!}ot# zHZGI@O6QX}DY~5(Ix@$@FwBD|Q2g~>B(l~Iui@KZT_}%&L2INFe;=k%@Q znwQ(GKwgrp!gKcO^}}~(?%G_xF)|gzau1TRE+5&ti*IXVdg;B-p&JLLm`_ORmOMD- zp`Fonax~j=v-7A=Z?V#j;ba#pD$F!RhhhVn_@GZ zXh`-x+Li@npM7gz+%PW8B<`M2FX%rArPr8`aun}K=nUTc*mGx;-BbXhi(Iqx@w#G# z`iG2yv)#th(wCclwEup#=dS5h_I;u*KfVlWP|)LezKonMy-UVc%@m&g;s}Ab>F77A z8mA{Gr0|aETl{BP8z%WvvRrJJsnV5BmWXuUb15aL!5bvuC3-HmQ{04%7QULo+Swas zQ`Nw1v^;`L^D}#47-n8EgEI%Obig@$D>0hZ>budvAA@sXcn3b&hf4 zK*5BAGFy3T%CLYTclf!pn>tbypKfs4n9!9{A!HQwY^&H>?w0%`412{Saaj^v551BZ zY6~~YcD}+`eC;Y@^F?XJH*L@P(a69f{w-qNtGaOI@hd_By<2%anoBqLU+m8~kQ7qx zq5XY2_0dUnhSJzv7Vc0DBX=)OJ-1OG1FS*@KOeEYpi}f}3H#4;4+`<8k8toliVOQ% zoGqwe$!_kpmdD)at8pIpZf-X7#QW_6?NfI5g6k8s)5yu0O;21*TP}9#wDd_K2x*fi-l(6>jM=Lb=8XUrv~B&hxeCnDoJ}>!Oz{@ z*?m5|SEl@iOFP!ZknoJT_b_WJHeZas;yJ$z#S zhl5S8Mn6Dn)^ZKT8yqC!I9E|mxp}Ua;5bKm05x>XwkQ1ht>kAGcze}w-97X&vbEAl z9+tRx`FO8Z8|P|>xeBkOoHdJ!-KTYP_R&c#bY9t^bBe+{4|L);4@BojnH+qvx0*4I zrJ_k9r0ta5bL^GS+owJT+tf$x^54nc?KFNw+q;Y@zBR%?t}$DwvKBLtJi$o{G2swFU zn#*LHPyel!-(S^j8_R}n{A9=7_$F(Rj6dyf=2aYK^1iI*$`~Qi{X(0^kz?`p$1d@? zc)Fa(w-GN#iFx>$VF&-Y@WtCRSnWr3x~2Oqwp9~%ZsfF1x%YJJb}rKVT5Zo$Qm*vw zL0oncuJe)h#c^jZJ!}57cAp60$NmG`3WqD5Uwlx)qx@s!^d`KHvGittu2X1ReWr=I!A{5gWSqL0s7Oz~RA$iWqA}58%`{zqhvN0>Kbuy^ z^F_aHyr51lObcz2-AZGOxp`S@Y-otbi&=Yh@8~e*Yo=jUa5LMgpP2_*<$W6R@KjIp zij-zuw@faN{vQEd#vMmezIeB;(g@KC?GCr*`LMC1E?rb>!{_$TPs9ZyU)NV|_piLJ z@~R+}F)5$(MtOzXW9ULrEA1G8nyzvWd)w|`t$GTkZPMzIu8dyOne-TW6LMh%duhS< zf(~}u>mR_GwoN{t8Fd&O`eYUIZy8>ASKa9;#FvwMa^}gVHxSYV+Mee$#j%Q0XNsCDzZ*ML$fR!L-tIGy!ol`%Q7;z(s?Cu@mIKb9I#9n4w$O16@nfC}wAS06g74sR0yFZ2#@S*&BixMk9A9h|&fr19_)&rc`u zS<+6_6`@UT+|}gmNvnCxBK+&>4B~8Vp4}bOcz4o5RG_pKqk=0v`QqQGzNko4s&u+q z4nIw^UYyJ5bycbAR%!o$*FSSFhdoQX@;F(q)z8vF(rf#Ps|Bf3eZilsH{?p_;eNgu zEVJi*p?NdOlj(kKYvhO8yD6EjTykWUWs14dxfGH9h>!%C_;j##Jxvakd@lBhv9nZC zh73Lf1pp zDY2TpfW4P$Sdw65a`Vj()w7>(PVH_izjf~YU|V0yPycI=sty(iUe0cie(>>&@uA0RHf&)Q<$^^ zYsWVV53X~h`KZf%jjC&~9^>QEQS6FlB>nbau!oer8_nYgt6tn6@O!xY&_Tn%d6!zhoyfZvIw((cgO;&@i(5-#VsFMb? z3(w;c2j1zErR z<_hO)lB`k|RuQO1F>+zXBNFb{)3$GqCGH@aOxZ&?NeVKvH39U1RiQ{AslO9t&{j#IK{R;BO z@tP_c*TyAEFoKtM8iwM$ zlosIVdAZ20DOxS;%F4$t zX(tD`49|pK7~&0S(Ze#P7NSD5_}@S7jPx52KwVW775pcSW>{NxdZSu~YK&)`Y#P_j zrj$=%qZux$I>}LwLn&JWSnNcQa6NFHLnO75e(7qcJgIBc>7C z2cpN7iQi8Psj}2rnBNP*MltluT<<(<$#^Zv_hj^RiMm5|)G3l&g@9~S+%w>8h>_t3hmb84k^@dRJFlpX3~qb?C<@!li4=tKP9 z_i7A|9y4~nuW^5u#hD{*Ef+(h#I8$32iJ@`HEZ2drX=^U=@L+6&I@hdWRTr%~^J!;lO-g2Y=I*S|U3$M_ z<Kttw|z&4r4=&RXE1k^JZT&@5=*c@A3X%-lUatPQ4B?f|Qi+TAF5so4 z48Z0a>|qQJXNb983{Km@=!02uaO6lZH)C{~`C);i7;L&|46@+|6(PsNV&@hP+z3L5zyd3l8WS%<-R z?PJn&hl-aDWveceuAQ_AHYhfRHf{-35#k#&0pDUcl)g^UmQ}pvm{Zj_FN? zC;S;sM@q9SQ^zZvTHrB@dl6AbEgtMFtNOtyTaQcom6@5UJ2uvRzFm5yv~v<=NII0ahvPYcJj^cd%HSx{o1;Ip%A7CPP09C>Q}XDt zw{9!=B~WDx4tMhFyC%W$KGLx4@(uew`}c%4#y!zSFyn_?M_^>U<_a>){;yt4df|)x$gBRO?y@W0Z+a& zjRHZ^A-3z$;X6<=nijXjL`Nlz>nhW`#fTq=ywsK2sJsr_PhTyh8HD_Dx^~ESpQ#zo zhGqC$sZorh+fYT+nS0qXjoR0(&%Sna@80tKXS3!a_FWPG#LM-VJiOG+L8eFf(tzAZ)4Dz->0zA;BGCo(s2yXTD< zw=bJrjVE_`wsiZxl&Fn++wW8Lg-`v;S+72o<6BUCX6}VEn_V}^>~gqTON-g;+A4cj zJGrvv{X`Ox>Ne=~Bsh*KOW(w+UaoI@z0|w7D6TAtJ?x=T8_y4JIlq}pej_tU)Jn## zly_v{l~iJiyhEyE*3*#b{;O_*FTa1tCU8xpmP}_GPd1$Sv?0h`fl+kVel;~KjC$VB zmXRvPf-OorE4Ot0usO?Hu6Qs=JR??b_Y>Uty-dINwNC$zQ~6!1X6Crer_|tBk(FsN z>BS3ihuUS>PlBW0y7%Sx3$gaIZW=Lq{mUeolq>b&D+``G9i?IUHLRvW-#pP{*r~pt zKrL!h`EE?0rFX#z$WR?^-jlNb_03g#E4JKOrukQDvE#{$?Weuqh&AxeNMQz8?Ampy z;5*9kt20AMJw7Pm55jR>@qxCj?E9y@gv&2>Cb{~%zhRq_!M}~Zv%_|05X1YVncd!z zd%PXmL?h(BjMZLjY}mBntPAJF@qhg)N!AuN+I1fSjK5xaVuU|`**pA_qxyaK z~Q+$Pc*5=qFW!pK0`%a1eg)jsSVA9%bja$+z# zML=^;#M%V84gTqJ7maSM4S4&?wwbrh&-;l; z1HmQc`Wo%~`*I8Q`&1sb7~eUqScB#nR60`g>Go%3Q23?Y*p<-E-gpZU4gjM{*ZMDYv*?|(yPJ`z$Qu#5-p#r{3YAE0>JAdYs>8dOUg4+@Ry>Y@uA*Z|Kq z_7o4;&yFXAg-0mhc1FnsIFfBsyCmUPft_KNvQ0?q#5CIP!oxD6zXe(D|XUYOC zMT8L5)fz&3I#E3BA@p$@4_657>H;Xyd_ah1?_mSNw!l9Q_)ql)VNa?r3him*ZQ}x= zZ4l~c7b-{~g$2*kpp&cN!BZLOo zIYVf`iz^E42AuBzx`0(szn})ZAhb7x_JPp85E}RjIu4mG`1biz)xcIoD^oo@X>fCZ zWTH{QU`mq#Bg@d~je*4XsIW1|?c77AUl?mn9AH<^($-0?jt(i)c=S45`*M zdmt~csip&)*$e03{%t?Wdm@rS#z=u`r?{!kwGk*8%rq=B5;#Brc3ouW3FHbCe7YN&_cNg5V1rLI)ASt08cqNZ=bHDIm{SG8V$)@pBh|5lMtOTmV~0H=kx9E&Uy} z5dQ}s(17V67swB(LpU?|lE&eX3wZ`|C=D2cVIVPz2#iG-4ih8+1`u2jkEEN&`;!i& zfx~110l6>0=b1pkcY+is5TFGYaDhoUkA*Od^Ct}R04_KlzQg&?$NecFLI;jtNC(0& zj|feeF1Wz)JrEAOK!opb1;C94(}C}REJKce^Sjcfk!F^$sFVZlZBavkrwh?xC3VYFeME}WIUK5BXKbE03U`C z!BmL=^GhHvk)63zGB)??0L-n1O`?)8FB)fVALzV=TlkjV@@Qb1Ul$kNEt#-)&;RD5Dp%ZK}Z#d0~|sD z1%yLJZ=i=D9IUlLtOkUfRtZ9ffJDy0G(m07g|s2ud>%Rw*pH1+(S?ws*FXwA2syqA zgeb7aM(A0BCZ7vgL7;6B*qjmu=X@Yf)<9I@5DF+bAQcE~ZqUZh=QXERfv!j5Y=KbC zhhT+Fm@gKhqY3jRvx8MWg1w-NL5D}4?IFT^zEsfD=U@&H*i(wcIzr%^2qfeLfyMzL z6i~0ohYg@qz(3%8W{AcI;*FrX0%@KP0Xu*=pOG67ojC&k*(5+#kQ9HJCSVWs9BAI6 z0WZ}So?*iTO!&bgQ0>5afsh>rfRDc|9$C{a0expahn)BJ93$m*O5A21~+%e;5K7 z&564)I1vm+L==TqfboD)w#%79b8~W~IZ;9PlLR)HIAl+wx$Qxt|BH}x^{@j;fC+Bx zWd)4Yztgx`+d`HUD@R~o&7~Lpt5SSDY-~{&V6UQpGMfJZ!6JhGYzzH?!Q&JBgIwle zI5Hk?c{&&t7`*f_9Ng4&czD>(q=&)d2|Wz%bc`ErDX$E0>-ID^l&5sJZdh2VF+Y+1YZmzV&Q&A2a|$FDLNP#?rV!-QdoF= zr-R{^!ILH}O-qKy?Zs)Om#G(N_;?$7Ji^jCl_taUfW`Dk@FayEh9@s6FG(66ffuJG z6W{@n4n|y(7K_2bqx>Q~;0hJi0gGW+crHK>BfvV74v!3L%Ed4o9v)d2!|+(JN#2P0sY^cgHTSDwC}u|&)=FxbMQqeon(UcgC>OXx{qV9Q`Je^S^bvVoPt zgA>K+@kmSL04t4yHN|3jKn|9`NbtK7Iy@2q)*_43f|=ZsdR}HM!IGs&OZqB?xO9vF z!vQ@#AmL!TxHw-NhP*T_c4 literal 0 HcmV?d00001 diff --git a/DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.png b/DeepCrazyhouse/src/preprocessing/plots/crazyara_training_data.png new file mode 100644 index 0000000000000000000000000000000000000000..b0b537dfdc7975c36552a847bc28ed877207d087 GIT binary patch literal 78044 zcmdqJbyU`0_bm!y3t~$npn#y#AV{bvD59tcqI8KM0@4i@7=#K~lqd*DcSx$F2q-Nn zUDBOrZr|VeopJ9t=Z1il2|m96^#e!D zEsgl1dYQ*4DE3jDJ$2%Web8{LgU0#!)q?S>VTT0H_`5hq>pr3zGT^=L!F?*7yI9A7 ze({KYtbSQM+mq4@H-CD$JQXO;6T4ixBjAccYdfpd$b8nwZGnotlP)r>zAV03p;ncB z&eDA$RwaEd(oaJaA}+H%*miQ`zyBHEUb9-g?tlJ^?Lq0&wfz6#|QQjV;2`aXO6(6!_T5$9f+gwr!HT>fV>{|NI(zKn{(_`1t(mEw4cR5D&a7$~efQZNeY3Y#C(lalQ zXqGQb4~>tHhn@A~WX;YrH=OMKLD4@nlwvbf?Q8I50cf-dFmZL-92~rM9-Vv$OMD=Tp8F+ZFW; zv!0dZ#eW+xE+hMI>7}pn1UzANox6*tkn^G|v};Lce?ILM^By;Blv1zj9z;OfhBs8$`)^!B z)nT$s$84#O9XqyuD^t|=oQt>Ga>Qkp#;AOJeD-s3Q8mdj9XfPKEzemz=4!&nYbiIV znI*1>bMo*!xzkmwd;9i}SK6+R)6zt!)>C(UdANnaVRodIam$vcOsni19Gj@AzkT_# zMn*;^*rvZdUq;(`e4T)R0K-67SlIpp2mW+)6o|Du-&csx8>)|!m`qGc3cKB$u90p1 z*oXPp<;V-n5%B{919J-tzw%t14jn!mt(`AjSy{Pu{rVSg-nt+WwFfKMfJtMK6;1!v+dvaDlw|o1HVo;t&IJVy;1S{1l}xK^Rwvr zK+y=h(KaR-R}CZSgsQs&W-UkYS?$Lr ze5L-==)*00=_IVV&%YH8v;F%!)vWcc?pKfPyLV4l8t%o0NbeuGOHQ9(+O38o`#wjW zHd~k)_#m{5CH+=hEND{8T~SftKD^$*mqn_yy!=Z|jo;?oJY)Dk^$FTa_+n$Kv7cbZ zy;raHDuqhdl?oj`>^)EuNp&&$@&TNhvLJD*#w;r?gc`#l=l=Z`6WwPSL`}~ij)c>r zJ$;WJJEpIv_r+IcrL;2m7`><|9Zun|PfxKkSl^=(5_&goc-*~9`K;BxJ;R*!!i5V{ zqwS$yUesT{etjeDB7tQzT$&rFxDu;&N&m=`zN*l!-d;IJM+ujOsW62A9&C1!e#zGL zv@8|BBF-7td^{iDusr(c=~Ml(7YC>rgg5Toxs&4Z<;(qbF?;Xaxnu9>csbY6CPd0f z?&8I5c<4jw>4&rpuVq@WWtg{rG;hz_bx7^myLbCk zj??cgd&;q^OiD^hf`Wo=>Qc=a=2ck9O|-OqC9H1Y7Jt6X$45Fos!Y`V8l{yhfh=() z>LR5(BK+59J99j55zeLj8$rWIK0ciL_PKYJ_@)?DvGi#D+bWU&oanM_i$80yt;N57 zO_(#gP4!=z8EsDxUfJgqb^qbRb@B1>eQ6=Wo6}o#v~4_zDku}ni?86Yd+GPIB~*@N&NCA z+nkZwXk|%~-$Quws>WWFj*N`-d--y`!R%HCWCEO~A`~Xx z{59(~Q&_AlIS2{~1yrH>AS?CS)3gOVe*9Q&Y|XnHKi)FPT3{E-N=x4hAGqZ>fGrvxO6O9gq02oe_*|e#!y*-ps%=rCYkn@C8mNiBn!tirs%xxA$bGsJ{iQ9}iT;@Fq4n$6OS&xhqxeYJ4xjn?{&av# z&97e@w{HD-<~ciqcR%O(w->`suJd7)sa#&1wYjhq6(fy^IHZ;H8Y{M)l{LX8S1Z>s zB1O8szFzIxwe2h{KHlCmhz4QfUk|Vbx&{W}ldoslB1gNs%lu;1lJ0$f#m7786xAd0 z%JX_pc|h7PY8o1m_!Fk4hp@2>?T;54FGnbh4>zSa2lL9*_PIz=%*Ib&iN0dmlKC9r zHrAl+Ha^kwJ)@Ffl{vM4nB^N+jA>J{r2UvXB0u=+zKP4RY65Yoaq1a@#=rI>Q_=|< zY^;m98i|u$ZW)&?>9!ISChK`Q_2wm%=hWV)>YADlI4XzK65VAzx9Z|RQL{Q&ZK``T4iW%F1Tj{w;qkpubWtQuQ=`w7W4$ z-+A$G#x6cBCR$orle$k4!?O!xFLUS^7<#jmu3u+BL8;4%vmfsq6>;Cr$hhTQ(5Opo zO-YI8ZeG=~(A6bRWYo)9mWI^S)R{IzF9pS(0_crLzaZ%Eku&iu;a=+A~K7lEYUV|Ek6O+qBSBc+mx@}T?U z&7bd)$2$iG&T42d$@?&|A;kbf`MrHxt|YVUZ^SbCHQQ#06f=CLkLLq<28@l3M|0)x z;vk~xOJw9Wadc{iM@CvLOc^WqbBD#6PQ*_;PY>>NjtyJUs-r6WK`KPJRF-;g$JIWtdfbw>tR(SRtFmrs*r+@I!@ip z8(ec!*n@J%&YehTav>%8m9@40IDg4DL#m33ijDbhtLn_FNA&dde>OBk0-Z1n7*&NF z;N>044T;gnvXpdPQi~~M-MRBaT4%-Q#>_h$n>KB-9%;F@i8J?>RH&3wF{(sjVq*11 zt>cxm31o8*KL5|=s-l3Rh|(BU2I&GPNW99D#KzP_Dn(+^ujevbA5{KGT@@n5AtIu3 z)(Kf`s3Bp)Mb5$p!*YL?<|mWV(<>Xww`yivvmHKs{#y1GX7@|Vvv~kdC5442`r6$o zsVEwg4bR~dzTGKy!_ZK`X~xv>_4h!l{ZWTCvv)BvGRDa{72aDXAS}Ej@8R1Wj)Z4m zXn1FJ#pS?Z0RiL*)+f6Tz8??H;l&@+;JN7rZINGB<2B~nDv>iP@et7$!`8IB%m%+$ zNuRE5XlR&QQFwXi?AF@%F`4WzVYP`mq1h)Nvu(8)t*7Nc4{RRL4yapfC`SM^> z#aof5D6+}MHJsPeOo|?D=b*URoE}%nvI!9T;!BNnH|sv_Ms^B1X6Hk4+Kn3@P0yV_ z56VNw!NCz%P&{-!-SjMu(^2dGZN98Bs;bG=MQv>%IGFryE6!Ru_IJa=*qmmD!;n=o zv$B#*>Uf-8T<+M|)NLLii4srzD^6WZLsJvPiQ~Y5he)5mtJ6|P*__3`6JV0kYt zrI3qhCMqK+siQfSp)v_m@i=9jf4+J$N!mX{?*F73SB{h_GE$ZiZIotG7p0cO{)$ig z2C&%cH*bDMoZI_5{q{LEHM$RS-l&>;z&KK{+^#G0MtO@P$1a3OmZ8!Tv~E!LV$?d0 z(85*G|J0{&EP@vR_jUjo9EOdXH?y;`QBqM+nbbxV0#jSq+LBr`Utab7`%`3ZU?$&; z3^5Ut+Jfe0Ms;;{pupG1?2c^NbLgGp^q{2U)JAu(yTOX@MMV#rQjB;*@tsH5byrtc z39eD+l_hat%rct*S{D}=$C=?WRFbA7ea~XXC9!L2YKb?0vJFcS+mCfmWi%zArNz_NB)vuOG}GAlIJno zVLV_UkIE;MY>I5_fs;TtulaS?05iOEo}Z|VxjGtLIorQM!&+(W@cw!&b0WGuUUmUAYePJ zhTZi@NZ^aRp7tJ4O)*fE1}jLaGOE}d67{ug>x0@Z)4P$15TGIzfnvTuZ)XUf|HdV+ zG<{^KihzdBxT&EA-&+mwcg8vjR|om2$43~%%oK6Rj^63~=6U1^Nh&gDd;duUE9ZQe z#J`}ZAP>CiD3TvU0jH+pcSlY1p8D!Z>j$cTe?5)##WL*oMmE_qWZT`;u!xr~UBWr@ z7_5uw`txTz?CE~Xi*eUQz?Vwj2pMGp?bUr!;gFDc3vR7@^JeM06ZfU9w?8C7b4~EU zDU!TBqq!zb>tpHL+S*>fee2q><8ce(+!Ng263IVA+C?EruP6%a4}@O|6ltZpXeXXw z#pwSkVf-Vt|DNuD2^#n@EW`&cN^1P~XCcYwOmn42x1gBRD17a6+(u z+*&zDe*}v4d9u2eHW)k;6&3x5Xa5r7ioGrG1M}$Y>np)pJo5F8#EBC&Yk57IzZ!yA zIk<=z?{*kzcU|;;vYWRzOBglX4?cTqkz7VbP#<0{5kX&{ad%b`yW)cKJxIG!-N5mgZ^_!=XoaG zYETd7aKIW7W?u;X{v5X1-mEpVfPPBlSm8L8QiR{*{##)8=&V`N_VTWW(>0lZ#@H z9zQMwm_jO8`b!~)1dbqDNLa_e=ZMA`XJ<+19v^g0w65l4I!torFHM~DanSf|>kY1V z5#kzt>%sHq+p&B)zf+C(@892@^c3p#A}Kc;on8 zjMos)$;lC}bi)ZxH)}1&XOJ%MI1S`0(aQqx8#Lji3mY4v8(5UB}A$w$xqb*#7fSdQ&PH$8g|C;_U$-@GiRzB({2A=5ALZx zz|MZs(9A6Ucapw*$YNWbi@HjJmXu1$jUQKlhJI#MJ>JQ61$j~B+O=!Z(D_~&g{-cJ zVy8^r5^px0F$v+!EiZ32C~7nb{Gu;w6ktI`+jTy|q`x{$e8lg~n`nrCf#pq2*T_3D z9J7tBmkyS&xh!(4UU9m+Z0CW$J>OsP7#SJit#hy7Z+RLM_EnX<#a3pC@aOS)Zp+Kd z0m2BcfQX2QdZCy+w+&DO{Xc{2p^T{-2bKK%d8sQC=RP1fSTnxi_^a2iUo$0?ET-!f zZ4SuF$||p_QsP*~A^*3o2#A1GBv)DTKq{e`nMYM{)Kwteji#BJPckkXEUT!vl4e{Z zXQ;35*SF03U>m!+;06$vrB}>HE!^=7>#6CE|M~JTU~0z!1&fXX%HFEbgm7kY%eC&( zE()MK|vKn*t)9i z^6=@?b0K{ikdYuBR0N4D$Gm!S9awrEBJ~>q{q+YF{Hq&h5O4o`l~7-1T9Rrf*M)B6 zcEGwz8D^SLNUr7B3qaIGa{3i>l`l3nme5C>`2$i?njE3X{q?ce?gOyc4AzFZE-&C0 zp&!Spi~z0+0ZP0TGsiyL0JyK?kn`$*?4G6Nhpfsg%MZ;ut#=*RdGU`QG)S4qPxtZ1 zl36=p=sAacrrjtDk(WkV)y1X%Pl9aLOI|Uce=ZIVDkY^VrrD-c%wS9fEr(9-c zCjXtzQ*kUu06(Z!R<&@7v8D$ZW{vQLv|bZZIlw^lC%VhTUkdQyEtN+KahioJI*tRG zQWb!#gWTwJ6x@~Yot~a9`tpUnAt(<)ABv@JI&F>#-?G>f^pIkp2}aqHjajH?q>=b=9mA=kW}<=nY* zh!)?J6d|xn6!JCC0A{f27M7O3-pHfGU>y%>W?wmrgvmgPAn00cyk;P3p@3oeE*5F$ zD3uj(h3z|c%A)EerKVm9X@C8ij`Kp$BlI+&W&zf(wX?I^%5?PIbI-gyX`lg4QPGzW zVS26^A?alKu@R&|YmbJ(y3M<)n*y|2GohQTKOS-Uc}R#46!H2za0xjS8YsIrZ^v9q z;dEOX-vIe#sZVCL%&rOI(m{<(en&?~h5=-PNVnA$316hzH4`DTdh-)K1S@N6w@ZG1 zj)+H9l*7T8WMTkSV2n%k_umCLeHv68aw?KQFt{j6&jWyx?=KJ6NTAC5Z0C@x1}%}W z8&N!doC06|fQo6_mP0QlCKePN3|PGZnj+-Evss*oc>Fjp%n8q}%wuC?`@mvyon{3= zjKO&6CGGj2Nxd;d_>j&8GFBuMcmNw;z-)`*MgiN=w)1#ooBBG?02EfVSjHwM-s2G! z?z=W68xq9{{}3>3DEuVEZyJF z1K^<0p_^cGoA-JucW-YmDUTp2kXY~$O9fcxR-H6kFS;&YJEE4j4=KC658yiK#t#OR zLOJNg6Dnv;nZr$7ONmPj~mJ)sVtfxi@pF-!luoe38*w}&tryG?=AUJ zDbxoN6G}BvJApHq5F~gIya_FE3=-^P=3}<}_G3DrU!TUCCy|a*&=~Pyky?-uqP_s| ze(*oAiyOMo*q8;1A2~P3l7FDjG|b80!ge4b?-Lh)vuP(M2^t6qgsS3$)!D0?85lfJ zhyb}faZ7rWO+gn@ro(#vU%!4qV_T0b#d+}HCa7-62=wBX;g$L+$;oFfT)w8rg5Mt#ob#~dKUL*X^9es>~dac~DS^Lfzx{+}N_<@9TR|7MVv zmlri{e7C#csO>Nv(rP5yK*w!4>gwtOa3WFF86w78pqv0t6aw4_P!!*#+OUz9)@Z8# zC&OKq>9u~4Ln9#^(lIjb+_B^2jT@X+y%j%i)@GR0JrTF+wE#q8qM`$}1kB#y4!HRX zg>K)mxkTrA%ZeXAuI0Pt3f>KmiQzr{WS4GNvG;?A566FqwRfPC23$%h%7v^LF!Z&IY0d{)>9V}5;~%tR{_rA@1Ri&o%GDvvwuLh0qs0-NJ;OK z0=<*dVJ9B~h+ewKXE)MvsnRbXpa_T!D@utRb4VleSz;pDCZZ|67BqaK>dUEF86-}$ zKz}PhzD+ld{Ui7^ zy5{px4Z)FAd#N3Q-@KuJ*1CmFW@>u+TAp*(L>5F0l$8|H7owtrxETKN<1Zkx9GdSq|zaqBaTF;IHp09tgHZf6iN37<-RBYK+w;h=|JF~GY* zQwDaQn(62>m6)oRHgFmI#|xk;01f&8&hSoUWo7RWStud@``%03rB{6UOE0RV7rpR- zdWIZ9*-PJ0vuy_#>4!Xwm)?ekGTOu^@VTLtF%w`ob5>CiYy_GyZTTxcUteEZp&Jzi z)z#1Fg>DeF!%%yA|2=?dC+=8~vi3+}vyFrt}a&dT^Ajhw8nd zb)LJmx;Q&plc+0ediyr%<>A|gz(iT1-@uT5AYQG8JL6*%DTp5Z8?y50%a>o&w*&6@ zyWj{t4h;?6k#h#>7*3X;Vfn>GofE;l321nn*ravbw!wjdqzkcC$Z3fANhmJ|4)5Q; zwsh;poNFIJlU_W3UdZaUM7gxuya1+a614i1FHh(1@3d5W7VU<98vxM`Nclt?km3_V0Pd~XiJey5U8_uqr4-;u zJJ~&fp69-!dZ9PaocnreR6YN5mmN@i=ZUmyE{5Ivb$?@Wg5Sd1+_TL4f7%?j z833j)?!$Q+FWReZF+HgA!q3l!MPEm!0B6m=v)J3)SI27}gM1V#d_+Cn^fB5#!1GWB7?Bil&`6Pi z^oyRWcX8hR`&4M;8v+XA(|KYU|BQ@OVz*z5nX@2s75@JH7V(H8URqj8QJv-AHFTpY z#2e`!qE^6*dLR}w;$I6J?>P7JkS>H~h7$0NZQHgLAncCXjWCgl3M&K=SWsAR17Kp|dbM8k*21WzLI6Gpas8wjcq7ETiL~VD1l9p$1J`G;B5)YUgfk$BujTmEc9kfw#~fi@l!qSCRZ1hT#G?$$R2L^sdD{~RihtxTDSvXdTp6phZZqK&a;tph<;xKt5xUY8FX{uN9 z;Qygn*ps>7m6Rlae4JxHeh<+*&;k$7skZ&@gr&n*@vzcE1t8YAHflS1)P3kV0sF{0 zIL-Gq|AN|}Z*Hyxe}LP{JQvR9M`U-%FM@D!VEr~6M;SsJT?pT~(OdDx!B)vFx1?|g z;PxLI1PVw^r)OZ`k=+YLFEJw{5=c%lM3NpG8Mh3Kf6m0jB#(3Cit=nu^>lqc8a6oa zH^Xzq-*gaZXr|fxAe1{67M9viD(8&pa^}}xIUK2@q;+CnscUm=WMm08%!-L8%&Z>- z>LKVrGSC6+5(LLWQ~1`{NQcHmw%uq*-+Wv6R?&V(-gq;n$d4a;-NzhiwRhc?J>ei66`?Yr zXkc~I6Vo` z_d$iL$#-)@li<_T8dT+%#h2d@0hfd0{xQ1|?pznk_c51ZRbD1RU;T?1BsJ*4 zgSB{(#mdV5<*U9`(Bwx`T+(cJLlMAM0@%Oh#u=zq;%id(FZK#L5 z=(DgxqQb&let&8c@)FIM>Q4K-3)`Efzws8!P?M@->^AQylg2QTjyVkwKCMap)I;X0v(bmXBB@uj6jLqEVP z2+Fr0I5jwBDJ1Tr_G9}PMNNqo2dv|dhD%=Ee4Y1kO<9jxnjbj~a%h0;;=bsPALYpv z?O1iPTVfvc=%4JX68TjHJcVovAwNy)cmKUah@Fr@MWQ>OsT%{)%r?#Km z7#3JTELNl3AQZ@%hGjKk(S5HkiUr}RHxwX9(`Hm?PuN?rpTw~M)+QwX$0r-P=)O0#=-7`+>anI2COyHEXfy zufB_(c`{C>Gi&4?xFAvI-~;=4M#gC-wz8x8^JTmHSF5bXvw>tZUZpIT^kdLhek-#>gbb4LNX86GNOvpGP4DB!S7 z)Xv=>es>iV{*B13J+z46ZHsZ;2Q)JW3XLyZ$vF&AO{fV{Br5#Rd1o1y zgooHNDaCX>>$RV zz{=3__BrL&sQcAJ7s52$4@B{|8JS#~ zw=z_5Z*X!&%fll!_E5x~Elu8x|J6O8j2r!YQ7Ni_bTnc6KD?C|2>;8&>OAHF8>s2r zJv}{D_nlsz5lLX<9c1H4pk*fgSeSRf&^2S4a~&mDhx}*;hlXx3(Ka=Jr5Q2EQn5(d zzpLP#u#11=c)?$+ul;6wUap{Z?+=+$v)8~KFQBHbY5FRr@ao79q-0@7E1V+4?N+R3 zxWLar>`9i(V)_mP?u+jzkbvP#oIu7m@rNkm4)Nm8;NWHh+HvH>14xM1d5v3gonDb1 z8y2{)@v9CN4t5vf#4!oSj|annb1)Rg{Txd5M4j><>?uRZn)2>^bJ0$;AK?@3e4ZBii=NYK)YB3rotO& z8JAl|zdEAH4tWILqFW*w3p0Pe!-*I${s)JNG)Ew?RKu?bfG5Du-+@ql;OVJvW>$#? zD+M`1QVR1XA;X{THR%v^6^bC36z(p_@5ztgzL=;!Eliwwr<<=4;o*zhZK} zEZ&};8$#Vy#0fryN3R=4Mb+akajpU#2*J{eJf?u~kA|j&@FK2en7v8&bE^srM%dozZ+Iiu`=Okf$rcW9&(|gFW&S03Ny9XpOu{_&elYU!$$BrHT zj@S%w?Y;-Y`?_b_-IUP!MWRqF@-lB-L{revvw03OYeivs2{BReqDm7{vk&7Yk1ZW*J!3s(` z&3NH-cl`Z(9^GO(ejN(vU}2z<7&5pE@4&@8X9vT^%Foalw*y(~<1E@cIemzXlm}fx zsCZ1+vL+|!qogcM0ED_Kz)$F5h!VH4kRD^?_Y0pK}b(6_}DA3kbVsj zb}{5t(hE{B|B(Ov=g;TB9LbhF3c%Cnp?acMssjSFSz+glCiNx`Tey_fP!6|$d!wuE;TkD8nNWz8(_2_yo`@Y4H90-9( z{G3u!%?w7T}vMCxG8jxrE(7`X8t&W4(Mb&*& zkviEaCaTYq4ksIHZ2-9s#PAK!b2XHAq`no^*;0fAagz=FiXeMMKozpK$=bv=19*pE zno@&92fz5WvGFy|>YtvTBBX9{V0r{D9X-7<^-lQQ3*kY8xHK3v`VH`yux7}-YpAFk zfFvO_eyOkD!^f8(*>{GLsHKgGx>QJ^#E=fo1gVnr&mf&%zw3wBrs~g$gux|;S*kA3 z8zJg?%-X4-}(tWlffrQM{AzT3KK)$@=HO7{WGRv(ZiTY z2zr??_{Cd&ksEg{Dk}QZ*XNm*HZ9MXGq()_*0p4VP2iH473!7UyP-5>mDqxSh0cwk zjJ8n`3)#2)Gw>&+a_-sHnXmip(RQeYLb_ibXk0be?$(ax9;sjG1BFLKtbzOK2aRHp=t#h2&L_QgyL5t%dn%|H+6ya--v3_1!QDM+s`kXK` z7!qCkJIRB4{396u3`Zv)`G!pWzni@J_j~_sR3fgWlg?BBq>%r9#sBZdy#9amW-AY2 ze|&kdR3FVTE zVIT+gEw(_~q}&JXe#3?hAVCYWyc2KH0K^24JKh1mMcN2ZQ#HP?<`HiuWW43BzOsb= z{+oGoom+bHl2TGg9}kT~8h7ORU+_NBYLqVlpE4190tF~XC)z5b8NHg%IQceKl=uq^FKvO&bp@aajLHI(djT!lo@-I1U~1g5oA% z-X__T^8ESFV}0mHyAbgL*$)GP#1IRQ@;|pVagV~{L&n(v%T*zHy)5vpv zgM^xAC5@I~xLIrV0}qd6_;vt(<*lt#>s^t+Kj1S0R;Pe4;&VGJ%@MB$I^X>8G2n^F zfE_eR3UF6`t6oKPOwQZeA4RLAsHCJX_tZ!k~2IUr%)P1dt9CcO3|v$Ud94Ze33%|A^^AznFUX?_Xf+9yEWi zmoU=N5z8^es`bR2=`R)q=9PM@VJqZ1;H?3~SaD#3*x!~cWrwGkm|^n1O~gkO7AGes z$f8v4fq{&8v!h3kK7$JZ0Z5#6*f)hy3U?TYxV_(l(|2}sxI^zJ4MJ!**LyPIl7Ju! zkFvPJvq&r`NL3&*KV_4-Ee@JD+%7nJkTdwL`!ABmO-j-Q5kaR^QIiWN3z*>>nNFOY zHG$APc_)ev^!-RK#~uiwA%}skXRRHudKDKI-o-9sH+COi^d21wl|6Y9 z^i>KTZ*=nSLJMIo-6~B($A5f4*Am@Uru;xLqP%o>6Ug@A`AYv!4oNAgI~EoN_4O~| zN~6HYQW>at`0y;|_~4*A2kRuB;2k~y8V6{dhu02Z(eYYT5Ng54z9Lr0umti984d)E zoxtRi_av-(p)j;jG|_y4>XQUHl%lh%i+mug5ghe*al%M&pw~}Y(JxBC;$b{_k0^m9 zpxd4X7NAPBuLG(8Yzv_aL(Vwx7ka3Q<2KN|vsrxF`F2>MEX1V0~Dp$nHCT zzN`bxhCZe`;^Gc|-qqdx08kA7j|;Zz(mXeGtIElYWpHqC<-0H_PYCsIm>vXT=u<1n zL9SqiZG{3N8?mDE@F?iB1}L0H45La?kb(}Fi-y~Yc&%IoplYPF^(0xsp+#CtaG7|3 zutJcc0FgLQ((W4+Uo?cDeS)yUIvqWB%&PO-rZ^YCz8}zFh_ewOiwpWH95%Y<=Dx_q zkO2-LS43k@%^t-v`7n{FnD%bEEi#cZ1wESVFjT zkBNz?993ypo*S&=h0XQ-_Uh{p9bw~HXSwpjc~XpDW;PgwU+KTj7T856xELptAHX|+ z*B@wUQ)Id>OTe76A74c!==%FN&c+#zj+<>cSrhXZ@2!nf-|LPvei2?;Lv@fAB6eH9 z?1J)z!txZ$gr4dK_*&vjpL%&2Ag7Z7Gn_cg!9ITUNC$@v-I(Xd)l<_ct70D$5|q%z znnOzR@8V!*M~&)+oAr=(UJ$xflg;M&uu4Y>+iKgv9trP~614Hu!QXHLcw63X&kF`s zN!I)J&Y|%%Ofa&%j5RV@^h6l~%5JE&5g%J7`*c+Se=kLF^0CsGC0f9p3&GW;3 zeey`hq^&~Q>!1fn$h^5G5E%j!Wsk1Vz2X?@$wKQJBdfe9c9kdp( zo(=915eE>k7*rKRHl&Dh(;LhLjv-W0o<;7N4a#U1 z20myl6!uyLkI z$bEFM=D|n5)g3?^GlBtsL8$6Y_@km3-C%hmL#be3+~zSTP>KNj2UTO~(D(tEfZxnd zUmSTu9PNGzs1{HRpP+tNA>z=E@qGMv6JCJ$F1K$FvvEe6hc-|q;FbpJ#pC^el4^aT zLD>#G2Di$mU90m|Zo(dWqpt8^n)DppCB#pNOdSDifqc>xPQ3^X5b_rZ@bc$(+{d7C(UD)NXB3EzHod2;)<>ZdXXAC6rWs;NK zF8IBBa>`ljmW@c$*OYvxQ1hAO{62?}<)XbyfP=!ffBVCdLhOGCfBWgWtCYmPj(lPS zVx1vr{2M*Z0DD4!0CVlxiFU5*qmwl9Lre{)evpDvfJ-|BKl9FI@PD zHXa3;Ign>{spVD#Qm952>XIbpeE3qLqodKP@xpqmGv`BXAv|5}&>kq9varw)#!CUA zRQOi?@XNj#v^Lg=)6UPkq=dwvR>EpW1Z2G=HNtH%{88+Pp+oRx!-^O(EDe)+U}P|| zmlgaEK+2TIq3aQOBMCCZ3@XD#=r9nzN)Rd-Sla@&7YW*S{DRUWq;j1kHI!K#oF|0V zVdh~cv@qD0^ZKgapWXsPaTUrtQ6I=ejl-XZd%fN_d#}zfF7CBIOuF!pQSDKHCM7G- zmDRg->uHmz=*ynAE%0H+jcm^v!ha`o-MZ85J_Kq$)j)ronEZ;{)o{|ua2wJX&sfmD zzP1{Oukh9!yq3HleHKDiix*l#IOfsTets!+e0sVkGSoG@M{2fe;xlbaIXY2fya&>0 z+I80yXXyD-5Z#LOl1QH=A%8^yWy8#&DrcN!?&`#aQ{85XhleRno$6kJ?;O}>53-#8 zHB>Gb9Ry*AEP#R3MEtm=r65`#4!=od#QhU&V7bu(LE#uDQI!=Y&Qe4~UJ~XeUtqiG zTYZEC1tB%AgQbO&jjeF-L>7KhOe(=B)a50Lnk8@ta@IGDM*89?L&hSaB$l5_oHl0O zMOF>w{x0>!z=P`PvE!3ylD)=+2j?q+Jik5^PZGhou@^*B3?0g4slGM0&MJg`tX&O zV_D+}PJAlNbWowjV4R^4&lN_i_N<(o$Is6NdP!s5eoBHc|9udXjVDAO<#=VTixw6D z4A+Ro--iMQwKVhdXSkNWB6Y#$a^S#$qECHauq|+2g`ax7L$$B@-3clJ2w}#g7tmXa z{-YPGOj|!C1(~BD9d+bZGNppmyMcoRNh=XRK+Bnc}{Ns?|jTPG3+)srVA_-_hdapN*mGyj zlumXyb6~nqQc{v=jW`n^89Uw4*c$tFj*os)cNZZ&&Qm|-QIg6&amk@d5Mm4;sKX3s z=X+!}q(~7+?FwdIVCBzi+GYqLL@U1I80#N=?n}n$doer zK1k2}ZxqQO_IT&%7`DA`y@P`aC@27)-|dz$`1GCp1!#r|`5l@Is;3Ju^+3#nFvI!a z<$(7@J>-FEp1?&dc^jjbN2|m+xwy#O4%`+KE;=4T=DnEv`U-$bp7&V|2f5({G*RvR z!kggWANaaG_W7t)uUFl+FrXZm2qGl?Xxj&`A8*TNE}eJEQ&HlDGbCo&EJvoT+Ckno})+SJ~p=;Vm;64C$POAaRYxHqs8K^V(G#37M{^lr$#tw*tTzX zC?)g?<)N$26B%6(^+rx?5fGQb{N|%4Prl&7qAIacDBU@;ZB7UX_1%2?`dI6bGrpCS z_`%BvR&I>L9C8#MR!saw)(K+p2Cl0LG&dm@?1SD3O4=2ei&8*_-TpOQUbZweyh6po zsNpTiQ%;VK$!KbRpi72IBfYvbDY;nxNo5P9m^Oz#=C0;?sK0G4vzq8OPaF=x`y$+# zD{cruy7~hzF7d3x?GM>r4il~+o(F(e$dHMJ#n}1JeSMa=A_x5(F-~R8G?U#J?jg28 zC=0~q`XN$obD=2KfkwOo0sK~h!P%YK8=>>2QrFbqMfU-*s+s+IEBvHB; z{K~!okN!ZEdF#re?fnIIXe8DyXISzS;PP)u)V-gyikkr@sCT*tHKe&N3`8EUlyxKX z3CoAxiryv+Re#raGSCC@=q4N}A-Dey55upW*EJcZk(InX|JAEkb=jbfUBknQ5N?SA z0X;45j+R)j84&hC?YtZTF*4XV8EoDw2L}Z4tQ2o^2Ts_|z_7XCRzhwubf&vOw?f*R zJ%Ikdf%#EP6#p(_XJP|eoqz(JQ=twkGQw8#i z@wDV)MqQ5Btfi#|e(;0a0wcJ5?TaNOU$#hoKkWavDTOG@7ZKlq#mh^RRg-mBwV%U1 zi)F>KyoX;?td*+)IL(WV6_DTqfSXuu-|yeQUjziGbnt)9AG$HWkx)tmP6_;+WG0=& zYi4fF&h1%&wN$dFy4iLyF;!r|mUIUaQ&Mot$u6{%F(-Thtcj2gP+ic^`=D|?O{=ig zLln4c*XB@SGEZePUQFR-|2RYRR~qVt$7^H%_Ox$5CQhi#TkNa z!s{O9G&=&|rekDO0;BIuk+t1&=DE{*ph_5EQ^?pT@HkPq5Z!Qk2GubhMUO@eeXeIU zynoO;Pkg)jejn69(jkXy`wr||5PJ#Z;q==C^B(e!H@O8#a(pcuQbaB2>^uoEpLhWg z&yc;k)Em$=1|KroC4lK)|51YnZbi@7WbG$^h#kBd50BZ@J0LD}csvmBX8{5ms(lDjs~)NWige!B2N-0elX4UWW-BrkRfdfcJ`#VU&kzO3k*=T|t8!cM zL!_0xw(F3qlA2{AVaDWY7jv+A{P3`8-ohX+w4*q!)6KA#YPrrElBA+#|7RWQ^bg2$T9=iDl~q^nY+D!R5B4K!fae9^!N|8hY}tgqGhznbOyW(4 z0tRn-ck^Pu93G?xECT{%<=nB)cS8IENlxVEegvd6G1ukGIO}t7Bn#z{@DK?3*#+!{ z?RQd~gK$tW8G>0(N;|vRW7$S+7YKSd&-wjtPT<*7r&u=Qst+#Y_;e4N(qPiOf_p*W z$;Kj01B3m*%YtnSY())#=VX7OVI3L1*e!~D<%=Hgc#BolZd-14_Ka}0{p{@QvzLHG zfG;zKU9~kxGDU+&TDWg9krx0SW_?p#PqvH!+#}VEE|EjqFbDhPE79(3?Ch4IP&8PP zrjhQ0Ml)ABZ$L)J{YYr2Hy?AXc_?XbsZGgqx*c$h9x4LP=9gkQ-?i)$I@Q~tbo~2s zaN>&D?*IJX=j=-Bs8NwRXaEL~1Sa!Wmq+uM_+Y|Cmm690ZQqX#r<;)SP;i)$u;5Rn zOK?h6JERyau^T2LTsbkRn4X2}NZyLvlC!tZteZd~BV!uy6y8Vo8Nv2_Hgk2ceRU3f z|55JVy;}Bn`$C+200X9dW>#ZJ0}*kFa|9LiKBA6p2q?ifG?W$i5e@2>XpQK%LcoTX zxZIDHG~6&fP=sPeoSU$Atnp=LWITts!z{x0lz7Mu#k{gxETQ#urNxQ~LH=G0$$;P* zf$va|nJ)xNi}fPJ&`AhmZNAkb*@)r^=nG_|3K1LgWvl$DaJ5U6veg8O|<7^Y^$E zpc}3&Q5I=)%<=6U6PhplbBXmKiN>aZrq)1N}sFKW$p!beE@eSzd zoTa5@h(1o05OgS5ILzkXR^1zhi4IilkIzhQC=OQS`q9QgSOD8QQ{9hq$dc5M%o2SNjvN_z~+2A`C4Fe)N0vgW+re0PKo*rPZ0$ z7ov}XeY}z?G9x1+snt6lY*`Q)i;U=@JAwgF)xIDnpHSzCO~5=EPa>fAVi5~Xt^pbV zLwW%zj7Z0zq7MbuTBfRi0pcSJLPLdPI9LR z00TTzX}`$|lkrvbW)jiEiN~D-D0pN(2H}g2(PK1)n)52LXR1;s%$9QB4N;5ld}7;Q zO^J~Of_Lz|A?dd<1A|gRss{xCS!%W%ur(|>>;1}3fmwLnA|ev6A%naYtDfP5%DllHcadyHAB^0bV_LfRh`^D zh@%MEBvvc;cO%!KL)7HdJ3HgDO5brEq9o#xF(%oIUKr$Op>@uV;%38?#K47{QXb>l z6*66guqq~98*&#S0eNsKJ`WCdsp-wrnzE7reADtPg<&|ET~mv-~@HF16nOCG@Hh}l$snT&ozcP#^G`-+6^{fBYgJY*=$t`V6K4FJrr z>h8815ywq5WK!LAdHO1a>*}%{adCl?8`iby`%1~d=H}g=VcK*L9xw_-=z9>U;%snz zOY8(ii0gte1jxy6gyyn1bZe;}L?1qnDynxQ56M1cu@f zAQkx5$xgHfwA)Bq^-`1Sjtq@kcWOZ6kPRk=hrk(ZTFq>|;fK}J_`nrne-8kz&o}|9 zKU4}&BCA4iyAPEBO}mPezi{(mo_5VX$imPf1YMWx$;~~Wn_lcy$k)V&fm`xu$Yi*b zR4_U-lMoPUFb9tCL7neHRYChg2#1~AF9|dmw^@ep3sUa^!U{0)$US@>+YO2rLJ}aY z%+fQw6B*|MY9SXu;Dlg@wj%g#hZ_sQis@lvLPhMtv4mkx8WyYWbeROm(?2m-gnPK0 zF!Nu4;^=JyBS$ZW_|cAff_j3?)fUKN4!aOhAwigS;8F~7vyApVL|c0BB{YCZpe4#`jQ;3^QjkW>P+u5MvY|&8ST}rdBXUBc*fW zrZQ-JxTTl0b{%I&et;IBxkQcAf)T(q?xeD!8`fI?EDX__g6mN~Y zy8KrMm5T?PuR@HYAO|dzvf@U{_}`(wm_Bs>IrIgRC(=sHviXuno2h8KW#czc2Qu5n zbs?w#-y{ZUz@6u^F~(T!$PT zSUgMQH;_uWN+`&+6KF?t;zWu%e(C0dTE}Ax?vGgu=NkRx-G~JQ4_TFjgoHXl%ec%OT>xXs zDl2>OYU>IN1g|aA^v+_G(H@nP2e0HjR=7bU$No8}wcyR_cWYh+Dsq1CmTBj*OgLv| z);32^IN{)+$&zbV70$Jg2X#+za9~$f(f;k*v#bb+I+N53l|fJM{WyQX>(`G`QYM+x z0-il9Kq)zc-=M_MB0r0sJmzvC4-NeMuz}NWAf0p9vK1T0!GmRU;<%<2vdCM^bL)U1 z0?gk1OEMQkG2G);iu7mcNOOD!(Ohj z%&vHKSiSyeXiEs+y0T?S%br;WY(BB+PJY?KM^WF5!QrCgAxcBr@t1zIE5v2Jd; zcwPd2rvgMNE>0xu4L(C>r_FJipI?%jt5GbJ2?*lTHlsGA92$D*?|%xa30zbHm?dN>gCeq%wZz!EWKA0_ z8oL&<6iUii$KHkPLPbg?Ns-jptFc8=sgO`fL>Q#+^Soll-1pq~@ALir_50&@JwA(R za$WE1{eHd9^Ei+5IFDC2+=Xbw23}i!9}+B}ln}yL);e;;yefkMWZbK}6^hK|2xVg1 zEx!@THt*WyRK9sj%O*_(gi2q93e~FN4)(wynpHclPN%}V1gBWfbIJMTVxc@=qH7!wvF%#A3kp!YX8`}fbgOkTW8`W%&CW6|?d zPXt+XTQQkZ2;p$6>mH;L86#mDfCflc_-)bG3ab<(C1MReH zM`~jns_9H!q^GZsvC1TZyL{rSGtbLB`u6XSufhe}ISDlf3Ii7L3}>qc)U1a7eUZeIJ(=S?pK9mg;9ld1N3GAjhb>ZsO4b1iZfVnI6Om%V9H!>2}YP`{|Fn)zd z-Q%n)duO9eNt9BJy@!pXX80M${Pojtj{*2MP!^WoKt* zc5}_@IpE`q!7LRb!^W2>S)l5j5R+-l3y3q(rcGm(W%Ra2?~fKjZbe%>he(RlWm{&% zBalQCh5Z>UghUX8=u&i5F@%(}_p}D|fc2A_+0;~EEABcXxO(srynOjUN?htbhECRqS(-F|QTS0C zy#tVl%!Lw41n(%iy5q+1ZU%^zWr$?{u8p?&S(~*>QgLz?L6Aym+>8JaLww%Df)bGLh)iUA=EZhkf zIm#q;w{|2H=^1Qk`E**?@}#&%3d&x21-&m>vqQw4ig%TsA)JR;4dhK;OEm}4bq?+Y zYN`blq@ZqA*U?p*6pVX(1Gx}*S+!=(M`K8{tyf4sL8kHS4qT8X10Hu=eGIko=^z2& z1tI!4is*9~8zz%| z%gW33I(NRNHM{jtYlan}GNtcC@}rWNQ-AL1akQcd+Gh%bP6N9$LWDXiQ^=*ruhM;U zTC~QUe@7}2E+G8IFboHyPWN$BvT%x&ntp29pK4W#v z=I%H0fB5sMCC)Xy58x!rca)d;BXyCW{`1g%t2S%?l3p!C)8m`g3xEIZT!Vp+I)C|_ z0~dF^c(Ee6VEmS>obCDdx)d#&yIX2XYRrZM9UXr{%hd7FnyQN#qNCHn@!laiy3K^| zQ@>6n>ZG&+k9_};5`)&B8}9Pb{E>wQ@555*07E^ED9y0iUT*yT!op;p`+5?C;08z5 zhR|vB+IzScS+G4B&<{ypNecYgkqG_e zWqPxGY~!v9__>Sx-k&rY)SHIs9IkE&jJeUv3qMLup?eGq{xmDbfE9I+M@~T_IAu!r zo$qLW2&*ktsenRE3;JGP3ZHNWK9Ao5L~GI$lqy|GU_)`&SLQbm;^F(F`9?m7m)~UF zUcF8p8ypv=lT~dP38dT3k+iK|Z1VY7JvwteQJ>zotTg&MFloa{!Dm18cDehVf--C= z*0q#PeoNkdU~$V}hk`1*#=OTjkO=Dv7A2#DQ9ljX6IJ!SjYi@JXn(iWJwF!ClISkilLPS8Jx4pKOtKcCO(wR=NAr^fZfF#^%c;x|2ZUnMbzdDFz&(RY4Z zkXhXB!=G1?xy!5{uA7K32551C_aHPL$DC-)n9EbU`|aF0;r>#9@t@DF8A)aGy^dHX zX5tP>LZm9t@P(m+2U$Gn+Ni4a^Jq}{jWL6l~bl!E>M_0+lZ zHM5-r)uMisRm`^f&k)}KS3h>C*WdTyY{jJL1^j?-tKI;yZT}ExSWRGg8<){M#n+Vh`39}Vq^hN2x4H)2_iT7b&;@VN%|pTo$Nv=&+mGpT5pYnu)R zmI@pVVl^qaX>zvXv3A{kl_aI`C9kG|<<8X?_{1yd%J_!s7k?&+BW_&T^tgIy-aAD9j8LZICeAN`{e!AqvY*~-Hx<#fXa;nO zLcRzOhrS?xX!y3mOVga10H(Jv}(R$&R^{jokZH&RQ?}j|mcpdBINDZ`& z4pHvSv&6=>ty@1ldD0&H6GTs)-MrBQ_C=MxtT-?*P&sByWoozF2y&S?(||QLOhZe< zVC`B0@(VO&qW*2;4-ypMCB`k>lhO%xpTEw<-}5>kl89E#*_3!mH3AZ!o) z#9wF)CWEV+3waC#6s-}E2Uw=Pa;vJl_BdnbO`ZE}U> zfSOnIHU6pXn0jYWGzE#h2iZU@>iAI~JaAwU+t44xA`oVB z5%wgRbY>_R+Av4S`+e7>$#7Z%j6TRIKQbK&UVn+EuvyfZ7w77U_oFX(Y|D-v-y|{T zw*DOjoSVCQf>}!`Zh4u$TwpujaSukA?CkE?0?WNt2(=FO<)8t&!CQHPQ4FB4FNTxb zB%sqhGD4Vvob5QHEize6opN1t0fG_fG%G|!Ms@-R{?M}J7HEs(JN+M5~fFy7%t zsTx)y?RNkP(MV_)Y7xcQ;1P|8inLT@SOS_!1G>JtLJAU?J>YH4`+$E@6!8X9?lD+4^v zPmaTWY4u^aP>0q;WH~rt70XK2n>y3 zmF*WK>9;$sL}`TbU=|rh#-d?NKvkRl{Z~QSzH|%NvBMPllUOzb{IIy~RXJ&i$qQBt zEBKe!am0ys{rWZYg>LP^Y^8#L(0A^`8oZ6PPD@1X3n3*Qt<)OBcyR08K`$2J6C07Z z5OaRNJb5aiqu!PFj@;?{1PPvohKxNx-?xUY`^XIx3`sC!GuQs;Qxg_xFnn+t0$`dS zeU^S)v@GKp%YjQjQ@~*zChx_5=%(J$=n1#Z4y#>DCYwaw zLTd|#25hQNAVWkT#|??Ng7WbtdJ4``6WV!{n_E{}eZ!h#z%IR6{;ixO0c5D@OOuVk zk0919s zz3I_AnMD9sh`JlB@{l)@w~x*BTn8}Ryko}|g5T^GPl!k8A$GtQHAnKYXYXDbzout1 z>`E{D*WP>jvQk~(0MuHyzPK#8>`6DRq4(gIZ0HN2l{uZ?i(o333)4tv2`?M_cStZy&% zMg#afJzdgDcj!=R&RJO}3bjj13kp_>y6gC59j~LN!0m z;HarW$Uu*O(%oOaETQjuu6Q&CCS)*zRR+EBPi=vG6rK_|tw}vELubNxg?5eVYmv^z-USg>be716?I&wap$mn0LE&7T@`5St#cXe?M?{b9jN`2%=AJ{4tI>@WxbvnwT zXNUfJ87VwlUi`yx_UJC zo`A@vPM!KFdmo})1)Hn7e8xvCUBJ)S1-nO%fi#d;kj+?){4qSBYl)xAyRcVeYF>Jf z_?F7%f@*c_zLT%rqSM3^`?(uS(DBNtZal@n)t3TB!Q+TjE5#sE9=))lbLX}~cgL2= zd(uO*3N0Hl{5?Dn{KSyMpp*sA23iO zYU0{las=$}H%V*=TFmD`o>b;P4<0N^T#-dkZUAhh@kunl$PE?$SKtvK{MY|zn?1x$ z1icrXt$z?4mt1-^DcknBbZLKD=X5op(aj(cnGmZlJPH^{Bz>@~<1i;Xfrr-Nx{KZzfk!}TNiBDZ^}SO~94c)4Ey=;vFVODCBsc3m1lvX7S+s zK>H?F;F5XU5d2EP4rQ_`8VX zJzEZCW4V!Q@ZrNhwrDYuj75!cvS{cp&bDr@R}i^tkFVNKLuNncv5Uz#Pp?|v`ID%L zk+F-&Oye%vD{NS5?&q7qJ~w)Ii17hUVhvDFP&j`B4Z+-0EhchP!D<=;w-NUxHb2Zd zP=65@;{}6F9hSTtn!EIatH|a@K5UBX95O07Usl@l_VIh(&L!>L!qx(Zi*0jcUQ^EZ zg7RK@c%SE5I>A3SYc>K{KBw7v;zU!#n=JHT4E{*|UGqv_JGd>rapdpnl>66WN5^KV zFShT+WC*Qp6wIra$9h=mz@*k?`_oKz{44*l^y#Y2AMJt0LOzqun~2a*{c5;j_3Uqw z^DODsQ+eC}op~%o+FSKnYt-m(7IVC4-qf#u{P>pczM(cTNro%6f4ySwcJNqr>RUDY zvF9>%ZTzOWEV_F;C*P``j|mtqx|#31>G;ZswMhT78KI+uxl}EC7qkP<^ONmMi0%7x zO1gZ$&dNxeH%Wb1>fnOOXfVhhO-9s*GE*LCVO@1SL47)S)D(kh`g%iI7BnWP`;c<2 zOrAO!bP4#ra<|vN)sF_pN|>A$SaHY%RElsRlqwvcq{ z86lP*_~V?wfDma>s!b%QALcIpe5Z6wFNXU1*{*o;>Qx~1lj383%St)Zpy$jRzR+&; z2c^TVSi^S5LP<0%d<&M88HwYpYe3)mt7ut$T@f-9a)GuP0?BO*!aT~H!^bje@&ht? zl=}0y;b_0Dbx0sbHAWf>6zRuxQpt>uL!~z~YN~>;{h1 z)NiJwstNHPo!BJ>JTjV9*Y7Unnz0xVDY^n>+rH90O%-B{j8sZ|#C+L8>=m`E>)W@r zlQ@h8QrxCw;m3D#8LHKonRTK)+K6uKIZ;}oBPffQR9*6SVkmWiES*l+)_1TDwh;n` zV^PkguCvq#WbeqB409E@-dv-877;plV=5%4)W7P+&`Z>roX`8g_M|J1j2;9BR-Byp ziRzv`uUKE%M3+8L^WA$Ek9T!lmmb`tDiEJKy!NWCDW+J(X0Qf3ddz4*jU`=jN~N2q z-or|`@ZUCWtaYFAofbzzQm=ojVurE2)H8;Nlaiw+5!1zCOu-sz!~ILJJ>0%(8JFGA zKbfP*^(r-%qq}%S%)w59J~u*!WtA=R$oWr3(=r;(nVZxFT2ahp zx?CbFAUKh;W~Bx|0z-k++tcQr*5%qQm-r@6J;%5;upsQWA#_;e_2|yX#_KMc-yL+% znH?lY(-MRVB4WLv^l9Zmg|$MlIvui^e$U4e56uT_(=6aFmyglK~z+8TMXw|=H zJ#UU?s?zB5bp|a`i)z)^{Eq6xHY?~y9y`fR+`0Puhd(_M5^+ zh%)Cv^px|pX6##B3o#kD{1Wda6)P$VIp~2HOtm5s0r;8)C{&V8o(EXcVwo6 zSSyR4>|H1ZrT-=8H?m2R&tgl+VaAXew@^n>Zc5^!EguM~pgX>k(fPI`cet615dwJa zex8!o9gm3Au8@4iUv@v{UQdL6qvrZ#j^=3Ytf! zAO9#MuXx13rB&~1Yz(s{YoB<6#BD$;!Yw-|?|d{xn_q2l=PjL(P2gk^_PlMqOEnQXa$F(S?6S3HgxYbJ0s*pusCH{ZZkx{+FJ#>8QAym7A*<9UCj!#)lhv z=N_9ln^_LZb7o%8+4tc9V|n?lK0mnkZ%^10Nem*PFWH0F7`fdXBVYwbC=47n?2>)o zYbc`6^Dsrt10lo)P{1}{<8LUSwraybEG$^`d9hy19Gp>o3Ote!tjK}WPP;qHpaRWo zk^NES(+uA==j@-4yTtZ~sSqVmwXxCDN5LUT{9`-A&Ab4kE5y4+^w_j)sZQ`2Xnglj zZtX4&`?+u0BIP2VXx?*2$5T_<1epRR4*={|YOlJIcd)ieRcGN#laNOi^XtCVEl>Z| zAuPN9KklN2e{|^nMGNq+nf&XGmQ6dcu2mv#FKReNr9EjFDTWeIPxLOMW<>BuA>J^|dg$t(xtJo_5s(ED+ykM22wRaV zh+V!^tfVnH3K$(rGbx8378j?&aWY>L1d@_14B`z;AZ#L&Mz7qKBj(UC+=Mff2?Wl+ z`epAS!Oj}$^;>b`#0jsZ_5Hj?RjuM{M|{W0F}^Y!ses3$s;dEQe12+oj?G<1>*D0e zzS|>ZG@b7#4&g9L2qXv68>3FsaKy$?gpDHKmlQ10%F3vPPhJ`Z6jol&d!pO!8}mU& zV@`w&>6AWpsIA4J+YYWBe^hm_|E^osxIt6ubaE|hePDb_hRQLktzo!ceZxSrpcYv@ zn_c*!k;S$ljW>3@R@yNC(xjoE+6_OR}!A zyH@?<6T?rw`SsUoO*agqy)XuzZaX@B4ly{wb{nBOY88h4?ACqzIx9WlkzO;OJBzkV zQCe{7-kskmO^402)d)mzyUS?taajG1Vd)*VBnKgeaw0Sr<;2I@KdvpG;M?QDH_45V zHV&4VJ&xpS4PJstxb`NO>j@{uIALbbH>q2{z6BfAl7U9Zqmgti%F4>pkfyw*Z9786 zF|LvPd?tefb?el*Ntio=G|6!0jr(i6z1aQ2&BJBWa)X~XX($vemjC|#L-coXQ&4&w zCS?=>M>>~#M`w2Q@u}?JqLG{P%Zm*oe2rG|>ksN)nG#ob-k(Jguf2C@YkAN&=n?|F zZ)WUyB4IFZrcflcADeVch7hv)^N4uusoRhV0eLmpRIQf!4{nPzyQ=@-dp^)t{kIe? zbZXXD|LFdUca46ieW&Y^y$ORAvzHBi(?Ahd^Gg2lpS(TBc&+AFKGo>oHR5;f+z9}J z^K&un$kzH+{dcD|lz)W=-Q%h^Q~8MH`;i|?dJR*5{=iK0wso~2!#QDYHI12V7;^KM zXR)pOEqbvZ1}l(dVbHhl1av1iD0LcXX@!uBc8+X2ZQ8UW6g>z%Z$TX){qG(=DbYc7 zn;JHOdvNi+L!8?nxZt%LH(HsRu3-Rm0!f6C5<=YcK4khQZ=eEni0^FaJ znNxdIePwI8E)@$efQ1JOrXtf4~3-EX_DwGYTGEf8E7{zP^f@wQ7w}JwILC z`Ra~tgZVb9ASS|!^kznFL+Bcl`1sgG>Cg)Halb$x30M$OMVcD>>hll7^yzWy=oig#1; z(%w(PC|~vH@nfr_d8Hq!mY*u+ngUsSP0|RuGFg*#<4)SVOUEa`lU56g5z#t-Qzj=?=uzTf!xpN*1 zFxnud^j~-Ix^i*1kREu#%S+Ac)~k1m-X+tjgoscICp&AGk2=PCw6pW?v-$Xt%Z|t9 zE&aXK&a2ZIOwaCL|NfyD|GGcp{%F~%YHw})&-R_ZqkIR2V)c{Rr3$xc-=5FN>|f)4 z<33aIH?#FM@~|k3_EK+Zx7Ek0tG}C&@g&7O29 zCDcl|OrC0PVX>7a>rvFlBRDVPJ%Q>qo+1)(?tm!Euk|mzl9OXkw7bJZ=?O-`I!!n} z1hmMZtOd6L$K`}YItG66@$pDO5Aus0jE(K-u2@f<8dSg5a!SGw8eqljT~83Gt`gg| zaTGcy$O05?MF{^k^8^PD^xfz6yf06hl{Q=sX~);;C?ztPd$fmxrTQDS*nO)}S7?ZK zbo?$}xbPH$vu=Y1{C4w9{fRh70JnLZ@VK;&D@&;((5c53c9B4bnxO^*J;v-Dn zhSF~HEDn#|x_R?{Ae0l(7+$^;S+2TbSp4Sc22! zDE)5T#?cNed+}l{qRbOm%VryQ&OSB&utWe^gsQ3CciuiVf5y;Z!x|UIdVlaZ72m($ zxOx*f_8$HAyX$x&>$h*OR4rh}34F15@#0x5TvZUK4?Dn(o=oW6p@ZDPt=qRB=JuGX zm_0A zy#zALHV$+`yO!z{<3t8KZqqS#?lkoRcC#NH+ej?>-g7o@Po@kTX=>_s?{1E4?FPLA zvge4V^=rw~gorQ)N*H=Pv(fH5Z-~!_?7XQxo0S6ro#AY{?;IIAE^rk>+Yr`Sllb9W z52v0<^bg&?e<0+|UT(0SQjiVES`MQkjg5w%PRHWeoo>j6mL?8q!S*d^0v&*o;yQFG zu4~q)AP8Hm($&VtN9VlK7{1{><7Xm9wzXZdJR0BHSwz_SIzQ4{kTn3>NXN4z4|_|GiP zOH9;mT@@5#tM~t3FvwrAaJa^oNVol6BUS%dl)I-H_Oax7WiO3f^`Ln62V>vt(#;9g z6|*(U;(C6Gkz-dKtDfHvTi-%hr#B7MZ{pj1rM6b(e3&M=Bf|p65rI5uDm7}_^a!_0 z)*)x$GLw^qb1C0evr(g9DA*Br2C0Hz%4^oFSqOD|i+W<4pWgvmPBU>96i7jvPE%7e z2!+cm2>P|Z{pOcERmJQIcCSuPGSwe`|E|8{$Lmy(=}<-+bc`o5QG64Dq<|cflwRAm zPLPMZk&2DRbF^S`&tRkF*s=S0nofLe0YCCU+4<##w}t~|CZ1Uo1&tTN6&Q5xJi76^ zp=j9?mq*u>8=rJ)Y|^^`c5{cGJuUc7a_}x=*798uH5-_iOk%@j7+C=m^ptc7S?1xo z$+vEeXSduzj{%w2MJ|)R^f*l8^c_Hw=1xwb34@aiS>+$SelPU9?AGW}*@i&hmnLYR z13r~U`V?YMeY1b*nLxO@BLpY8z2&slifGxrzg1kXZe5p1d`)zd9lLIQI)z2f0t;nm zPNK2|*B^guGiOeztx0=@Vs;x)z1M1A=dHVTr6e}_N66W`deNurVCAhqNN(N+?fzEi znL;Nep4FWNJ~~-#=+mhbbpHHJDGe9y@k3u%euL{tEY(WpkU7#FCahX<}1R zbM*&W|8i_%xRqD!WvZ|C)?J-{+gpEpUv-W5U;N81`3nz>OH#k@zk>exu?rP($Jsp3 zw5z$*YjJ#`BB|3rol1xI&-T4(!1mla=vh)+E&4px^X8QqzRd%#Gw|cxyiGNE&;2e3 zeCK|9a!%e-`C+EMxZr>*W^t_OE8X7Hcm4@XV)WXz zD@F*=%xKXu2!V1e`HC6T98a0VmNw+9YD)&zsM@Tl3eu^n3HT|FT|qlcfp%y6Tao3ind#n>;qL0$_{5fPU>37;Px$B229T9 z&KnTkR9gcABFUDfDuPKnHro$x@jg8Bxze3BeR>B212ffYCfY`*kd$x}_hk3-VLKXD zZO3Buc5MES?U++gPYca&@Y!ouuG|nT#~Qyfwfhl@Ka~W&hZFqa5-3)yx#CSXT++FYFVktp@U@-?xZ-tys2mFgg1h4N+=A&Y}n z1j-d_2T%?%!{W`lIXy?aZl#rl+>#Va8;}0aZDM?Iq<}7cR8SDivzE); z)%BNODX>C0CPQv6QfqUta!)$11&<#$ZvNq`@WAx66P#e#+qW6m_4S1Jqkfn)emxm0 zcjJ-j3+sPfVf+;fUKuv&rx<>*iAh{F(Bo$opu?4du=upM54wA;sG+{@J`Mk)eg9Xn zzb`(adeSz-`jcjqe&=g@pX}-Tqgg=Ct=v|;UD6;@X9FzyjxGGA>fQC~YArP7cB8{~ zDy_=eZX;T>IyH<0N=UpxLbSB82_Z#xvN<-8oqG#?VVl64A@c00jJNFGy_>%qO6N%C z`ckf38H=R|ipK%8c}7VhL>Xa;kV7yuR$5yj$xDofsz1C%V=B6vO(rdA#i|dtiTW9^ zmC=088BpZQmp6Ho78Vvk$XA`9^jH;X^h@=WWNO)HVhxcq|GKd_kkBd%1L&Hmn65rZ zM?7CmStJGgQu~r&R)SY7VV9hxr^GGeS9a8G1gT1M*zDU-bxLzcx@}vwv{9aX>k|p2 zYsPSpgJ+>$1;<_AgitQM(UF9Ux(u*S%vimqdR$Y$kcJv1=Mf<@*barz(B`C47gy@j zZQPQQ(!9izaAL)uu5h4^plyzR>+EnQginhPY-XZbMx6f$4VHoi#^V9R}*hqAP>8SE-o3&Hl%!oDBE*W|v;VURC zJOWuDm!wKetaP_8g-@V-@c>!()@H2^{&y^?%#Db3ex~u9}(|9(J*tX9jIQzN#)y4Rl6iH zoO!7Vq;%xWJt)X1V_+p?iR=_6LCaV_A zvc5#c76W4>1?Y$oBhq*bQZorF0=Rj9+2oU?0x98SSCijka2MwxUhV9xz8c?X{qZ%| zL%u&nXCchEls(^lx1Yz0D9#iMjD2EggJ;Yjb}U~r4cD9#Og5t264vwsSdz&MfXlcOG_~2^5x+|LO>7cCUbtY z%7PXlOn9T_8ITRwn>Fh;SwQ9ZhiBDX1^NOr+Q`DC>nwRONXdC}dmlY_nEt7bqR%pu z{yNrHB$DHu-8}!wOjZ0%!bqbF<5hx$K*{>>THFJ%0%s?6;5x6~IgIlyaNq`0Z$}Ut zXJ+NxyXO)aZsi;`cmDkOV&EAL488r~)60ppRRUk89dBNLm2Ede^{TOR*xBAuj>N_m zXqA*naC}F=1*#xQqc8}JwBX9JMVx8}K9#MQ(Q`JL+z@Xog3wb+$FokAG+YhX+N#%- z3t=H4PixIk@6v&p{;%ahIcvGzwSP+o+}bqEqTFFW)qybxKz_V4}q=hT@5Szq_6q zj_3jw3y1yQsH@Bo(Q;$-Z(eulmAx-PqmRM0x4(a3LaLJgq?~PM8@{TMT#4zxSPDgi z8%wj-Hc<(y;(GVkAV{)1)Z-Kz7qUINRNb+fXdq7EtbsF{Pj)F&iLX(EOK4cS^1`yOv0jqlh%Z=AVQula9)~R1Xt$l3Nvu(Fo_Z zk)JSu8i|lbDMPb7TSHXl(`L*tQ;F@7&}KN?1P$O2=?^0ts6VIO?+w6yK4~g=usBs29RyZNu!b%U!&2GY+OLGLyX}6Zy}t z>`~kwp)SUL9SrAj^jme+1KVY}Ikor3YO8|iPiK3KGd0IRTUW{9&@7T=kl^}C&!-1g zf9iR&gGI=GHkGDxF;={6`a@KlLZX8!(~xlO7xt454kWMzM62tLMhd%@>N4y8W{X@kPNP{E8~BQg71%H z4A25CrX_BABJotJ;`^I5lpB>4^RTIw&q3C;G)c(udP*S?rMt@($lUm=-I4AKCh%;~ z?%zKAJiBGa7cIb;mMd4RIA*D?3VfR<_3|}&0FFdqW{HJA%>LHI#6$*%0wJV|nSZtb z&hn414o|rqIcX9DSeESUf(H){&;S#aYQ}=f*O|t1F7{N!|KinxEAzFz=vYzFv`(M; z>h7nH*J_P>nsv3*#G-7csylDJFu3%5i_g84|FEa|*RJ^Qy_Fa3{=$k$hI7|yt9Jv9 z{HT)%>ZG2b<8kVtJ0u|Y*PuLshZRC2Sq@m=xA<{WJzZV9g?7?T0U1YrIGvaMnJpPV zMZGHT7wz$^y^5&;NLu%t4BAF?LYy$Zly)&j9JGXgqA%qwtY+k5rWf8k6%MVu5wLorE>e*inn%gueXa2M1$29PbR^P4~$QdcMX0maPPSOMF&u?xh5kuG!q zG}$5a3iMF)VFKw>tAeQO+d~-g=al5+aV!vc6)E5{9K1%Mo=*H!wmnh*r=LW2V9q0p z!6gt>36QZStKNoOnAUygR#G?5v2dHU!%a{Q;1luq0brLCi1a-MA)ToI@Z$?X%lZ|c zigUo-#fDYLcjV~tBG){IJk9CD2V^v!_<-mt5%Jy@3XyOF(F2Nf@MMNaPbn-CdYc6Z zLflKTEIF(LX~brd`??K2r*=fq6eeSh=yK-8ui_C+0y6_I_VaulBBdWXs+oizxgiJy zBM>$UX@GM6j{U-%V7wfkVk0WOTPl7`d+3X{rnsGjOiG?B#rz>|@e$T=6`4~5VCzA1 zRg%2eB<}r3lIMHH@zhQLWwY~ec{`aQ>?CJh@?|MHMrAx{bafB!( zj=W`+-J}(xrv$0ofQ8jnA~`(X$oMl8^Fs#JtGUVo+g4ebBo8afa@5;;N}l>H)CWS_ z0?MD%veHn@?o<^^UPvry_upd4bIpb?&oonK?8-402_O#f@!OByyysP#Fk1lbG?t~*)+H4zWM{Uzc+~PVL#gMAcj&2? z>$L4=nqB2=WYlSUjlHa3sfji@0dE>MXkbMNRgz&Mi8`^OWKO@i4_5%34+I72D$`hp zk%UgWxh(>3+BkjRnyQQxg6Jdn%mm7NS-AXd$bujicaD38J+1ID@qfN+giQ3gZ=O9=uk^^Jyy}$t;GA_15YM z!*lPP!sF5#8fgke?AqTaW=;5}osgCY4Qyz=-(>L_Qcg*W{!N48RbMri$0qrH5q^BG z(@qZ6HSe#`zc#I|7T9t;1_jTws{|i$-`YKFwNE~r z=>NYl!+$eGsUPe9X5gfC_8Oc}Y*H()+P)^Hf1!qqt4BN>*K2kAnR@v)rIVSr52J8d zi9VHv8(=oVs+uLUC>Wb@j1Bl$#nsjNIDv;I!nh<|ofCHZEmpyXO;7O;3)~B3dIv%TulB`tnRXZh4-vK?5+X;_~ zHg9^=T~ecTXNVQrecT4J97M^(8*D{e!dQl_@W$aj7@AS!EVjKF%aRJD-X6@R}tSfcgm9xutVXuklfiAlf(CbVs#O6P!gCpF^RjY*9#&&s+w2_g) z4Iv?HrIUo0%mMC7d)l_dh~QiwnvJ^#c%LTxl{2_jMD$d3Ze)dSb3gk5_&fw``p-VL z^u~pXaCF=+`~;IL$=TT-Hil~}s)(z@xaZsS_$o}y*)zUzSJ?H}bS_L)*k-q<7f%7A zRMGr%E!mz;2sp&v(QGcmfB@mK=H@ES>T~8<%)|~wFjRzRB_Js^Bg0ztni-JHEXJK8 zx!+-w0jeM*QD}NiRjl81ylx~9rd+#rhd%8Ijy5lUp7UY4e>FwK?yBB%2kAXG{R)Fw z&*-(=%yr?~YQvycIGCXe*T)3>IJjuhn!SDX>eaNK4*kB(fBbm5bKydnQVC+ z*9Ndxizn1oKlBvc!;5OE>sDr$Q4?r$XfchmDDBcd#5DMRSmOxiKZJO|ATXw)0@k1N zZah_=d>#Zc$?+bXt*2ya(~=In?h&Mh@5$u4NFqk10aEoO7D=_IM$Z3*+CCU$rk-e^ zT-tiKX%$s4VjegKkD;5QmZF=^EU8ks6uMcVRQl34MzNY<{1_h%O*(>f$x^|g00If; zL1x)u0R%K}X2)pN--fOhbwPT#t+lG>;d9`&G4Ftm;-|zujB3Zy$$EU4BRwo(* zsj&eKQbp@2+05HHT7b1yez@DA>(3YGysFlsPIkWBwR;_E6_;M0e1D^yl_2XI#QJ*e z<#8~vC5v@@qbY9wt+T8-+|(|aWb%$QkaNWWD=MbG4Oh*O-kJEZKvxL=>;IlBteY+gfLggx*UpObJc}sLy zMClMH2uC^tz{sMLz^2l=f~FJ39ZI>7)fQ_%p2~qj$e~NZYZdv~{Na0a5qGD2ZjjaO zY@LMSm<+#7>RsVq5NCn8q*Hq>z$8*4DC-5PTqY`B!|#MbYYW|iZ(|_4yFD#tI8KLR+7heU$j52Qaa6SE3fnOJ+D382ZI46MTER{#N}K-LxFg0_R7 zZhFYL!8FxXLAb1s;Fsy0;5UFrd<@@bYP2sgXTwp%l3Bmxu_>T}dTenZ-z6j=2LH08tAy0aM z&n8!(qAl@3e2JX#)8P!E-W9%Sr>1J0KYxBO^opgO-5!Z~N^>F&t4?%;42|kTuSnA* zcJ^KoKERfdJv;D20j^)#>ZB(9UQ1y>IQ=3LX0<|4{>x7-%S-bnft~Ik(V7-BG4g|_ zdX1;_+ivu8Yx{n~#Kc)-S}F_X*WFp+>~rDd3>mf-lrF>&eF|CaOOd3US3}AFN)qPtJP9BXg-Tqgpjr`{BX=n=W}Hl!7r%4$p5aE{j0M4 zTdA_pD|ZR`9>qnr<%g1-cxwrV!b+JvkN2ruD*G8dS)5*eP6+3txR>TutS#+bf6_wO z)Y;n+-0mOwH$M}*HOIq|5wDp!r=36i`U&e(TX=@p-aqxnUF850U3n$7zw26i7#ch} z6vAk8R)&6?HiH3G=}@5VKoE%%76D<@=0t}QC zZn8sBRM7{2ih=sAE{PuxvNlp!PLP8!7o%v03#CTRMM`~2v}Wui03iED(vDE0VlnTH zAxdy=wo9*K`eps3>H~NuB_$cWNoX+x{zPJv?8tuovK%U_bh&+M#+M{%(P|SoEn1Y% zZ9N?5UtejnNm?!vLO>`InvLH-4fMn?Go8+zDBoIozxJb}9!^xkzrtG;B+I;4FDrJd zkePH2H9?zvhe87D#Is9I;iMv(o)F@MD(K7^BT>`I&jrPyDg%>?KtQM)={h3cu>HgO z^jqq1(f5);4cfMSJm!PL#vYW}RBZ4v!@_5(Vaowf+-XX=HRrPw zA|Vm1%U$fOq!$eG&~(SEcdZf_N%mCC5g z(RN>jsNUCF)zyL#SBbVoN;T|)EZFVTHajO*0MSG{`hgh+gt8@#Ip56pgi7-x?1}!WIJw0+#-kpUa z^jk3M?b&A&KTHh2)7-v_<4n)OSPrxIM^AJ>RA~5m2n|{eVo09ZGX>}#s=n0sfBw1c?zGRiue(iVt+D6WpO`+h zk{uzfZ8pdW99sG}Tb}9SPq$Y)p`BY;lyQSEa)Pgh3huVrx9q`8QOKj7bwW6$3PReD z-M6(m&ZP9Js}+n{dxw{fReA|W6A!OiRZ&vSDmEw@VbZ)R=Y$!QG}ZcR%pm`ps_N*g zSekEb+Tsry`3I0C4uZ`D>zY#nmT3Plb|epq5x6k!gsBSkR{b9oOzF>0Zx#3>f}mdkE4IsV5VbSh#l(XFvnm*|`?{Mc<% zA^~o|raA2lT42+vDo5xqYOZ3YMUGalRsY!hAp@0AvauVR_Bdo?%bUAXpTpid5tbt` zExEI6`}P^$oEmmCGc)rJ0tZ{*(5$O}{Bh#a<;%95Vl+L51<4Ia>4$W=|A5+C?Gd=% z;d|Ij@+vVbrVV-2syIA3Pk!RV^b2ntL(omVBeRqqJG4{Qrj;#%x4%R5K0d^xOlxC@ zZryJ76Sot7b6?;2m$H}y9I)xZf)S}x_#$04Ztk<0ei=dER3&Slx)J&>+L`n z;>!rNOn~310G}aUTq%o&6 zh7Z;_zw9Hkq*P`nD>xd=lB|}p@q3Nn{@9p{EJHf<7#nQ>r%FkvWRx$RGZ@>;p+un* zV?s-IrT+QxgGb;^{pG*!2gt<#QGMDbMC}xYla7{5fAy9Hwgt*IytJU3Z+7qrBmW?f znUwK#2KsUC!Q%R>9}|7(<}1^f`Uu4jX8rXkefFl6-DM{7dVcnr8IFCr8YH^at9|>} zFN4WzMak^~ZUbH#wP;~?G;K-nnS@-AB%RqqKdEjVTMdAg?h|407TP#TCvYlFiU;eQ z|6b@W0?!d9#}9Ir!nr%wo^;!jT>7dH&QbC?+FA{Q$VuA?#*kEw$BbPgw7Cy9`Sv-Q)C%S&|f95lPOkg4z( zyvCGi$eb-O3y0uJX60NI2q2nOUCw$R-_QH~35MF|q~=HpMj!_57XF%kUx)tvZCTxL z1Zwn4EETQ2>I4&ip!#dG=W7I+%@{qWtq(fV=Lv5eI50l?u2$0Co;adu8+%%$1wT10 z))Ypo2B(r2@jVWv)pFyF^S5l)%&q?F%dBPKW!@r1G>IfA6apeVM7!~(K^^_J(A+-w zZ^;CaC}LKvq8=C{s%?gbP~~b%SRqY9Xvvph>_NCFAXi98h$+tByExoYnjC1w!%!5x z+1EH_4cRAwqlekOJ47-9iszib;T*HT-InPE8qh*(3-$-4oOo;T;M|DRe0K9pk#~?2 zC}Mb^9l1vvj8>h0kL>6O)s$$PPM(yBi*!BcEsC?YolCn$B=b;TXC!{Iq=h#%>J9l(`pRoJZIW}Z%F%$y zJ|mVN8y}=l#PwR}C6bAheVoQI3Hr#IauObYj`Q`Zma1#-Ve}A1v><7i0#by1(z%x; z3ZO->VnH{nBuoC8NWBiB@oSEKo5_HHjHx_g#)8%6%;1M)OWA3t!KKY9fVSk5M13A) z@WF$v`Bup+2HsR8s-g~Kuuy4E6IslHMg|cCzPca{4ItMQ5hjs z8WF;lk>D2)=CU2G@9l5RF!XH-D6}>|N$T3ucRc7>v}M%aB=FV})3~{bKUR)ZDEer6 zIC4wkIpTYdI0Z}yzvEm~%J)EkP{{Eqx9HKtplA?64GD!x7b6sB$Y_DHP$%tZYs0Z+ zWdREhQGzH%HHYNl;pB^A&83gu3xoc5C^$8VMKY6!=*XNtaQLEDO3`Y-_)P2BxH8dP z;G#rUhT@!Pw+jB2)`>C=5SWf~4vDm!F&Szb8zHS)Ff>eTF(@sArF)pGZI7%6zs|7$ z9sFr4XOgc)xkrkfv*f0R!fo=iB)+J}O`@n!1~|wi9{Pj=W2?FZm!6sK5ej_}u%E85 z=LnR9u)`X10RT^AoRmF&3kr8OLWnDL00dWuh7C7O&OaO!6hfj==@P{aXWz4746eOl zV-f;4J39{*`7Ktd$X5VBVF=eI9&=oVdDT(!1x$bA-K%Lfuwkq8ffQ3x_AsDP43+;L z^{l9mxV;kBozRiXlq(NK+RYg!Agtn=iDENpHV0IOW2|ie>q`X%^1ql{SuwjhrM&s1 zNx8qjyzGqFLV_sL$AdOrK-k|N4xN=j(B69Ym$?(b5!=v?$u%unq~EdQ$oD?aa=h$r zPs^=*`L^r86?4Cv>SCDDJvFXTrx=T^nwvsmjQx zhH2kSJZ^91bN}=aZQZ&vN5zlJh^fA>Pslg(zPs>e#e;saHT2uA_FHV+a+6NOC%cNh zuDW~wI``ur<)_Z7DEXi*BWTyfxMo{1)te!!31d&s@bka_K1f0$N8`q1*dn2mZQ>2( z|2{kK(;;al_Cxb9vHJ*yMFvKrx!EAyBS3}n__ICu(Gq#B@h?xftd4l0p&j=2oj(i) zn%%cuS01hk^mfQ;;ebDnpck8vGLFSAIV6-uUCbMI505(_W;v(EESnCMJD51fBimI* zq?Z1f%9Ji9vJ*;|8+%dt0mVkB;67eMyb7BxlMCrGU-~-DjJe{Y88!;ViJt1!c-4uM z!bD7_r=+wJI#$MerP5-JmFpaa{DK+xmoa3S6aw3VTFZ@aDk392E4F_!%?c z{nIugsLrrI&%~ZIU|G>L?DT09ur;y7iz)EE(Jo-g@!W12iv4=sb`*QfYs`k?cK0~& zSv+iG`~ypoS5lE*lKKw{t4%ekQ2jAa-OpN+^ft>h9*ljL`b7(%{>7s@b*f5r$8uB0 zs=pdGyqdZIQ2gg_`~Sbyzul|rXHOy8?Hve<^S+>qPkY*+;ofwQU=42Jaa9iamXMn- ztgO%bTLBA5WFusDgf;eBNO?@MGZH?DaE*SCZeeUZkbIB4Gs0Q_o9vURH|*_o1S_tGW~SemvNnDh9juFw&@zb|2m6+HtL59hM>66OYx`3+n_EFztI^ytB4QiU0?Ruv=^FQ#bBblt?!BM?Y8LPf@J z4Ob3GBSIX=(h{JI8LEtg(4CqId#4JLQbuS=?w_djxb49j)E zkgVXs^!H+H3MJ~FkdVM(Yvo8Hi?O-P_;HNUxG87Ei5*&e^MbxF-FR$H{u~3nAU5z&oE6c#n*p> zJW`Opqn(_9K>sq@$YcKeThg5`@hWesg+<7F%C&T6c~f7>IH25Qxag-cc`>4o)`nr6 z7GkF2*msMHd6IW~5pOimY^XdqEtE76mzFHB zv@~;c06w?)R*_zNxp^|jE1cZ%pR`3E|^DtA&s1 z=;>N@=+Ptk)M0+y=>RgCtHylbjT)woHc_iS4fNl;w-e((wX~yyoi^II_FZ7E>KT#R zW!G!>IgL%M%ZDE=+GLX+5Po^>3KNCmB`XIIHH_j2l@pd*rRtc5@@N*aHKeP;NdPnzvFdFE2|E$#v;Ez(^l?R@sMiO z2L_@ky}|a3gte|ntbALE&wa;)cMsR8I=5?0lP*}~=k!A4*~-w8Ql`8{j96P>g2|K2 z9Kd{^xiuftyG0GElvSxJu2;JT>cAxrY0PHYX`2 z?gz5W;v2E}ZgGTAPq>*10P$_2lng>5mK~##VpT$7v+h&ZNVgS!(^N&N-jn0Vgd$^B z;SM{w)0EY#Ryomuz#2hC#DwQIh|XMc^-tJbp@oYW4ADKpf*_!cXNxmE{u&KI%GB;k zm8k8JK;G7x!U_!KzA(%0j&zc?;}y{$%LFiLh$PjHxGd_iNZ^;SktmmmdjiWA^j-E6 zSCV@()%17I>d$|!c4DkVRQME#=W_ntQ+4X{sr07sRBl2OOxI!a>9 zI6?$IP0d9)icp{Rm(J&6P=mg6cJ~x{e7ow_slR`vC^)KBtEQ`zMWUNb9#Dyf0DJx- zzu`JjMgVq^?1hDfju4p4`fut^9~sZT#xT}6!*;q#nV+KODCab9DWOlllAfhh zr5JHFyIw&qOBc=wQ*)=1NlRX?B#4&>*uZM}CP?@WVS+0<3fhdfpnZ&xieVaK zS-fy1{{+q`GCYj3OmH?Jy9oQcm?v`ljG!0@1s0uk%oBmh#sqf%%Xqv`Oz?~mQ43yx zPVZhkIE_&6QBl#vlG6sS>esK2wr4MgIhE62dPq{FR;ntTB38mrLIyHD(w_TnMNz4fpNtF=L~Qj#3I=p?{>W|MYY| zkjXB3#Uu8q03?z6rjFidAX9GAIF=b%`WnvbqpQEtd~MT8mi`y}b;G2ERd z!GLu@Zj&fR{k~=0y434BZ>tdK{cr8(L-cY!xLxH3MEcQ_njF*9?W$}1@x}R~+=O`D zWOgToUCh$JGa^!W_s%BwE~BeQc($TB2hFN`eM-Q%ZEL$Ih}Kr4hk;Cr6A@QAXTnV0ite3vCBZkR%1cj(k9 z;qfBqu88Or7hHN77#Z0iKRfDo<_c&AC-j5DHr#o1Q7TDV+Cskhp9Gy33bhLpdK(K@ zk@SNL{2LCiZjSN|`?&qU7lE|zp|zMpvI_W4L*YAY-n1FV)z%^G)vz~BTMZa+_NIdu z42t|(xk=?K6Pd?nyk*Ol zauH_2we!yqLybAq0po`^iB9TmNwm)&j>RuL8Y#N$(c zBfw3b<@54l6PXDQi;J^Czvd4`KQpswnRLAg-N=2%#&q*351XX6QWA=Zu7wyt`S-2weCL}}8^@%dnjJkLxeldHXR7Y(L2BXtQrZ+iA;)!T4KPZ!#0PbSz_!ki#%whXs{DQ4%g?AVRGpXD}XGT%6A| zb#x-6;|FdQ^$dy)2B~i2?p$&=YOq8(rY{BBaQEbavo}O(x2spkzccRuNd~;;Xz4>clGGVO@ZGUIpoiL=bje_@^2e3{LO(a zs%FzG_-pB(21QffIbAQ!jJ{mK;^@tHv#_J>|WB&w)Xmn05Cml(iG>70b#a{qC!n&@yYDjw}@CqmYMN;PIu7=^VzO z3KBabG?7HZ8ys}-IzF#rG^fCcq&BQ%m@zFOq?B#3i%iQ<2}L|r+oDA;X&j+69#-o=iLUEB7Yo|@YyQZg*Jeo%zp?y zW-97&Dlml7!6YlQJV;-&ys*o0X)=_u|4=~ggsaSP(XDjKIUR!xPz-2fa^+G^I;WuI zH=Fi=`Y9u4)+ypMENTjM9gLSJ(z|k|A#L~h@F!L*!|b-ujzz5In2kM`N&U zF%8&^xHc#?c0v6*bqKIy5z*-NsK;gBzhqzh+{xODy)04zZ=nO9yO~dza0EA7dt_fn z(PoWzZRgHf;!pF(zl!fZo%`0zWn@xgzoZ)l=Rd%o*aRk=+*RT0>+7`BzHr{-@{|XI ztcz(_aXN$32~axbEU(*6mxAd2TI9?#$pA6A_9ae4J5QFPdVC330U;R(&VMaoi-3Nb zMyAY-h zZ|=J)&BH$_O75gW@f9r@R}bOAz#l0tU7lkqK_nbIoa z+f6%q8BNZ5akTv*cfE@CpCw@bMuXmAu%QyX*GTylPl4qq;hV{M+J4|i8 zN6muv^t)v0n?G~@nEfkT|MkwZJ02QUv9kSNb$XOoxZOR&eR6`tKOP&KQLoFH{S3^A zrx3VS#l4hbbNKMf{UYfaBv1p%x@N+g$(3VuW;9gzx~k!+OBNl#nCVxW2@?#6N#|ZN ze;SH3P?&1dNB$wH&fP~K1MB|NTZjZ7-etB;ZBPyUW`$& zmMjjq?zu!oie6+#w8Z^3BqMb&!~yG zCYp7p3K&#F*bb!MkFEQ%gy$femhSCBT3RmQm75>}tNm0XFi z!$EQ^KuCO&GkDbEWSs?f&=3yU&K4z zbZ1#(;K$qnwbIVzcNjK#)zzU}4;)y%E_uplPR00C#sSWD0dzRGmKs&o1M*HG)k0kw zoUQtXfFea$ahla%m=@u=h4Ew}Q{YI*V)OBt*69ty7hl)-cI+|;pt{^L*z8l34%T_z z$-6GuJzd~my_s#>)2B;TEu9h;KH^Lmuvvz2$JsI;gg&X^ls_~#JS5b%z*cm3?b=)W zrGZGGK&8DWgQ(Is4JZM8jE#!2sFRc7m?KC);#B$t?8i1Q^iTD~3+j?;JoWX`ZPZXh zl~RNQ7o9ruFsHF)-8P?It;xQh*00>Mbt}+jkIrFxHOCDR|TrhpFe+uyL~O4 zeTLWt?r*(%-dfXw)0^&adYVH~K;J5|A7+b&d-=S1^N^ryVl|F96J#1UVS?y(BM|3T zn#72ZOe{G)Z09oGyY%XY^iO$!I32h00HzGvvAv)fWad)VjxXQP zl~6*6`11l=mTvw8D(TeXOK zcW1lvb^c{Y`L@a{d|2~r)`JAcWM|elX`~tQ5+i^=$&159?i*$0*K;l`1iy~U_>*8f z%3VY6r>308yyA2`Y3D2INe*xwca*KhUqy-yX_W^X?7&=6IZ24cNzt8>w$sD| zAu@(Vs$H=LI1B5ik|yp{sRaR^1Rard`1EN#YwPWAJH5`ia^(Z`eQr$vV}^`&EzIfx zTmc#mOo9lk1@)>vFx~df2X|??aUcrS&W#o^J7L07BXw%s_U#YdE%$BB8<- z1e!-(J7)X)f;G8uD*_{-QVte9$>k_nt~QHU^@>b$+HJha=9{9dTrxf58_cKu+L#hE ztQE)%IC3{6LTT4h($CKsez?ZwQc0@@uVNOhr;X)8oE5l*n#5$5H3~8}1Xq|M80$}` z=ES&4s-M-(ac>_N@s&hd5PIN%3#DT~4A1JIu~IYc98R7>`0CN&h_!X8|8YO__P_G# z#+Paw|CS*(z!J@Rem9 zFvDUG&{*JFG7Pp>dVBtJd21DcRCs&hA;;oa0Y6t?Yc75qCT4te_+egJEm%x#yT^~4cs$kaD4dSbVy5M#@4kA_b&!n;rGq@`KY8>=~EPmtJTohjr1Txb`wu@$CWE z+qP*#&YJ7?jKsw5TqBMLaKMNM`IfRLhC|J>iiq821KZo!?61E3?LOB1ANOTMd(_H^ zOZ5l$_B{ewa{iq%6}%^*3e|QF(;x+zLR0cjeOt%KUnH+BXgBy^M`=8WDdAnr3HCh;tkv0>>_1-upb25K82PU zu?PJY*7)`P0_mG4Oqj4BhPuIt&2d2izDo{;OGoBg-u6!GY58h@U+G3_tFajN*~HnS zJ#W|9^F{vtZqJ~`Dow7+qjKzbUd(bnjqKOA?@FXKYCaV)L#*BbrBo)M72`ZmdH`zM z>>D@wT(5vCN|%^V%}j(1U`x%fuzq%yl6J$d8&%J8g#akX2CIW|Vuh^!pWA~<1hgp@ z)6{zMy&y5eJSqycGXGi($aZewJOG@9mZJW`;^$}w<$Kd7t9$wa!6mev*`IjdR(D6Z9oci`!?Uv{);qFPeXL&) zx*@PNPvL5T2>}K5d=*D1m<~mkh!8kADd;`tkI#?TSrx>}@>)JV&3}Kbq1_szy51JC zMlZN?IMU2TJlQku&T~~l8<$;vI9M~D0IBCSG5v!Dd{n9w(HJ89C`p_668BbPH} zTs@_Kz&0qEnqwvvpp7@OQ3nrdWQ9s%?x0Oy<$05fKuKH1&dpAxFK8`6_?xU>t*g4h zOVGJ4WS-=&tQ}qr*szOY=7^Lu9!J_m`z5*+8;=6}Mz}NQ<8QRa;?1s2Ex#@51FwyB zj{_AJ6eNG$)5)nO*p|u6Qbm3t1u3HUF4a&uR{Y0?>c!b}4L&VneeN4$#(Nb^+(l!R znOodgu~@f%`fbnB9=jiJP8&VpJrjz`u((@ZN7M!Fv~Ka#W+gxQuS2QUAN|8YbNSY- zZFH#?RV-J6KRoC4r?X+z&*MZjH>h@F_@D67eaDKSQiCDq*!%NaGxUUfy52q7-2LI@ z;Xo5Z=bl#vqNl?%aRC^AF{s4NVG|FT_s z-Ohf^;8$}@@j9OO+w_w+uC09y5eAEA!sXZ!)&utSf?-;Y{)8tqf)o$f&3a&HVrEw~ zb{8!Af9!zFgx;1dQ|2=yC*%WOxDfuKrDpN-tp08FU>*cviVF(R8>9?-*J5P;BBD86 z|BV|rDozL2r}+P&VjeEscS@By_~R#(>g#lxPk~~{7w!#bqD>iz?-p1YVaXb@!k5Cy z5NNW0)*;LmG5gl>C$C-|OewFnF)=Zba%&kp&_`kKAR}71qX?2J0UdBMups8B_Fd9c zi-EMDL-0gSp^HlIZ$X`4TGhQE5f5E6D&G-mc*?JfJ3XM}7QYIT;yE{)Zo<*BDFj~O z6b1<%@z$`0F46YN4KkAT*N=CiY?i-45&@n%1ECJ%vy|Dwq?H%P%$}P4I7*6jDSvb- z3sDFegCW^@;1!R+(j#$x(9JCVXKz-YQDL|5kXQr^Z@Z#xm{J;yR2%(^=Fj@t#nMW1 zq-4C)*h&rJ$br3kFOy_W?hN)438zBaj&2@kWs((%g*X9Sgd7J^89sryT6%f9HmypX zW?KHyC(f_DWM9Xwh|oMo)v~-U1845be=HVUx%R#6?cFA~V`-9vw_ch%btlLFLjGpTsn&5f@B)BO z8DE|;8d`b zDM3`~BA_y*7NL<}V$AS;UMQ4ZWHx8J+?je@fHVL0xY!RFvPlcEc)Eam29PN_lrSM% znV=E?7w4@SyUg;N`m`@oA@wc_0%j2Q4v6F2bEHd=0^E7v7%687S5--wW`Rm+x& z%Fr>O_*q&)4axu#b3`=Y6y_qEhe&VqlDZij9AuQldbn6oN4`^Pd_ed3qo1@h_<3&l z;n$K9`6G`p8^Pa2n&WzvSsi!=U5WTZ8gwd;ZZumURKy4aM~Q|o80=g4 zxVglE5Ljk|km9havHL8VB+6%Em5Upg7mwB6V}}oKwB3(ng3?p}mXkB$=F}ncFM{^B zxO&2{rxNT#d4?-(BIyr4d+q|aT^nM!gBgqx2>GhYgH+Yjr}Gc z9e|UiqiI|C;4EmF0|z-v{Oku^Xa$~s`S$Tmm|j}}ER!eiHNL=O zfn-Iu=UUD>XCG%e%@Y{TSyyat0Wr?UxV#HSLrF=TWV{KN2g{#VYT4|8A42&e0P9=T zbwKmZlWX&B-&d)Sx)ZGe^{^*`RUv~24=^gUr1{BFR$ejQ`u~T@yO`buID9jMdgaIx z@40qLPVkiFvs1EjiX;w*kM~S}bB}$Ce(!gDyLZ;7e~=k7SD0g?XQ`Pi?fdBaqPq)y zg}<|7P%5On3S09 z$J>i!%C|gWL~aH+m@o8Cy7Ygx2Jl&5>2&{LtNz_(aG}LRo~g*2w|^UzdUNa^)3vc} z1<&5Dv;NJ!MM($qFAuF|a2xdpN^3b(9A9Z8#QDoG%sIqKvl;%mSYJl@?1^y;oG8gV z&cfMZ;XqH@H0K#@RBuGU+!JoZ9}NQVs?_9?U{Ol)twC&kBPiFE3W=*9-fSDpEOM)q z*0<$s)9zJsNpLX1M1Bc_-m-1d7xjw3Y^0~tQCTFk1?e669j6E)f)J~U8Yx4EVLAf^ zuNWP6>Aw~BAI3N1MkXI4H)!h&yod2_jhZHW#c{|WhG8Wck-bS~5|+x%%;!Z>iq&1Z z^j!Mc*9_;brnH=JxAt2NQkGDy-pUL0efA?WDggT!%PsDfI#r|61WL8ze`KiP0UJ@} zp})=TVdo~dm<@vlWv|4*%rf^6B)`HcnB1Cvuc`}Ovv~qqF_LM=K%a>nXbIo{1MkaEfU7US<3alF^ z^)EGc)TH7r{*TVvwh1vK_JBp!3wKNly?^!%+X2td&Io&0EB(UytPS6;XP~?>p-xb$u4828fg|kO-sPoXj|;E z(cWP`+22!n408e6In5o{aG^Xr;)sX!8uG8Imq)m&NG4GJ^mIqE+v%sgR&SL^*_z(nP}|??KZOSaCFN63mh3XqFi#(G=0$-?-;a z|DD*g;;>CATO#{Nq>MagDN9k*HF)OWO{>BcZeYAA225%(>Lyc}aEF9GA{>y|G1%>U znVcvPaSbf!6YOCb!lX)x5wGL0aA8B_W^Vo);!KpyHkLQY4qEEI5UYq_M&Gt-Hh0&q z&m~7irYL2E5O9V49f+c4O!GKq%~!%REHZ%!G`KL**8t2`!9gcLx=>GejHmK?jNYy> zEaD1uKVFQE`h5AaWh(&-N410WoD9u&jrMIfgJylHAFokSHcRVQvHz%WMY~N#ZwaFe zG(36tWH^8$CtRj(gVfl2;6QriGGzwKq03KxRBwD<_(gpW1iQY!d)KZ_@1*Zt(r2aT z!ekVK2^vwxd%S>$|M`Uj3{$)hBQmTi^7h9_Gp|9t1$LTWVNeXx`jGGrpzn z!LyruF(D%Tr~K=CqKxeqQ1t$}CvEoRqtN(p6xS*AlwlXv^%>xg%_K#ECvw%x2~00G z;z2E^{z*Ps0gi0^rGxP5tsJmcb@tOZE)dUDc3F!&i1i{s=VxP9rB{~M&$Q_P|0EN z$CJa=iq=EG!v^B}{h?1`To2zcu!D_?!8A8@i!~r1vHY{UyqY>!&SX2xhD$p)Cpr0! zyYpc0P1`rMM$*@PZJRoRrJT4P>>fv_+6b>THu4&aXtE#DH@3y0trlbgtdYcJ4q{vX z1znql#Gt}ghML)2hIS~JfXJMeUGfXn+lxi3oe%xSlq)mK*DsvYJE4!Y7d=h1YX5;Knlk^9f2=Z#Li$>LphVCFJo2jyQ9WYCXz(3C4ZyT-uJuhqL(w z8WmP9wBDUj*1c_QXCc}A^yau=r{TlvSZ>0#v6Yhhr=tv)&JKxt+L>~g>l8KgzXhYZ zHVNB2Ho06V&($xeP$)LcK`(mzBOw8Ogn04rg{CAOKmKE%2zcNv924fyvCZ!F0WX2d50x6`z`*ogJVKnuf{#oe+t;TzHj=jUQIi1 zc%xFbscFS-OH#HBulB>gg5NH(I~RTB{dZpb?59SD+D}`(rFO;nua6&dX-963cS5W2 ziET5VroDZ=z~kcYV*9F$Z{rJbLYk=0G zc?o+zE9~omYZk60NMU(+b0 zuXuqky!Ud-XCuo{&e-j@(xQ5xmlCpORADHn0qpOm@&hM*38cK-s5q%ZPc!bw@>KZH z_xVHXv^Jo3Ql^TWDIRK8_7W(2dMO5?c-`bs&~O0itdDseQX+?=6~jQ;;Qk|S>@Bt>yJSb!=&6$J})F8dP@XbMDxoP-<^N`m%%mQ_vu|G=W@i-P3J zr#Iq2t=MwRdLAU_PcljvVz=AJ8ZJjFpO?j<%sF6$ebXNAQX=uSaGsrUA2;$yTDu*o zWiOAdhSf(Hr$_bGJTM`A2O?67LtnW}ptL>FObN4{cl0xo2+)p*-t!;N=jt3`LyiMk z2ajeI07KvZ?c*|rM=HwBQKQLtDP9G&fq5+vj0*jsr!ij`sl<9s7ZW+A@ne*1k^x}y zA+@xx9y`@eXzfC1IQNaqp1XE>L0>rKjO2{hKRr&1KC(gEXDd-^h*=@eNuC%bVe-Q@ zLX&e^9vcyXmCli?WxYECu|gL!Km~=z7C4JuF11$IZre5^xqA9y`rNviZ`Js0M`EY$ zT1KwDf2qTrTQuSEy&=={lbv7WJcS~})KFVihw^c0y6d21?}X{&6r&ERT&Ef7j3z{* zE;fML3fGorGy-d*-~eKEr@)X8NNsFruM&!D;6S~scrK9DO8((#dFHUhSoOy+vsHFC zQbYu4436xLI*AB6>p9}bxr?cp_qG1P#=Lgz!0e}P9ez!I{7dlt#g5mTICOV(a5$0P zqr~feY0B8ns`ja3%OvC%k%NatdMjXMB%z}(*wRoG`<|$Vg^3CofnC$~OcYGL1Jw+- zPwKRU;O}e}^+mvUQRZCI2NoLZ;e*zkv^ac>CKD zv7-vFw{4`5HVSKr>$K!4pqD?4+~0%}$PG&GYT+jbH-VRJIpU@$4PKp$6}6qYnW=yG^+ z%hj{9^E-#RJig$gXe9RbTsAs+>H78SXVszfdQHldfh{X;Vf@JEo?HWEM*t=DR-Z&d;V;TWlRH* z{qVEL?hb_bto*$8h)d&VCiuV90vy;&{evK0p65{HD2eYQ6X3EE1r~c;?9B%se99ng zr3uL@7y7kSydl~$FMlJBTjjd*Y(ci^y(hg>%q$k~sQgpx?;{U?A6)*IZTHo0YO=pI z|6rJSy3I(>nR}+T96W61o_7s?iyOVbH!8fGdhRx zdOvXZ3E3yg3a^)O*Laik&J?hBO~>qP)>vkc2!(jC^>T9$3=gEe)YHmCB{c=OX#{J= z;R&7hem7)FU(3=eH!5Uo3Xr5jKvvT8}vt@0WGiS~_g(ndd z$zdIOD#?(3oCk6b8Yh{S6fg>eZ8Y&PpW)l9ym_4PBV|Bp@)dzXN`e7K4vgHdsqQ4U zaz2Q6f*zaO`6g%7?H|a?JWP45Ty~WMyDu2!v0okvpMX7S09D)W`e)6NLyX)%;qLs+`Uw_pJgy;jN(E zaMDVM8+vr1!n?)(Bn?D2KOmq`FqalP{@beERy6ujk(r!;axDRim=tx1jyGRPP3ZQY zUq~XyZ4KYRfmBCzhN@w7&wE&7CaF=RE(__gu3fgNo?(D>JI>qzW+Z90h1XgfI!*Xl zJ_yY)-i?$z#ya)t#rLegEUcW>=S`Z7zIZz#dx%+!4BupL`_d-@IF*9e4gHud$}8>R z+zv0xvY3!Xj%*h1U8fk`RCEO`T`1eA$$5+S>>qxxB$RS?*V>=az$CZ>;YbioSF@4l z?Zq25G?7%W0AYdhi0#a1&^B}YXiEoOzz}`VXp$0*mlo60snYYC(u4Ed=?TRAEAmtW z(2a$pQKq5?QXsGC3^l&RQAXLTq$7o80rNmKh|)b&yCG4On!~NFL5cMKNhQZTm0AQ7 zOHZXR2*RZIO27BW)7Mk(RYCl0^dguP(IlYv z=#I+Q6=|&T+*c-94=8*yGAO76eV$S@=))`DA}UFzIMN@l%;$6J<9R`)5A#vdS;e-e5BjJQvl1a?|cON7AfV3*ZX z6_ru$a-ss{EzzsHJr+>F%SS9Hp*aXmq>el)T`BmoR`cJuu&bUSk>5CAz*l8>R#9Ku z`1j=@ML)KMYS+nYHfPUAoIMp1zb@TY#@(FfSp2szylm%5cWv@@95)XG2UCrRZcWQM z?Mdapgw5EX13Zua;>+A0$o6EjRTly;YO+E9;;}mzoStFyF;^aO=cMGtDSn!;iJ#=u zWM3!yzfFi5Chj-{I z*H-6*m`4}}gzael6F-ypa5e)0My$R?}e+mq{5(x1Jp#yK{xaed}@?K$hef9YfTr-E#%8}F`{`P!iV zccK>@{_yKHA5g~iF8y+9*%j5Y`kG~l%#J=t37#uL6D3OoxYvupYCQY$?t;}!h`--s zZJXdL#H5ilZ2qjIA-0fal$pfV@Pc<@o`%7+Jm4t45xOJh3}L8}i2;Uk5_d(MdtgW< zt}bKMU9CF4$A_~FO-t`b!V5<`ak4dd+NrK!SB7JTvJ5P)Lpti%X>_SD@O?uT;M9>R zZdt_^eDqmcga|~TFlvk&I!AM^Fi494lH_J^ID?f_wuhcPd9r`xm8>ksPc{%McJ^$& zyxsG$`!AZ-%WyAe+OOx-YW1GE+QITM8nQRg#GR2JiD1HCyrOI_?JppTBSQtp3rQ?u z0SH+@r zO*4&?n5aKsmJ`$3)0naWp9Wu{&TT&& z1c7M!udO;+I+-t$|NY*3wK&5$YRm~9_3js>*FF{|5?s14u5=U?+Af-IdFRC3l5 zY(Og$TJ-Sb#N&v#ex7;3P&P|3!43K0ErVYdWPA8;m&0n93U z#*1@(=AQmNSIr)EF#GB38nc`njC#`_H>&%yIC}j2ZE;ZVx-b$3dfb7Dgc>lfBWWhN zi+s6F)A!Ln-DNSgazc^2?t_g-wJX-_%T5d3}HKXF%dO&S9lzWNavTY6NZ+M!*6 zO|U}(ycHzeFs6ra0fSdVJ}D1PGN=93^W|~zX^I1to`|1qPM#~Tdkh(}Y|5RBHO_DZ zPE8y8Tl9rA$3A`R66+c!d$${BdGtfR>R!i<7mF$&IYK3j0|!>+r!$_WWrpxPDNv9O zjHSEni zm5$7s32hJPbbf9;Cl|n*$>}>}WTps{*oC_w<(Wyz_&_|B^Tcuk-C0d9VYuvNt1-#Z z5tN{<2JUd#tFhQ;$W{vi>e~vS#tWRmwKnGLeFZ6UL58zayqM#oC-%g@v}t^$7rERT zOfE%nw_xO3@d_(KH|wC1F3` z>+qlPh2T7nx&a@*dek)*&=4KL4k>OFFP@MjU3+e@CmW>F?|=_9eYS7?fWXLbY`hwD z3&I3OPfg3) zG@rN+RX}8d)KjQndPX0mgr#5PNKK|1H#v~yp^oQ2o`ECw)7DW6GggU%Vlia5{oXZP z)KbOg@OIcHOGGD{Qh9D^&!w+&KW<00Bk9%I`>=uEMD(kYrsg*e48wJnN{HC)@_sl&d(dP0on8D+N-u(10fUDth0S<+nNqIOD zI1`vhW)C1K;BRE}Kvv{t@f$ep*sN(MmKqu2h|~9L|NA?txK?HFOw5}TXyVbD{mP>? zoQ1yfY2V^Y2RphR7`L-o;-?!FAH85eh0vPLFsyz9=NCE~=`zuXK_zO`qf%ou*VxxR*TlC@0Hb0d& z_}%-<3$+L*Ls-ro4h~uOyr#GyYrO=O&G6KXWj4NUh&^Z@yVcOX#rX?}hH-2$I4<|Q z8@%XbBRMMc`bQMospxO2ZFX%P9tKYC%H2StHl|uSh2#U=)H~C zB0?J7o4C7fjhD|2u)&nwxi($wR&U$(b0YYMh$=Aq^D`Igpghe~Nrlgt_n;Hy;|9!S zEC1>w=dn7PVm@HPdp@}C2UID?hSv>6Y)|lcy!o{`#cA(bUD`jRprht06tj3g8T3#Qi)9e?yD=_-_MpqX9Ub z#;}3-y=MDnbh4X90Cn%H;Es^Be=P5Zv6kieqS4^*|6J?+=Wn9C^9#)Cd$$GW>~ah{ zp@$n)zUR!)z5lczM zMya6e4Y3PcPoZP=%f)v~S_rSte)vsPnr0s}0B2vBjqDt4U+f%MV};KGL!JfBSg*5z zAR>Ee9oiwqQUfpD-F%%?0}b`25@H<4LYb|g-+K=Ru^m&oc+2S0wJT;tY45+;eL>+v z)cfMr%#7`APbgn&{~nz17yzoI@lZ+_%hF}MJ8s-%bX7MCSqxs5Qv0&Xzs;;;r6)3w z(=vXD2-7(8Vs>39wELT{Pn|TWc*Kq_$5?C z6f51i?6}UBv+v;Q)s;2+iqYMtl!-{GJL%3vT9ApJ$j|v*(rMxE;rZRs*!`HD*)lMO zYK^dy2B|}zJbBXkPz|q|ZQHJCFufnA70~%mif#l^y{P|W z|Kf6>PK+pKh2h~%0KwTYYKBNSj__bvb zEzm%R*|X7KCWTA=rg$x0yCg-v>UfR=p$Wp{C=gKjGc)x1=De`}QQftn@ts2CmBjw} zK1!JW8!W44*3rS*VW}+KGwD$kV{CGO!j)f)hFJGZ!Sg&V$@D`&yR8%7M#>M;SUFA8 zikUl5!$e^fRo{pG%=Or<99@ID^BC51!DY%LM~3~H9LA0f=c!iuCaRU=BO+S+hxgm= zzsYI{xaq1aZzvZ^1)unqKn=d8N73B}4XVC(o8fm#H}suWn=Eg3ATNY9HjK(df|&l1 zBkSbHWOyw%9=xAugES~vqmh7Ib?_*cA1#@c2@MF7Lo}Fp^yA5DEzf?~V#(UI-5b>V zqp6v{^em#(&mTI*^5(rp^E^PkuY?PyKazhTAn6$a$?z)zYN_NEG0LGx_uo5s`WuGWDtaM$vh3DAfa~PE+U@n9 zPQK!8V|{x3_fL*J(7kWc?B!vXIs_q}lN((2G^%{1d}>nl=wa@VM?)?>`vygT=4MZJ zBD2{mXUq}%D*hF6a(KSBnpb=~c8zD!?1gjZj=NP4&k)u+O_?HD0u89$z9)7OsDdk9 zT!BjgY2O}>Z-=LNMP)wFq=cGFj-M`>G#9NJS2?A&o6DAM*AEjA*NMY=sj<`Dt8flb z9Yvc+G0i+RiyCNU#~%SfFYWC0!_cFdYeRk;IoSErhC^!TPm4}oKvOLI5A9=X&lWw& zkNS*U$NVu(ER~f_Q>8-}NZ|q%-!Nh!uhFhX+raa7zjV2vKk*5R19yGWs$q+%-TAE1 zPsfy=KQ7wm`MV~WCNyUPhx`g~!q<&o1Vh&E^LdLl;nX<1h#T!NQNoh5nP$+4A)$<85JJIZ@!xp`4kd06x2g3_{?KSlJ^&5sJ7U&aYdm? zxqd#1L0sxfmd}ZDF&IiGC>H=8L)tx#emrjL(Ihy_sZKZyg(BlJ#-H_3ZskJXXPQj8 zeZCjVDWJAnh4Y+2tt&S^mpq6~AeAw7#c0|1TkHpG#e}tX(YLwnWW&s^_Tw{iFh3^TfPnu9v z*-lfYJQ-DG;Vz(2bNnA*>kA^1dq&-+UOwma#@QgsxW5A&?xf{GiE4^H)Qvh~hS&E3 z(b1#NXpDQ2?UGVcQ-_D|-wlB`uJ0f9&d{LMn}dRu?p>#QHfAaum#6_^_v)Hn;&hCD z{zPN`1V{#jcCllRO^zPxzWfNv2AiwJg@?vQp)Zw%+TE2k_W-`H9-&ci8}>N;xFgs8 z{W~<9X7|^U`v#nJ2nP$@qtV39cZQBY5^-c9-RU~+L{^HlT{bOgrjcIEeh%*Gt=(B~+EaD zqLRt`GrXdzk@sEogAXoWol`dv$;d5s;`$K{>(@US1Hz92PU0vM`Y_%c;>d%tN+}N(^lW@%c)hs(ZD0JzirEw=VN$^Ew93?qyjKYr@=dB z?9~g=bLZju3XeN-#5MQKD^?7vhH!%4n8tvmp?A`v8fKQCdSUXFr-P$LXY5OkVGye2;~|Z2N19?x z!(}Jegx#6=UHG@-8*ScNVZLeg!n^bD?>*dSj+L<_FIO91rKN6AQYM?;{+1ypmS7k{ zbYo`*X4(RnuQh~=M~{b^2J{EDv9qIUla|sLj0;2hYYtrc+uM(N=k}}A``bV2N7s(o zHNc^j%EhEMctuJj0n{h%OlY-q+45b{{RdW^@o12>YmAFob0?v77vGt8BC*TZv3EA~OJ_L+X@qxq$7S9YH}_vaNy9#=?exz1+(%;>jf zeG69NJllQ{)Iu0kfx_gaSfD?23~4=ScT8qt(f&HVWmhf?3D_b|TQCB#P!ZgD^^z>M z2Hx}c|C+625WNxNqDDZ-`WN0Vn*KQHK+vX5cc(vgrFB*SB1s}nbq7v-9uxfG#?KJ}7B}`*M04WJfv)V@g^k`AlC8;O6yIj$$a~Tl zd97T&<4{pQ1-c^#P~{ra@Okc&1HVmQOyw(<+++r`03To(m11@OQoOpqJbI;K&&JWm z(pw#Ck~lGH;Y^NPJG3vFoMAeHAPVIcv}o4Vt#yyFtw(?um>fJjHY?>2z}V)=b2!hf z;K9lvj(J`*6)k%X<);=lgyA``B0<<@OVlXUxgc*O`N8_@h>amR>_anug$Ag4aqsvo z9PJv^rMXTtr9OSzk4`0`nF#pmH*;29!W|r&caJE{EmmCuZcr#}%U5{=R$lC52w=+B+USK!DOkCcGhf;d5 zu|7zkrkW2s^@Zm7X)-sMKUkoWye4=%$6MjM#tSM`9PmT~w1xCisFXD?q}NnxBS(+4 zMIbTrn%GC~x>ttZ!rjkRDTu5>{tbjTsoU@E8g#hic4!z-!ZEkz&6#rwHZ}dpQwOJk znpKK-I;tp!h|3X-FjX1^d%tvS0B_FcGt&qbJ#no;>O-5K&&Rf^;MnWEW5&d|Nyi?i z531imJu+Vvs&=un{N*2UVIAmvxxGTEy8Vk2a|x?WTD-UzItd{4XIGnlRHh6{*qifz z^U}0HAlRFjz8gjyqSI#W?$i6l(WcqXWTBKT!|%6^%dg_s1c`uK1r`K}(Fe108lA|N zwhd&fDnTs{Pw|-PKozWoHDyZnGUuk4<_5$}yU=m*Ylh4HTrx|voOdCag0C5Ble64E zB!#sJBZ?~$K0-*LzpJNyH;NB=^l83jzLTUyS-13RuvbpB`&sGwZ`JGw_bkcM=mt|D zTC9O4OO~oAxk%=Wr`t6usO-ld4`b25UL4H2znpOmRDP-qAKW+%qlY>2J8nE6fwpLq z;%256tCW;(8mP_=Kore3JSqJS&n~@V{_{XOBWst;y)myR!a*Bmea;m_huzg@ z|3Q~HTyM%W5Q~F z;d}1UV_P^bXPop@_#!Yxq3Xm|!g<_fTQf5y&mEXNe)6QHMQ3xev#m_>AA&1rXp|(M zsNT)8=Ewjk$q@M~=+bIPC`Xm2-d{39xvMC+6r-T#n=~e|EU;qEl<{FMd+)_BdEfBG zn^{j1^Q(+036$1oDG$75GA*K`6W-HUtQJ_pGDr|Ye>$q2B}S=Qxdm!2z(ObBxzUI@ z5kwQbyLp1>Xp*X2K;`Cb>h+WbtNz-`#53xQyoa#^i$8a7Ot4ym20kNqaH%wc7TB*D z(^=HkYuJKRKcrE~fYqx#`;ep}fVcK$xG{86TL2I=Ji{_g0WOVRL_|8=VX6pwWMnsB zgX%kzRXFC&usf{tp%l)sW7N8!wI0q|L(aiTv+2_FB-a`ovhmZV=apZZn?&;5QUo1c zyLJsO|cJNXE)?<(S6a4`pO3^;SL| zgeQe&cFt_}bJ;rs6P$AHE5?GnA|+Ow)l?Fh22Vy)Xuqbn_pQxQ6d1F{BUwAn^cW6O z^RJJF9QBH-hw4lkUawrF!BHGhXi{6L(6E%Y44C=CjPS%UC-FuqN8Lg9eK}x0@)ofE zx1Dqsrb{u#?K>HMhh9bF(wS-BOfnQSc%ZAKWTm$#vYb!|Elm!neWaS@@xR%__sJ6W z@h#*@Aj;d~G79pnu1jW@tS9fMx6@(?;ypbSSdOa!S%Dl2fQ&bs4?S5%@erTU9Z;+p zQ^>SBv||Guar`wS84pC~I9>)MTzZb;+Z+Nd)`p;oP>2hLv7|R`o;8r#xhpL~s^&e> z#ogdOASZN!K+I;kwQ?>HpNJ9F)DT$hn?s*|zz#0C7z>TtE7qK?MAH=GN*zHL-ShTp z+AKcB^U|pkFBhN@JIEuTFTZ(+j4glr-;y7{W3h$4Z}w^sG&8SsW4(^4WA_eWU%7qv F{{UoWVmSZ+ literal 0 HcmV?d00001 diff --git a/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb b/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb index e518ae55..8eeb9d78 100644 --- a/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb +++ b/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb @@ -38,7 +38,7 @@ "import numpy as np\n", "import sys\n", "sys.path.insert(0,'../../../')\n", - "import DeepCrazyhouse.src.runtime.Colorer\n", + "import DeepCrazyhouse.src.runtime.ColorLogger\n", "from DeepCrazyhouse.src.domain.agent.NeuralNetAPI import NeuralNetAPI\n", "from DeepCrazyhouse.src.domain.agent.player.MCTSAgent import MCTSAgent\n", "from DeepCrazyhouse.src.domain.agent.player.RawNetAgent import RawNetAgent\n", @@ -116,7 +116,7 @@ "# TEST POSITIONS:\n", "#fen = 'rn2N2k/pp5p/3pp1pN/3p4/3q1P2/3P1p2/PP3PPP/RN3RK1[Qrbbpbb] b - - 3 30'\n", "#fen ='2kr1b2/1bp2p1p/p3pP1p/1p5Q/5B2/3B1p2/PPP2PrP/R4R1K/QNpnnnp w - - 0 18'\n", - "#fen = 'q6r/p2P1pkp/1p1b1n2/2p2B2/8/6n1/PPP2KPp/R4R2/PNNRPBPbqpp w - - 0 26'\n", + "fen = 'q6r/p2P1pkp/1p1b1n2/2p2B2/8/6n1/PPP2KPp/R4R2/PNNRPBPbqpp w - - 0 26'\n", "#fen = 'q6r/p2P2kp/1p1bpn2/2p2B2/8/6n1/PPP2KPp/R4R2/PNRPBPnbqpp w - - 0 27'\n", "#fen = 'q6N~/p2Pk2p/1p1bpn1P/2p2B2/8/6n1/PPP2KPp/R4R2/RNBPrnbqpp w - - 2 31'\n", "\n", @@ -181,14 +181,14 @@ "#fen = 'r1b1kb1r/p1p1pppp/2N5/1B2N3/2pPn3/2PKB3/P1PP2p1/3q1rR1/QPPNP w - - 0 25'\n", "# example to check for infinite feed-back-loop\n", "#fen = 'rn1k4/p2p1qNp/b1pp4/8/1p6/5BP1/PPP1PPBP/R4RK1/BRqpnppn w - - 0 30'\n", - "fen = 'r1bqkb1r/pp2pppp/2p5/1B1pP3/4p3/2P1P3/P1P1N1PP/Q3K2R/NBNNr w Kkq - 0 13'\n", + "#fen = 'r1bqkb1r/pp2pppp/2p5/1B1pP3/4p3/2P1P3/P1P1N1PP/Q3K2R/NBNNr w Kkq - 0 13'\n", "#fen = 'r1bqkb1r/ppp1pppp/2n5/3pP3/3Pn3/2N5/PPP2PPP/R1BQKBNR/ w KQkq - 8 5'\n", "#fen = 'r3kb1r/pbp1pppp/2n5/1B1q4/8/2P2N2/P1PP1PpP/R1BQK1R1/PNpnp w Qkq - 1 10'\n", "#fen = 'r3kb1r/pbp1pppp/2n5/1B1q4/8/2P2N2/P1PP1PpP/R1BQK1R1/PNpnp w Qkq - 1 10'\n", "#fen = 'r1bqkbnr/pppp1ppp/2n5/4p3/2B1P3/5N2/PPPP1PPP/RNBQK2R/ b KQkq - 5 3'\n", "#fen = 'r1bq1rk1/ppp2pp1/2np1n1p/2b1p1B1/2B1P3/2NP1N2/PPP2PPP/R2Q1RK1/ w - - 14 8'\n", "#fen = 'r1bqkbnr/ppp2ppp/2n5/3pp3/8/3P1NP1/PPP1PPBP/RNBQK2R/ b KQkq - 3 4'\n", - "fen = 'r2q1rk1/ppp1bpp1/2n1bn1p/3pp3/8/P1NP1NPP/1PP1PPB1/R1BQ1RK1/ w - - 1 9'\n", + "#fen = 'r2q1rk1/ppp1bpp1/2n1bn1p/3pp3/8/P1NP1NPP/1PP1PPB1/R1BQ1RK1/ w - - 1 9'\n", "board.set_fen(fen)\n", "#board = board.mirror()\n", "\n", diff --git a/README.md b/README.md index 8e5a002e..c0c9de58 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,12 @@ See [LICENSE](https://github.com/QueensGambit/CrazyAra/blob/master/LICENSE) for * [Project management plattform on taiga.io](https://tree.taiga.io/project/queensgambit-deep-learning-project-crazyhouse/) +## Main libraries used in this project +* [python-chess](https://python-chess.readthedocs.io/en/latest/index.html): A pure Python chess library +* [MXNet](https://mxnet.incubator.apache.org/): A flexible and efficient library for deep learning +* [numpy](http://www.numpy.org/): The fundamental package for scientific computing with Python +* [zarr](https://zarr.readthedocs.io/en/stable/): An implementation of chunked, compressed, N-dimensional arrays + ## Links to other similar projects ### chess-alpha-zero diff --git a/crazyara.py b/crazyara.py index 0eb53286..4c32751a 100644 --- a/crazyara.py +++ b/crazyara.py @@ -22,6 +22,7 @@ # Constants MIN_SEARCH_TIME_MS = 100 +MAX_SEARCH_TIME_MS = 10e10 INC_FACTOR = 7 INC_DIV = 8 MIN_MOVES_LEFT = 10 @@ -126,8 +127,6 @@ def log(text: str): constant_move_time = None engine_played_move = 0 score = None -# transposition table for future 3-fold-repetition check -transposition_table = collections.Counter() # SETTINGS s = { @@ -145,7 +144,7 @@ def log(text: str): "centi_dirichlet_epsilon": 25, "centi_dirichlet_alpha": 20, "max_search_depth": 40, - "centi_temperature": 10, + "centi_temperature": 5, "temperature_moves": 0, "centi_clip_quantil": 0, "virtual_loss": 3, @@ -188,7 +187,7 @@ def setup_network(): for i in range(s['neural_net_services']): nets.append(NeuralNetAPI(ctx=s['context'], batch_size=s['batch_size'])) - rawnet_agent = RawNetAgent(nets[0], temperature=s['centi_temperature'], temperature_moves=s['temperature_moves']) + rawnet_agent = RawNetAgent(nets[0], temperature=s['centi_temperature'] / 100, temperature_moves=s['temperature_moves']) mcts_agent = MCTSAgent(nets, cpuct=s['centi_cpuct'] / 100, playouts_empty_pockets=s['playouts_empty_pockets'], playouts_filled_pockets=s['playouts_filled_pockets'], max_search_depth=s['max_search_depth'], @@ -310,6 +309,18 @@ def perform_action(cmd_list): log_print('info string Time for this move is %dms' % movetime_ms) log_print('info string Requested pos: %s' % gamestate) + # assign search depth + try: + # we try to extract the search depth from the cmd list + depth_idx = cmd_list.index("depth") + 1 + mcts_agent.set_max_search_depth(int(cmd_list[depth_idx])) + # increase the movetime to maximum to make sure to reach the given depth + movetime_ms = MAX_SEARCH_TIME_MS + mcts_agent.update_movetime(movetime_ms) + except ValueError: + # the given command wasn't found in the command list + pass + if s['use_raw_network'] or movetime_ms <= s['threshold_time_for_raw_net_ms']: log_print('info string Using raw network for fast mode...') value, selected_move, confidence, _, cp, depth, nodes, time_elapsed_s, nps, pv = rawnet_agent.perform_action(gamestate) @@ -318,7 +329,10 @@ def perform_action(cmd_list): score = "score cp %d depth %d nodes %d time %d nps %d pv %s" % (cp, depth, nodes, time_elapsed_s, nps, pv) if ENABLE_LICHESS_DEBUG_MSG: - write_score_to_file(score) + try: + write_score_to_file(score) + except Exception: + pass # print out the search information log_print('info %s' % score) @@ -344,7 +358,6 @@ def setup_gamestate(cmd_list): :param cmd_list: Input-command lists arguments :return: """ - #artificial_max_game_len = 30 global gamestate global mcts_agent @@ -367,7 +380,7 @@ def setup_gamestate(cmd_list): opponent_last_move = chess.Move.from_uci(mv_list[-1]) if gamestate.get_pythonchess_board().is_legal(opponent_last_move): # apply the last move the opponent played - gamestate.apply_move(opponent_last_move) #, remember_state=True) + gamestate.apply_move(opponent_last_move) mcts_agent.transposition_table.update((gamestate.get_transposition_key(),)) else: log_print('info string all_ok is false! - opponent_last_move %s' % opponent_last_move) @@ -530,14 +543,6 @@ def uci_reply(): log_print('uciok') -def handle_uci(line): - #new_game() - print("id name " + client["name"]) # + client["version"]) - print("id author " + client["authors"]) - # print("option name OwnBook, type true") - # print("option variation crazyhouse") - print("uciok") - # main waiting loop for processing command line inputs def main(): global bestmove_value @@ -547,7 +552,6 @@ def main(): while True: line = input() print_if_debug("waiting ...") - #line = sys.stdin.readline() print_if_debug(line) # wait for an std-in input command diff --git a/etc/media/wiki/CrazyAra_v02/training/crazyara_training_data.png b/etc/media/wiki/CrazyAra_v02/training/crazyara_training_data.png new file mode 100644 index 0000000000000000000000000000000000000000..b0b537dfdc7975c36552a847bc28ed877207d087 GIT binary patch literal 78044 zcmdqJbyU`0_bm!y3t~$npn#y#AV{bvD59tcqI8KM0@4i@7=#K~lqd*DcSx$F2q-Nn zUDBOrZr|VeopJ9t=Z1il2|m96^#e!D zEsgl1dYQ*4DE3jDJ$2%Web8{LgU0#!)q?S>VTT0H_`5hq>pr3zGT^=L!F?*7yI9A7 ze({KYtbSQM+mq4@H-CD$JQXO;6T4ixBjAccYdfpd$b8nwZGnotlP)r>zAV03p;ncB z&eDA$RwaEd(oaJaA}+H%*miQ`zyBHEUb9-g?tlJ^?Lq0&wfz6#|QQjV;2`aXO6(6!_T5$9f+gwr!HT>fV>{|NI(zKn{(_`1t(mEw4cR5D&a7$~efQZNeY3Y#C(lalQ zXqGQb4~>tHhn@A~WX;YrH=OMKLD4@nlwvbf?Q8I50cf-dFmZL-92~rM9-Vv$OMD=Tp8F+ZFW; zv!0dZ#eW+xE+hMI>7}pn1UzANox6*tkn^G|v};Lce?ILM^By;Blv1zj9z;OfhBs8$`)^!B z)nT$s$84#O9XqyuD^t|=oQt>Ga>Qkp#;AOJeD-s3Q8mdj9XfPKEzemz=4!&nYbiIV znI*1>bMo*!xzkmwd;9i}SK6+R)6zt!)>C(UdANnaVRodIam$vcOsni19Gj@AzkT_# zMn*;^*rvZdUq;(`e4T)R0K-67SlIpp2mW+)6o|Du-&csx8>)|!m`qGc3cKB$u90p1 z*oXPp<;V-n5%B{919J-tzw%t14jn!mt(`AjSy{Pu{rVSg-nt+WwFfKMfJtMK6;1!v+dvaDlw|o1HVo;t&IJVy;1S{1l}xK^Rwvr zK+y=h(KaR-R}CZSgsQs&W-UkYS?$Lr ze5L-==)*00=_IVV&%YH8v;F%!)vWcc?pKfPyLV4l8t%o0NbeuGOHQ9(+O38o`#wjW zHd~k)_#m{5CH+=hEND{8T~SftKD^$*mqn_yy!=Z|jo;?oJY)Dk^$FTa_+n$Kv7cbZ zy;raHDuqhdl?oj`>^)EuNp&&$@&TNhvLJD*#w;r?gc`#l=l=Z`6WwPSL`}~ij)c>r zJ$;WJJEpIv_r+IcrL;2m7`><|9Zun|PfxKkSl^=(5_&goc-*~9`K;BxJ;R*!!i5V{ zqwS$yUesT{etjeDB7tQzT$&rFxDu;&N&m=`zN*l!-d;IJM+ujOsW62A9&C1!e#zGL zv@8|BBF-7td^{iDusr(c=~Ml(7YC>rgg5Toxs&4Z<;(qbF?;Xaxnu9>csbY6CPd0f z?&8I5c<4jw>4&rpuVq@WWtg{rG;hz_bx7^myLbCk zj??cgd&;q^OiD^hf`Wo=>Qc=a=2ck9O|-OqC9H1Y7Jt6X$45Fos!Y`V8l{yhfh=() z>LR5(BK+59J99j55zeLj8$rWIK0ciL_PKYJ_@)?DvGi#D+bWU&oanM_i$80yt;N57 zO_(#gP4!=z8EsDxUfJgqb^qbRb@B1>eQ6=Wo6}o#v~4_zDku}ni?86Yd+GPIB~*@N&NCA z+nkZwXk|%~-$Quws>WWFj*N`-d--y`!R%HCWCEO~A`~Xx z{59(~Q&_AlIS2{~1yrH>AS?CS)3gOVe*9Q&Y|XnHKi)FPT3{E-N=x4hAGqZ>fGrvxO6O9gq02oe_*|e#!y*-ps%=rCYkn@C8mNiBn!tirs%xxA$bGsJ{iQ9}iT;@Fq4n$6OS&xhqxeYJ4xjn?{&av# z&97e@w{HD-<~ciqcR%O(w->`suJd7)sa#&1wYjhq6(fy^IHZ;H8Y{M)l{LX8S1Z>s zB1O8szFzIxwe2h{KHlCmhz4QfUk|Vbx&{W}ldoslB1gNs%lu;1lJ0$f#m7786xAd0 z%JX_pc|h7PY8o1m_!Fk4hp@2>?T;54FGnbh4>zSa2lL9*_PIz=%*Ib&iN0dmlKC9r zHrAl+Ha^kwJ)@Ffl{vM4nB^N+jA>J{r2UvXB0u=+zKP4RY65Yoaq1a@#=rI>Q_=|< zY^;m98i|u$ZW)&?>9!ISChK`Q_2wm%=hWV)>YADlI4XzK65VAzx9Z|RQL{Q&ZK``T4iW%F1Tj{w;qkpubWtQuQ=`w7W4$ z-+A$G#x6cBCR$orle$k4!?O!xFLUS^7<#jmu3u+BL8;4%vmfsq6>;Cr$hhTQ(5Opo zO-YI8ZeG=~(A6bRWYo)9mWI^S)R{IzF9pS(0_crLzaZ%Eku&iu;a=+A~K7lEYUV|Ek6O+qBSBc+mx@}T?U z&7bd)$2$iG&T42d$@?&|A;kbf`MrHxt|YVUZ^SbCHQQ#06f=CLkLLq<28@l3M|0)x z;vk~xOJw9Wadc{iM@CvLOc^WqbBD#6PQ*_;PY>>NjtyJUs-r6WK`KPJRF-;g$JIWtdfbw>tR(SRtFmrs*r+@I!@ip z8(ec!*n@J%&YehTav>%8m9@40IDg4DL#m33ijDbhtLn_FNA&dde>OBk0-Z1n7*&NF z;N>044T;gnvXpdPQi~~M-MRBaT4%-Q#>_h$n>KB-9%;F@i8J?>RH&3wF{(sjVq*11 zt>cxm31o8*KL5|=s-l3Rh|(BU2I&GPNW99D#KzP_Dn(+^ujevbA5{KGT@@n5AtIu3 z)(Kf`s3Bp)Mb5$p!*YL?<|mWV(<>Xww`yivvmHKs{#y1GX7@|Vvv~kdC5442`r6$o zsVEwg4bR~dzTGKy!_ZK`X~xv>_4h!l{ZWTCvv)BvGRDa{72aDXAS}Ej@8R1Wj)Z4m zXn1FJ#pS?Z0RiL*)+f6Tz8??H;l&@+;JN7rZINGB<2B~nDv>iP@et7$!`8IB%m%+$ zNuRE5XlR&QQFwXi?AF@%F`4WzVYP`mq1h)Nvu(8)t*7Nc4{RRL4yapfC`SM^> z#aof5D6+}MHJsPeOo|?D=b*URoE}%nvI!9T;!BNnH|sv_Ms^B1X6Hk4+Kn3@P0yV_ z56VNw!NCz%P&{-!-SjMu(^2dGZN98Bs;bG=MQv>%IGFryE6!Ru_IJa=*qmmD!;n=o zv$B#*>Uf-8T<+M|)NLLii4srzD^6WZLsJvPiQ~Y5he)5mtJ6|P*__3`6JV0kYt zrI3qhCMqK+siQfSp)v_m@i=9jf4+J$N!mX{?*F73SB{h_GE$ZiZIotG7p0cO{)$ig z2C&%cH*bDMoZI_5{q{LEHM$RS-l&>;z&KK{+^#G0MtO@P$1a3OmZ8!Tv~E!LV$?d0 z(85*G|J0{&EP@vR_jUjo9EOdXH?y;`QBqM+nbbxV0#jSq+LBr`Utab7`%`3ZU?$&; z3^5Ut+Jfe0Ms;;{pupG1?2c^NbLgGp^q{2U)JAu(yTOX@MMV#rQjB;*@tsH5byrtc z39eD+l_hat%rct*S{D}=$C=?WRFbA7ea~XXC9!L2YKb?0vJFcS+mCfmWi%zArNz_NB)vuOG}GAlIJno zVLV_UkIE;MY>I5_fs;TtulaS?05iOEo}Z|VxjGtLIorQM!&+(W@cw!&b0WGuUUmUAYePJ zhTZi@NZ^aRp7tJ4O)*fE1}jLaGOE}d67{ug>x0@Z)4P$15TGIzfnvTuZ)XUf|HdV+ zG<{^KihzdBxT&EA-&+mwcg8vjR|om2$43~%%oK6Rj^63~=6U1^Nh&gDd;duUE9ZQe z#J`}ZAP>CiD3TvU0jH+pcSlY1p8D!Z>j$cTe?5)##WL*oMmE_qWZT`;u!xr~UBWr@ z7_5uw`txTz?CE~Xi*eUQz?Vwj2pMGp?bUr!;gFDc3vR7@^JeM06ZfU9w?8C7b4~EU zDU!TBqq!zb>tpHL+S*>fee2q><8ce(+!Ng263IVA+C?EruP6%a4}@O|6ltZpXeXXw z#pwSkVf-Vt|DNuD2^#n@EW`&cN^1P~XCcYwOmn42x1gBRD17a6+(u z+*&zDe*}v4d9u2eHW)k;6&3x5Xa5r7ioGrG1M}$Y>np)pJo5F8#EBC&Yk57IzZ!yA zIk<=z?{*kzcU|;;vYWRzOBglX4?cTqkz7VbP#<0{5kX&{ad%b`yW)cKJxIG!-N5mgZ^_!=XoaG zYETd7aKIW7W?u;X{v5X1-mEpVfPPBlSm8L8QiR{*{##)8=&V`N_VTWW(>0lZ#@H z9zQMwm_jO8`b!~)1dbqDNLa_e=ZMA`XJ<+19v^g0w65l4I!torFHM~DanSf|>kY1V z5#kzt>%sHq+p&B)zf+C(@892@^c3p#A}Kc;on8 zjMos)$;lC}bi)ZxH)}1&XOJ%MI1S`0(aQqx8#Lji3mY4v8(5UB}A$w$xqb*#7fSdQ&PH$8g|C;_U$-@GiRzB({2A=5ALZx zz|MZs(9A6Ucapw*$YNWbi@HjJmXu1$jUQKlhJI#MJ>JQ61$j~B+O=!Z(D_~&g{-cJ zVy8^r5^px0F$v+!EiZ32C~7nb{Gu;w6ktI`+jTy|q`x{$e8lg~n`nrCf#pq2*T_3D z9J7tBmkyS&xh!(4UU9m+Z0CW$J>OsP7#SJit#hy7Z+RLM_EnX<#a3pC@aOS)Zp+Kd z0m2BcfQX2QdZCy+w+&DO{Xc{2p^T{-2bKK%d8sQC=RP1fSTnxi_^a2iUo$0?ET-!f zZ4SuF$||p_QsP*~A^*3o2#A1GBv)DTKq{e`nMYM{)Kwteji#BJPckkXEUT!vl4e{Z zXQ;35*SF03U>m!+;06$vrB}>HE!^=7>#6CE|M~JTU~0z!1&fXX%HFEbgm7kY%eC&( zE()MK|vKn*t)9i z^6=@?b0K{ikdYuBR0N4D$Gm!S9awrEBJ~>q{q+YF{Hq&h5O4o`l~7-1T9Rrf*M)B6 zcEGwz8D^SLNUr7B3qaIGa{3i>l`l3nme5C>`2$i?njE3X{q?ce?gOyc4AzFZE-&C0 zp&!Spi~z0+0ZP0TGsiyL0JyK?kn`$*?4G6Nhpfsg%MZ;ut#=*RdGU`QG)S4qPxtZ1 zl36=p=sAacrrjtDk(WkV)y1X%Pl9aLOI|Uce=ZIVDkY^VrrD-c%wS9fEr(9-c zCjXtzQ*kUu06(Z!R<&@7v8D$ZW{vQLv|bZZIlw^lC%VhTUkdQyEtN+KahioJI*tRG zQWb!#gWTwJ6x@~Yot~a9`tpUnAt(<)ABv@JI&F>#-?G>f^pIkp2}aqHjajH?q>=b=9mA=kW}<=nY* zh!)?J6d|xn6!JCC0A{f27M7O3-pHfGU>y%>W?wmrgvmgPAn00cyk;P3p@3oeE*5F$ zD3uj(h3z|c%A)EerKVm9X@C8ij`Kp$BlI+&W&zf(wX?I^%5?PIbI-gyX`lg4QPGzW zVS26^A?alKu@R&|YmbJ(y3M<)n*y|2GohQTKOS-Uc}R#46!H2za0xjS8YsIrZ^v9q z;dEOX-vIe#sZVCL%&rOI(m{<(en&?~h5=-PNVnA$316hzH4`DTdh-)K1S@N6w@ZG1 zj)+H9l*7T8WMTkSV2n%k_umCLeHv68aw?KQFt{j6&jWyx?=KJ6NTAC5Z0C@x1}%}W z8&N!doC06|fQo6_mP0QlCKePN3|PGZnj+-Evss*oc>Fjp%n8q}%wuC?`@mvyon{3= zjKO&6CGGj2Nxd;d_>j&8GFBuMcmNw;z-)`*MgiN=w)1#ooBBG?02EfVSjHwM-s2G! z?z=W68xq9{{}3>3DEuVEZyJF z1K^<0p_^cGoA-JucW-YmDUTp2kXY~$O9fcxR-H6kFS;&YJEE4j4=KC658yiK#t#OR zLOJNg6Dnv;nZr$7ONmPj~mJ)sVtfxi@pF-!luoe38*w}&tryG?=AUJ zDbxoN6G}BvJApHq5F~gIya_FE3=-^P=3}<}_G3DrU!TUCCy|a*&=~Pyky?-uqP_s| ze(*oAiyOMo*q8;1A2~P3l7FDjG|b80!ge4b?-Lh)vuP(M2^t6qgsS3$)!D0?85lfJ zhyb}faZ7rWO+gn@ro(#vU%!4qV_T0b#d+}HCa7-62=wBX;g$L+$;oFfT)w8rg5Mt#ob#~dKUL*X^9es>~dac~DS^Lfzx{+}N_<@9TR|7MVv zmlri{e7C#csO>Nv(rP5yK*w!4>gwtOa3WFF86w78pqv0t6aw4_P!!*#+OUz9)@Z8# zC&OKq>9u~4Ln9#^(lIjb+_B^2jT@X+y%j%i)@GR0JrTF+wE#q8qM`$}1kB#y4!HRX zg>K)mxkTrA%ZeXAuI0Pt3f>KmiQzr{WS4GNvG;?A566FqwRfPC23$%h%7v^LF!Z&IY0d{)>9V}5;~%tR{_rA@1Ri&o%GDvvwuLh0qs0-NJ;OK z0=<*dVJ9B~h+ewKXE)MvsnRbXpa_T!D@utRb4VleSz;pDCZZ|67BqaK>dUEF86-}$ zKz}PhzD+ld{Ui7^ zy5{px4Z)FAd#N3Q-@KuJ*1CmFW@>u+TAp*(L>5F0l$8|H7owtrxETKN<1Zkx9GdSq|zaqBaTF;IHp09tgHZf6iN37<-RBYK+w;h=|JF~GY* zQwDaQn(62>m6)oRHgFmI#|xk;01f&8&hSoUWo7RWStud@``%03rB{6UOE0RV7rpR- zdWIZ9*-PJ0vuy_#>4!Xwm)?ekGTOu^@VTLtF%w`ob5>CiYy_GyZTTxcUteEZp&Jzi z)z#1Fg>DeF!%%yA|2=?dC+=8~vi3+}vyFrt}a&dT^Ajhw8nd zb)LJmx;Q&plc+0ediyr%<>A|gz(iT1-@uT5AYQG8JL6*%DTp5Z8?y50%a>o&w*&6@ zyWj{t4h;?6k#h#>7*3X;Vfn>GofE;l321nn*ravbw!wjdqzkcC$Z3fANhmJ|4)5Q; zwsh;poNFIJlU_W3UdZaUM7gxuya1+a614i1FHh(1@3d5W7VU<98vxM`Nclt?km3_V0Pd~XiJey5U8_uqr4-;u zJJ~&fp69-!dZ9PaocnreR6YN5mmN@i=ZUmyE{5Ivb$?@Wg5Sd1+_TL4f7%?j z833j)?!$Q+FWReZF+HgA!q3l!MPEm!0B6m=v)J3)SI27}gM1V#d_+Cn^fB5#!1GWB7?Bil&`6Pi z^oyRWcX8hR`&4M;8v+XA(|KYU|BQ@OVz*z5nX@2s75@JH7V(H8URqj8QJv-AHFTpY z#2e`!qE^6*dLR}w;$I6J?>P7JkS>H~h7$0NZQHgLAncCXjWCgl3M&K=SWsAR17Kp|dbM8k*21WzLI6Gpas8wjcq7ETiL~VD1l9p$1J`G;B5)YUgfk$BujTmEc9kfw#~fi@l!qSCRZ1hT#G?$$R2L^sdD{~RihtxTDSvXdTp6phZZqK&a;tph<;xKt5xUY8FX{uN9 z;Qygn*ps>7m6Rlae4JxHeh<+*&;k$7skZ&@gr&n*@vzcE1t8YAHflS1)P3kV0sF{0 zIL-Gq|AN|}Z*Hyxe}LP{JQvR9M`U-%FM@D!VEr~6M;SsJT?pT~(OdDx!B)vFx1?|g z;PxLI1PVw^r)OZ`k=+YLFEJw{5=c%lM3NpG8Mh3Kf6m0jB#(3Cit=nu^>lqc8a6oa zH^Xzq-*gaZXr|fxAe1{67M9viD(8&pa^}}xIUK2@q;+CnscUm=WMm08%!-L8%&Z>- z>LKVrGSC6+5(LLWQ~1`{NQcHmw%uq*-+Wv6R?&V(-gq;n$d4a;-NzhiwRhc?J>ei66`?Yr zXkc~I6Vo` z_d$iL$#-)@li<_T8dT+%#h2d@0hfd0{xQ1|?pznk_c51ZRbD1RU;T?1BsJ*4 zgSB{(#mdV5<*U9`(Bwx`T+(cJLlMAM0@%Oh#u=zq;%id(FZK#L5 z=(DgxqQb&let&8c@)FIM>Q4K-3)`Efzws8!P?M@->^AQylg2QTjyVkwKCMap)I;X0v(bmXBB@uj6jLqEVP z2+Fr0I5jwBDJ1Tr_G9}PMNNqo2dv|dhD%=Ee4Y1kO<9jxnjbj~a%h0;;=bsPALYpv z?O1iPTVfvc=%4JX68TjHJcVovAwNy)cmKUah@Fr@MWQ>OsT%{)%r?#Km z7#3JTELNl3AQZ@%hGjKk(S5HkiUr}RHxwX9(`Hm?PuN?rpTw~M)+QwX$0r-P=)O0#=-7`+>anI2COyHEXfy zufB_(c`{C>Gi&4?xFAvI-~;=4M#gC-wz8x8^JTmHSF5bXvw>tZUZpIT^kdLhek-#>gbb4LNX86GNOvpGP4DB!S7 z)Xv=>es>iV{*B13J+z46ZHsZ;2Q)JW3XLyZ$vF&AO{fV{Br5#Rd1o1y zgooHNDaCX>>$RV zz{=3__BrL&sQcAJ7s52$4@B{|8JS#~ zw=z_5Z*X!&%fll!_E5x~Elu8x|J6O8j2r!YQ7Ni_bTnc6KD?C|2>;8&>OAHF8>s2r zJv}{D_nlsz5lLX<9c1H4pk*fgSeSRf&^2S4a~&mDhx}*;hlXx3(Ka=Jr5Q2EQn5(d zzpLP#u#11=c)?$+ul;6wUap{Z?+=+$v)8~KFQBHbY5FRr@ao79q-0@7E1V+4?N+R3 zxWLar>`9i(V)_mP?u+jzkbvP#oIu7m@rNkm4)Nm8;NWHh+HvH>14xM1d5v3gonDb1 z8y2{)@v9CN4t5vf#4!oSj|annb1)Rg{Txd5M4j><>?uRZn)2>^bJ0$;AK?@3e4ZBii=NYK)YB3rotO& z8JAl|zdEAH4tWILqFW*w3p0Pe!-*I${s)JNG)Ew?RKu?bfG5Du-+@ql;OVJvW>$#? zD+M`1QVR1XA;X{THR%v^6^bC36z(p_@5ztgzL=;!Eliwwr<<=4;o*zhZK} zEZ&};8$#Vy#0fryN3R=4Mb+akajpU#2*J{eJf?u~kA|j&@FK2en7v8&bE^srM%dozZ+Iiu`=Okf$rcW9&(|gFW&S03Ny9XpOu{_&elYU!$$BrHT zj@S%w?Y;-Y`?_b_-IUP!MWRqF@-lB-L{revvw03OYeivs2{BReqDm7{vk&7Yk1ZW*J!3s(` z&3NH-cl`Z(9^GO(ejN(vU}2z<7&5pE@4&@8X9vT^%Foalw*y(~<1E@cIemzXlm}fx zsCZ1+vL+|!qogcM0ED_Kz)$F5h!VH4kRD^?_Y0pK}b(6_}DA3kbVsj zb}{5t(hE{B|B(Ov=g;TB9LbhF3c%Cnp?acMssjSFSz+glCiNx`Tey_fP!6|$d!wuE;TkD8nNWz8(_2_yo`@Y4H90-9( z{G3u!%?w7T}vMCxG8jxrE(7`X8t&W4(Mb&*& zkviEaCaTYq4ksIHZ2-9s#PAK!b2XHAq`no^*;0fAagz=FiXeMMKozpK$=bv=19*pE zno@&92fz5WvGFy|>YtvTBBX9{V0r{D9X-7<^-lQQ3*kY8xHK3v`VH`yux7}-YpAFk zfFvO_eyOkD!^f8(*>{GLsHKgGx>QJ^#E=fo1gVnr&mf&%zw3wBrs~g$gux|;S*kA3 z8zJg?%-X4-}(tWlffrQM{AzT3KK)$@=HO7{WGRv(ZiTY z2zr??_{Cd&ksEg{Dk}QZ*XNm*HZ9MXGq()_*0p4VP2iH473!7UyP-5>mDqxSh0cwk zjJ8n`3)#2)Gw>&+a_-sHnXmip(RQeYLb_ibXk0be?$(ax9;sjG1BFLKtbzOK2aRHp=t#h2&L_QgyL5t%dn%|H+6ya--v3_1!QDM+s`kXK` z7!qCkJIRB4{396u3`Zv)`G!pWzni@J_j~_sR3fgWlg?BBq>%r9#sBZdy#9amW-AY2 ze|&kdR3FVTE zVIT+gEw(_~q}&JXe#3?hAVCYWyc2KH0K^24JKh1mMcN2ZQ#HP?<`HiuWW43BzOsb= z{+oGoom+bHl2TGg9}kT~8h7ORU+_NBYLqVlpE4190tF~XC)z5b8NHg%IQceKl=uq^FKvO&bp@aajLHI(djT!lo@-I1U~1g5oA% z-X__T^8ESFV}0mHyAbgL*$)GP#1IRQ@;|pVagV~{L&n(v%T*zHy)5vpv zgM^xAC5@I~xLIrV0}qd6_;vt(<*lt#>s^t+Kj1S0R;Pe4;&VGJ%@MB$I^X>8G2n^F zfE_eR3UF6`t6oKPOwQZeA4RLAsHCJX_tZ!k~2IUr%)P1dt9CcO3|v$Ud94Ze33%|A^^AznFUX?_Xf+9yEWi zmoU=N5z8^es`bR2=`R)q=9PM@VJqZ1;H?3~SaD#3*x!~cWrwGkm|^n1O~gkO7AGes z$f8v4fq{&8v!h3kK7$JZ0Z5#6*f)hy3U?TYxV_(l(|2}sxI^zJ4MJ!**LyPIl7Ju! zkFvPJvq&r`NL3&*KV_4-Ee@JD+%7nJkTdwL`!ABmO-j-Q5kaR^QIiWN3z*>>nNFOY zHG$APc_)ev^!-RK#~uiwA%}skXRRHudKDKI-o-9sH+COi^d21wl|6Y9 z^i>KTZ*=nSLJMIo-6~B($A5f4*Am@Uru;xLqP%o>6Ug@A`AYv!4oNAgI~EoN_4O~| zN~6HYQW>at`0y;|_~4*A2kRuB;2k~y8V6{dhu02Z(eYYT5Ng54z9Lr0umti984d)E zoxtRi_av-(p)j;jG|_y4>XQUHl%lh%i+mug5ghe*al%M&pw~}Y(JxBC;$b{_k0^m9 zpxd4X7NAPBuLG(8Yzv_aL(Vwx7ka3Q<2KN|vsrxF`F2>MEX1V0~Dp$nHCT zzN`bxhCZe`;^Gc|-qqdx08kA7j|;Zz(mXeGtIElYWpHqC<-0H_PYCsIm>vXT=u<1n zL9SqiZG{3N8?mDE@F?iB1}L0H45La?kb(}Fi-y~Yc&%IoplYPF^(0xsp+#CtaG7|3 zutJcc0FgLQ((W4+Uo?cDeS)yUIvqWB%&PO-rZ^YCz8}zFh_ewOiwpWH95%Y<=Dx_q zkO2-LS43k@%^t-v`7n{FnD%bEEi#cZ1wESVFjT zkBNz?993ypo*S&=h0XQ-_Uh{p9bw~HXSwpjc~XpDW;PgwU+KTj7T856xELptAHX|+ z*B@wUQ)Id>OTe76A74c!==%FN&c+#zj+<>cSrhXZ@2!nf-|LPvei2?;Lv@fAB6eH9 z?1J)z!txZ$gr4dK_*&vjpL%&2Ag7Z7Gn_cg!9ITUNC$@v-I(Xd)l<_ct70D$5|q%z znnOzR@8V!*M~&)+oAr=(UJ$xflg;M&uu4Y>+iKgv9trP~614Hu!QXHLcw63X&kF`s zN!I)J&Y|%%Ofa&%j5RV@^h6l~%5JE&5g%J7`*c+Se=kLF^0CsGC0f9p3&GW;3 zeey`hq^&~Q>!1fn$h^5G5E%j!Wsk1Vz2X?@$wKQJBdfe9c9kdp( zo(=915eE>k7*rKRHl&Dh(;LhLjv-W0o<;7N4a#U1 z20myl6!uyLkI z$bEFM=D|n5)g3?^GlBtsL8$6Y_@km3-C%hmL#be3+~zSTP>KNj2UTO~(D(tEfZxnd zUmSTu9PNGzs1{HRpP+tNA>z=E@qGMv6JCJ$F1K$FvvEe6hc-|q;FbpJ#pC^el4^aT zLD>#G2Di$mU90m|Zo(dWqpt8^n)DppCB#pNOdSDifqc>xPQ3^X5b_rZ@bc$(+{d7C(UD)NXB3EzHod2;)<>ZdXXAC6rWs;NK zF8IBBa>`ljmW@c$*OYvxQ1hAO{62?}<)XbyfP=!ffBVCdLhOGCfBWgWtCYmPj(lPS zVx1vr{2M*Z0DD4!0CVlxiFU5*qmwl9Lre{)evpDvfJ-|BKl9FI@PD zHXa3;Ign>{spVD#Qm952>XIbpeE3qLqodKP@xpqmGv`BXAv|5}&>kq9varw)#!CUA zRQOi?@XNj#v^Lg=)6UPkq=dwvR>EpW1Z2G=HNtH%{88+Pp+oRx!-^O(EDe)+U}P|| zmlgaEK+2TIq3aQOBMCCZ3@XD#=r9nzN)Rd-Sla@&7YW*S{DRUWq;j1kHI!K#oF|0V zVdh~cv@qD0^ZKgapWXsPaTUrtQ6I=ejl-XZd%fN_d#}zfF7CBIOuF!pQSDKHCM7G- zmDRg->uHmz=*ynAE%0H+jcm^v!ha`o-MZ85J_Kq$)j)ronEZ;{)o{|ua2wJX&sfmD zzP1{Oukh9!yq3HleHKDiix*l#IOfsTets!+e0sVkGSoG@M{2fe;xlbaIXY2fya&>0 z+I80yXXyD-5Z#LOl1QH=A%8^yWy8#&DrcN!?&`#aQ{85XhleRno$6kJ?;O}>53-#8 zHB>Gb9Ry*AEP#R3MEtm=r65`#4!=od#QhU&V7bu(LE#uDQI!=Y&Qe4~UJ~XeUtqiG zTYZEC1tB%AgQbO&jjeF-L>7KhOe(=B)a50Lnk8@ta@IGDM*89?L&hSaB$l5_oHl0O zMOF>w{x0>!z=P`PvE!3ylD)=+2j?q+Jik5^PZGhou@^*B3?0g4slGM0&MJg`tX&O zV_D+}PJAlNbWowjV4R^4&lN_i_N<(o$Is6NdP!s5eoBHc|9udXjVDAO<#=VTixw6D z4A+Ro--iMQwKVhdXSkNWB6Y#$a^S#$qECHauq|+2g`ax7L$$B@-3clJ2w}#g7tmXa z{-YPGOj|!C1(~BD9d+bZGNppmyMcoRNh=XRK+Bnc}{Ns?|jTPG3+)srVA_-_hdapN*mGyj zlumXyb6~nqQc{v=jW`n^89Uw4*c$tFj*os)cNZZ&&Qm|-QIg6&amk@d5Mm4;sKX3s z=X+!}q(~7+?FwdIVCBzi+GYqLL@U1I80#N=?n}n$doer zK1k2}ZxqQO_IT&%7`DA`y@P`aC@27)-|dz$`1GCp1!#r|`5l@Is;3Ju^+3#nFvI!a z<$(7@J>-FEp1?&dc^jjbN2|m+xwy#O4%`+KE;=4T=DnEv`U-$bp7&V|2f5({G*RvR z!kggWANaaG_W7t)uUFl+FrXZm2qGl?Xxj&`A8*TNE}eJEQ&HlDGbCo&EJvoT+Ckno})+SJ~p=;Vm;64C$POAaRYxHqs8K^V(G#37M{^lr$#tw*tTzX zC?)g?<)N$26B%6(^+rx?5fGQb{N|%4Prl&7qAIacDBU@;ZB7UX_1%2?`dI6bGrpCS z_`%BvR&I>L9C8#MR!saw)(K+p2Cl0LG&dm@?1SD3O4=2ei&8*_-TpOQUbZweyh6po zsNpTiQ%;VK$!KbRpi72IBfYvbDY;nxNo5P9m^Oz#=C0;?sK0G4vzq8OPaF=x`y$+# zD{cruy7~hzF7d3x?GM>r4il~+o(F(e$dHMJ#n}1JeSMa=A_x5(F-~R8G?U#J?jg28 zC=0~q`XN$obD=2KfkwOo0sK~h!P%YK8=>>2QrFbqMfU-*s+s+IEBvHB; z{K~!okN!ZEdF#re?fnIIXe8DyXISzS;PP)u)V-gyikkr@sCT*tHKe&N3`8EUlyxKX z3CoAxiryv+Re#raGSCC@=q4N}A-Dey55upW*EJcZk(InX|JAEkb=jbfUBknQ5N?SA z0X;45j+R)j84&hC?YtZTF*4XV8EoDw2L}Z4tQ2o^2Ts_|z_7XCRzhwubf&vOw?f*R zJ%Ikdf%#EP6#p(_XJP|eoqz(JQ=twkGQw8#i z@wDV)MqQ5Btfi#|e(;0a0wcJ5?TaNOU$#hoKkWavDTOG@7ZKlq#mh^RRg-mBwV%U1 zi)F>KyoX;?td*+)IL(WV6_DTqfSXuu-|yeQUjziGbnt)9AG$HWkx)tmP6_;+WG0=& zYi4fF&h1%&wN$dFy4iLyF;!r|mUIUaQ&Mot$u6{%F(-Thtcj2gP+ic^`=D|?O{=ig zLln4c*XB@SGEZePUQFR-|2RYRR~qVt$7^H%_Ox$5CQhi#TkNa z!s{O9G&=&|rekDO0;BIuk+t1&=DE{*ph_5EQ^?pT@HkPq5Z!Qk2GubhMUO@eeXeIU zynoO;Pkg)jejn69(jkXy`wr||5PJ#Z;q==C^B(e!H@O8#a(pcuQbaB2>^uoEpLhWg z&yc;k)Em$=1|KroC4lK)|51YnZbi@7WbG$^h#kBd50BZ@J0LD}csvmBX8{5ms(lDjs~)NWige!B2N-0elX4UWW-BrkRfdfcJ`#VU&kzO3k*=T|t8!cM zL!_0xw(F3qlA2{AVaDWY7jv+A{P3`8-ohX+w4*q!)6KA#YPrrElBA+#|7RWQ^bg2$T9=iDl~q^nY+D!R5B4K!fae9^!N|8hY}tgqGhznbOyW(4 z0tRn-ck^Pu93G?xECT{%<=nB)cS8IENlxVEegvd6G1ukGIO}t7Bn#z{@DK?3*#+!{ z?RQd~gK$tW8G>0(N;|vRW7$S+7YKSd&-wjtPT<*7r&u=Qst+#Y_;e4N(qPiOf_p*W z$;Kj01B3m*%YtnSY())#=VX7OVI3L1*e!~D<%=Hgc#BolZd-14_Ka}0{p{@QvzLHG zfG;zKU9~kxGDU+&TDWg9krx0SW_?p#PqvH!+#}VEE|EjqFbDhPE79(3?Ch4IP&8PP zrjhQ0Ml)ABZ$L)J{YYr2Hy?AXc_?XbsZGgqx*c$h9x4LP=9gkQ-?i)$I@Q~tbo~2s zaN>&D?*IJX=j=-Bs8NwRXaEL~1Sa!Wmq+uM_+Y|Cmm690ZQqX#r<;)SP;i)$u;5Rn zOK?h6JERyau^T2LTsbkRn4X2}NZyLvlC!tZteZd~BV!uy6y8Vo8Nv2_Hgk2ceRU3f z|55JVy;}Bn`$C+200X9dW>#ZJ0}*kFa|9LiKBA6p2q?ifG?W$i5e@2>XpQK%LcoTX zxZIDHG~6&fP=sPeoSU$Atnp=LWITts!z{x0lz7Mu#k{gxETQ#urNxQ~LH=G0$$;P* zf$va|nJ)xNi}fPJ&`AhmZNAkb*@)r^=nG_|3K1LgWvl$DaJ5U6veg8O|<7^Y^$E zpc}3&Q5I=)%<=6U6PhplbBXmKiN>aZrq)1N}sFKW$p!beE@eSzd zoTa5@h(1o05OgS5ILzkXR^1zhi4IilkIzhQC=OQS`q9QgSOD8QQ{9hq$dc5M%o2SNjvN_z~+2A`C4Fe)N0vgW+re0PKo*rPZ0$ z7ov}XeY}z?G9x1+snt6lY*`Q)i;U=@JAwgF)xIDnpHSzCO~5=EPa>fAVi5~Xt^pbV zLwW%zj7Z0zq7MbuTBfRi0pcSJLPLdPI9LR z00TTzX}`$|lkrvbW)jiEiN~D-D0pN(2H}g2(PK1)n)52LXR1;s%$9QB4N;5ld}7;Q zO^J~Of_Lz|A?dd<1A|gRss{xCS!%W%ur(|>>;1}3fmwLnA|ev6A%naYtDfP5%DllHcadyHAB^0bV_LfRh`^D zh@%MEBvvc;cO%!KL)7HdJ3HgDO5brEq9o#xF(%oIUKr$Op>@uV;%38?#K47{QXb>l z6*66guqq~98*&#S0eNsKJ`WCdsp-wrnzE7reADtPg<&|ET~mv-~@HF16nOCG@Hh}l$snT&ozcP#^G`-+6^{fBYgJY*=$t`V6K4FJrr z>h8815ywq5WK!LAdHO1a>*}%{adCl?8`iby`%1~d=H}g=VcK*L9xw_-=z9>U;%snz zOY8(ii0gte1jxy6gyyn1bZe;}L?1qnDynxQ56M1cu@f zAQkx5$xgHfwA)Bq^-`1Sjtq@kcWOZ6kPRk=hrk(ZTFq>|;fK}J_`nrne-8kz&o}|9 zKU4}&BCA4iyAPEBO}mPezi{(mo_5VX$imPf1YMWx$;~~Wn_lcy$k)V&fm`xu$Yi*b zR4_U-lMoPUFb9tCL7neHRYChg2#1~AF9|dmw^@ep3sUa^!U{0)$US@>+YO2rLJ}aY z%+fQw6B*|MY9SXu;Dlg@wj%g#hZ_sQis@lvLPhMtv4mkx8WyYWbeROm(?2m-gnPK0 zF!Nu4;^=JyBS$ZW_|cAff_j3?)fUKN4!aOhAwigS;8F~7vyApVL|c0BB{YCZpe4#`jQ;3^QjkW>P+u5MvY|&8ST}rdBXUBc*fW zrZQ-JxTTl0b{%I&et;IBxkQcAf)T(q?xeD!8`fI?EDX__g6mN~Y zy8KrMm5T?PuR@HYAO|dzvf@U{_}`(wm_Bs>IrIgRC(=sHviXuno2h8KW#czc2Qu5n zbs?w#-y{ZUz@6u^F~(T!$PT zSUgMQH;_uWN+`&+6KF?t;zWu%e(C0dTE}Ax?vGgu=NkRx-G~JQ4_TFjgoHXl%ec%OT>xXs zDl2>OYU>IN1g|aA^v+_G(H@nP2e0HjR=7bU$No8}wcyR_cWYh+Dsq1CmTBj*OgLv| z);32^IN{)+$&zbV70$Jg2X#+za9~$f(f;k*v#bb+I+N53l|fJM{WyQX>(`G`QYM+x z0-il9Kq)zc-=M_MB0r0sJmzvC4-NeMuz}NWAf0p9vK1T0!GmRU;<%<2vdCM^bL)U1 z0?gk1OEMQkG2G);iu7mcNOOD!(Ohj z%&vHKSiSyeXiEs+y0T?S%br;WY(BB+PJY?KM^WF5!QrCgAxcBr@t1zIE5v2Jd; zcwPd2rvgMNE>0xu4L(C>r_FJipI?%jt5GbJ2?*lTHlsGA92$D*?|%xa30zbHm?dN>gCeq%wZz!EWKA0_ z8oL&<6iUii$KHkPLPbg?Ns-jptFc8=sgO`fL>Q#+^Soll-1pq~@ALir_50&@JwA(R za$WE1{eHd9^Ei+5IFDC2+=Xbw23}i!9}+B}ln}yL);e;;yefkMWZbK}6^hK|2xVg1 zEx!@THt*WyRK9sj%O*_(gi2q93e~FN4)(wynpHclPN%}V1gBWfbIJMTVxc@=qH7!wvF%#A3kp!YX8`}fbgOkTW8`W%&CW6|?d zPXt+XTQQkZ2;p$6>mH;L86#mDfCflc_-)bG3ab<(C1MReH zM`~jns_9H!q^GZsvC1TZyL{rSGtbLB`u6XSufhe}ISDlf3Ii7L3}>qc)U1a7eUZeIJ(=S?pK9mg;9ld1N3GAjhb>ZsO4b1iZfVnI6Om%V9H!>2}YP`{|Fn)zd z-Q%n)duO9eNt9BJy@!pXX80M${Pojtj{*2MP!^WoKt* zc5}_@IpE`q!7LRb!^W2>S)l5j5R+-l3y3q(rcGm(W%Ra2?~fKjZbe%>he(RlWm{&% zBalQCh5Z>UghUX8=u&i5F@%(}_p}D|fc2A_+0;~EEABcXxO(srynOjUN?htbhECRqS(-F|QTS0C zy#tVl%!Lw41n(%iy5q+1ZU%^zWr$?{u8p?&S(~*>QgLz?L6Aym+>8JaLww%Df)bGLh)iUA=EZhkf zIm#q;w{|2H=^1Qk`E**?@}#&%3d&x21-&m>vqQw4ig%TsA)JR;4dhK;OEm}4bq?+Y zYN`blq@ZqA*U?p*6pVX(1Gx}*S+!=(M`K8{tyf4sL8kHS4qT8X10Hu=eGIko=^z2& z1tI!4is*9~8zz%| z%gW33I(NRNHM{jtYlan}GNtcC@}rWNQ-AL1akQcd+Gh%bP6N9$LWDXiQ^=*ruhM;U zTC~QUe@7}2E+G8IFboHyPWN$BvT%x&ntp29pK4W#v z=I%H0fB5sMCC)Xy58x!rca)d;BXyCW{`1g%t2S%?l3p!C)8m`g3xEIZT!Vp+I)C|_ z0~dF^c(Ee6VEmS>obCDdx)d#&yIX2XYRrZM9UXr{%hd7FnyQN#qNCHn@!laiy3K^| zQ@>6n>ZG&+k9_};5`)&B8}9Pb{E>wQ@555*07E^ED9y0iUT*yT!op;p`+5?C;08z5 zhR|vB+IzScS+G4B&<{ypNecYgkqG_e zWqPxGY~!v9__>Sx-k&rY)SHIs9IkE&jJeUv3qMLup?eGq{xmDbfE9I+M@~T_IAu!r zo$qLW2&*ktsenRE3;JGP3ZHNWK9Ao5L~GI$lqy|GU_)`&SLQbm;^F(F`9?m7m)~UF zUcF8p8ypv=lT~dP38dT3k+iK|Z1VY7JvwteQJ>zotTg&MFloa{!Dm18cDehVf--C= z*0q#PeoNkdU~$V}hk`1*#=OTjkO=Dv7A2#DQ9ljX6IJ!SjYi@JXn(iWJwF!ClISkilLPS8Jx4pKOtKcCO(wR=NAr^fZfF#^%c;x|2ZUnMbzdDFz&(RY4Z zkXhXB!=G1?xy!5{uA7K32551C_aHPL$DC-)n9EbU`|aF0;r>#9@t@DF8A)aGy^dHX zX5tP>LZm9t@P(m+2U$Gn+Ni4a^Jq}{jWL6l~bl!E>M_0+lZ zHM5-r)uMisRm`^f&k)}KS3h>C*WdTyY{jJL1^j?-tKI;yZT}ExSWRGg8<){M#n+Vh`39}Vq^hN2x4H)2_iT7b&;@VN%|pTo$Nv=&+mGpT5pYnu)R zmI@pVVl^qaX>zvXv3A{kl_aI`C9kG|<<8X?_{1yd%J_!s7k?&+BW_&T^tgIy-aAD9j8LZICeAN`{e!AqvY*~-Hx<#fXa;nO zLcRzOhrS?xX!y3mOVga10H(Jv}(R$&R^{jokZH&RQ?}j|mcpdBINDZ`& z4pHvSv&6=>ty@1ldD0&H6GTs)-MrBQ_C=MxtT-?*P&sByWoozF2y&S?(||QLOhZe< zVC`B0@(VO&qW*2;4-ypMCB`k>lhO%xpTEw<-}5>kl89E#*_3!mH3AZ!o) z#9wF)CWEV+3waC#6s-}E2Uw=Pa;vJl_BdnbO`ZE}U> zfSOnIHU6pXn0jYWGzE#h2iZU@>iAI~JaAwU+t44xA`oVB z5%wgRbY>_R+Av4S`+e7>$#7Z%j6TRIKQbK&UVn+EuvyfZ7w77U_oFX(Y|D-v-y|{T zw*DOjoSVCQf>}!`Zh4u$TwpujaSukA?CkE?0?WNt2(=FO<)8t&!CQHPQ4FB4FNTxb zB%sqhGD4Vvob5QHEize6opN1t0fG_fG%G|!Ms@-R{?M}J7HEs(JN+M5~fFy7%t zsTx)y?RNkP(MV_)Y7xcQ;1P|8inLT@SOS_!1G>JtLJAU?J>YH4`+$E@6!8X9?lD+4^v zPmaTWY4u^aP>0q;WH~rt70XK2n>y3 zmF*WK>9;$sL}`TbU=|rh#-d?NKvkRl{Z~QSzH|%NvBMPllUOzb{IIy~RXJ&i$qQBt zEBKe!am0ys{rWZYg>LP^Y^8#L(0A^`8oZ6PPD@1X3n3*Qt<)OBcyR08K`$2J6C07Z z5OaRNJb5aiqu!PFj@;?{1PPvohKxNx-?xUY`^XIx3`sC!GuQs;Qxg_xFnn+t0$`dS zeU^S)v@GKp%YjQjQ@~*zChx_5=%(J$=n1#Z4y#>DCYwaw zLTd|#25hQNAVWkT#|??Ng7WbtdJ4``6WV!{n_E{}eZ!h#z%IR6{;ixO0c5D@OOuVk zk0919s zz3I_AnMD9sh`JlB@{l)@w~x*BTn8}Ryko}|g5T^GPl!k8A$GtQHAnKYXYXDbzout1 z>`E{D*WP>jvQk~(0MuHyzPK#8>`6DRq4(gIZ0HN2l{uZ?i(o333)4tv2`?M_cStZy&% zMg#afJzdgDcj!=R&RJO}3bjj13kp_>y6gC59j~LN!0m z;HarW$Uu*O(%oOaETQjuu6Q&CCS)*zRR+EBPi=vG6rK_|tw}vELubNxg?5eVYmv^z-USg>be716?I&wap$mn0LE&7T@`5St#cXe?M?{b9jN`2%=AJ{4tI>@WxbvnwT zXNUfJ87VwlUi`yx_UJC zo`A@vPM!KFdmo})1)Hn7e8xvCUBJ)S1-nO%fi#d;kj+?){4qSBYl)xAyRcVeYF>Jf z_?F7%f@*c_zLT%rqSM3^`?(uS(DBNtZal@n)t3TB!Q+TjE5#sE9=))lbLX}~cgL2= zd(uO*3N0Hl{5?Dn{KSyMpp*sA23iO zYU0{las=$}H%V*=TFmD`o>b;P4<0N^T#-dkZUAhh@kunl$PE?$SKtvK{MY|zn?1x$ z1icrXt$z?4mt1-^DcknBbZLKD=X5op(aj(cnGmZlJPH^{Bz>@~<1i;Xfrr-Nx{KZzfk!}TNiBDZ^}SO~94c)4Ey=;vFVODCBsc3m1lvX7S+s zK>H?F;F5XU5d2EP4rQ_`8VX zJzEZCW4V!Q@ZrNhwrDYuj75!cvS{cp&bDr@R}i^tkFVNKLuNncv5Uz#Pp?|v`ID%L zk+F-&Oye%vD{NS5?&q7qJ~w)Ii17hUVhvDFP&j`B4Z+-0EhchP!D<=;w-NUxHb2Zd zP=65@;{}6F9hSTtn!EIatH|a@K5UBX95O07Usl@l_VIh(&L!>L!qx(Zi*0jcUQ^EZ zg7RK@c%SE5I>A3SYc>K{KBw7v;zU!#n=JHT4E{*|UGqv_JGd>rapdpnl>66WN5^KV zFShT+WC*Qp6wIra$9h=mz@*k?`_oKz{44*l^y#Y2AMJt0LOzqun~2a*{c5;j_3Uqw z^DODsQ+eC}op~%o+FSKnYt-m(7IVC4-qf#u{P>pczM(cTNro%6f4ySwcJNqr>RUDY zvF9>%ZTzOWEV_F;C*P``j|mtqx|#31>G;ZswMhT78KI+uxl}EC7qkP<^ONmMi0%7x zO1gZ$&dNxeH%Wb1>fnOOXfVhhO-9s*GE*LCVO@1SL47)S)D(kh`g%iI7BnWP`;c<2 zOrAO!bP4#ra<|vN)sF_pN|>A$SaHY%RElsRlqwvcq{ z86lP*_~V?wfDma>s!b%QALcIpe5Z6wFNXU1*{*o;>Qx~1lj383%St)Zpy$jRzR+&; z2c^TVSi^S5LP<0%d<&M88HwYpYe3)mt7ut$T@f-9a)GuP0?BO*!aT~H!^bje@&ht? zl=}0y;b_0Dbx0sbHAWf>6zRuxQpt>uL!~z~YN~>;{h1 z)NiJwstNHPo!BJ>JTjV9*Y7Unnz0xVDY^n>+rH90O%-B{j8sZ|#C+L8>=m`E>)W@r zlQ@h8QrxCw;m3D#8LHKonRTK)+K6uKIZ;}oBPffQR9*6SVkmWiES*l+)_1TDwh;n` zV^PkguCvq#WbeqB409E@-dv-877;plV=5%4)W7P+&`Z>roX`8g_M|J1j2;9BR-Byp ziRzv`uUKE%M3+8L^WA$Ek9T!lmmb`tDiEJKy!NWCDW+J(X0Qf3ddz4*jU`=jN~N2q z-or|`@ZUCWtaYFAofbzzQm=ojVurE2)H8;Nlaiw+5!1zCOu-sz!~ILJJ>0%(8JFGA zKbfP*^(r-%qq}%S%)w59J~u*!WtA=R$oWr3(=r;(nVZxFT2ahp zx?CbFAUKh;W~Bx|0z-k++tcQr*5%qQm-r@6J;%5;upsQWA#_;e_2|yX#_KMc-yL+% znH?lY(-MRVB4WLv^l9Zmg|$MlIvui^e$U4e56uT_(=6aFmyglK~z+8TMXw|=H zJ#UU?s?zB5bp|a`i)z)^{Eq6xHY?~y9y`fR+`0Puhd(_M5^+ zh%)Cv^px|pX6##B3o#kD{1Wda6)P$VIp~2HOtm5s0r;8)C{&V8o(EXcVwo6 zSSyR4>|H1ZrT-=8H?m2R&tgl+VaAXew@^n>Zc5^!EguM~pgX>k(fPI`cet615dwJa zex8!o9gm3Au8@4iUv@v{UQdL6qvrZ#j^=3Ytf! zAO9#MuXx13rB&~1Yz(s{YoB<6#BD$;!Yw-|?|d{xn_q2l=PjL(P2gk^_PlMqOEnQXa$F(S?6S3HgxYbJ0s*pusCH{ZZkx{+FJ#>8QAym7A*<9UCj!#)lhv z=N_9ln^_LZb7o%8+4tc9V|n?lK0mnkZ%^10Nem*PFWH0F7`fdXBVYwbC=47n?2>)o zYbc`6^Dsrt10lo)P{1}{<8LUSwraybEG$^`d9hy19Gp>o3Ote!tjK}WPP;qHpaRWo zk^NES(+uA==j@-4yTtZ~sSqVmwXxCDN5LUT{9`-A&Ab4kE5y4+^w_j)sZQ`2Xnglj zZtX4&`?+u0BIP2VXx?*2$5T_<1epRR4*={|YOlJIcd)ieRcGN#laNOi^XtCVEl>Z| zAuPN9KklN2e{|^nMGNq+nf&XGmQ6dcu2mv#FKReNr9EjFDTWeIPxLOMW<>BuA>J^|dg$t(xtJo_5s(ED+ykM22wRaV zh+V!^tfVnH3K$(rGbx8378j?&aWY>L1d@_14B`z;AZ#L&Mz7qKBj(UC+=Mff2?Wl+ z`epAS!Oj}$^;>b`#0jsZ_5Hj?RjuM{M|{W0F}^Y!ses3$s;dEQe12+oj?G<1>*D0e zzS|>ZG@b7#4&g9L2qXv68>3FsaKy$?gpDHKmlQ10%F3vPPhJ`Z6jol&d!pO!8}mU& zV@`w&>6AWpsIA4J+YYWBe^hm_|E^osxIt6ubaE|hePDb_hRQLktzo!ceZxSrpcYv@ zn_c*!k;S$ljW>3@R@yNC(xjoE+6_OR}!A zyH@?<6T?rw`SsUoO*agqy)XuzZaX@B4ly{wb{nBOY88h4?ACqzIx9WlkzO;OJBzkV zQCe{7-kskmO^402)d)mzyUS?taajG1Vd)*VBnKgeaw0Sr<;2I@KdvpG;M?QDH_45V zHV&4VJ&xpS4PJstxb`NO>j@{uIALbbH>q2{z6BfAl7U9Zqmgti%F4>pkfyw*Z9786 zF|LvPd?tefb?el*Ntio=G|6!0jr(i6z1aQ2&BJBWa)X~XX($vemjC|#L-coXQ&4&w zCS?=>M>>~#M`w2Q@u}?JqLG{P%Zm*oe2rG|>ksN)nG#ob-k(Jguf2C@YkAN&=n?|F zZ)WUyB4IFZrcflcADeVch7hv)^N4uusoRhV0eLmpRIQf!4{nPzyQ=@-dp^)t{kIe? zbZXXD|LFdUca46ieW&Y^y$ORAvzHBi(?Ahd^Gg2lpS(TBc&+AFKGo>oHR5;f+z9}J z^K&un$kzH+{dcD|lz)W=-Q%h^Q~8MH`;i|?dJR*5{=iK0wso~2!#QDYHI12V7;^KM zXR)pOEqbvZ1}l(dVbHhl1av1iD0LcXX@!uBc8+X2ZQ8UW6g>z%Z$TX){qG(=DbYc7 zn;JHOdvNi+L!8?nxZt%LH(HsRu3-Rm0!f6C5<=YcK4khQZ=eEni0^FaJ znNxdIePwI8E)@$efQ1JOrXtf4~3-EX_DwGYTGEf8E7{zP^f@wQ7w}JwILC z`Ra~tgZVb9ASS|!^kznFL+Bcl`1sgG>Cg)Halb$x30M$OMVcD>>hll7^yzWy=oig#1; z(%w(PC|~vH@nfr_d8Hq!mY*u+ngUsSP0|RuGFg*#<4)SVOUEa`lU56g5z#t-Qzj=?=uzTf!xpN*1 zFxnud^j~-Ix^i*1kREu#%S+Ac)~k1m-X+tjgoscICp&AGk2=PCw6pW?v-$Xt%Z|t9 zE&aXK&a2ZIOwaCL|NfyD|GGcp{%F~%YHw})&-R_ZqkIR2V)c{Rr3$xc-=5FN>|f)4 z<33aIH?#FM@~|k3_EK+Zx7Ek0tG}C&@g&7O29 zCDcl|OrC0PVX>7a>rvFlBRDVPJ%Q>qo+1)(?tm!Euk|mzl9OXkw7bJZ=?O-`I!!n} z1hmMZtOd6L$K`}YItG66@$pDO5Aus0jE(K-u2@f<8dSg5a!SGw8eqljT~83Gt`gg| zaTGcy$O05?MF{^k^8^PD^xfz6yf06hl{Q=sX~);;C?ztPd$fmxrTQDS*nO)}S7?ZK zbo?$}xbPH$vu=Y1{C4w9{fRh70JnLZ@VK;&D@&;((5c53c9B4bnxO^*J;v-Dn zhSF~HEDn#|x_R?{Ae0l(7+$^;S+2TbSp4Sc22! zDE)5T#?cNed+}l{qRbOm%VryQ&OSB&utWe^gsQ3CciuiVf5y;Z!x|UIdVlaZ72m($ zxOx*f_8$HAyX$x&>$h*OR4rh}34F15@#0x5TvZUK4?Dn(o=oW6p@ZDPt=qRB=JuGX zm_0A zy#zALHV$+`yO!z{<3t8KZqqS#?lkoRcC#NH+ej?>-g7o@Po@kTX=>_s?{1E4?FPLA zvge4V^=rw~gorQ)N*H=Pv(fH5Z-~!_?7XQxo0S6ro#AY{?;IIAE^rk>+Yr`Sllb9W z52v0<^bg&?e<0+|UT(0SQjiVES`MQkjg5w%PRHWeoo>j6mL?8q!S*d^0v&*o;yQFG zu4~q)AP8Hm($&VtN9VlK7{1{><7Xm9wzXZdJR0BHSwz_SIzQ4{kTn3>NXN4z4|_|GiP zOH9;mT@@5#tM~t3FvwrAaJa^oNVol6BUS%dl)I-H_Oax7WiO3f^`Ln62V>vt(#;9g z6|*(U;(C6Gkz-dKtDfHvTi-%hr#B7MZ{pj1rM6b(e3&M=Bf|p65rI5uDm7}_^a!_0 z)*)x$GLw^qb1C0evr(g9DA*Br2C0Hz%4^oFSqOD|i+W<4pWgvmPBU>96i7jvPE%7e z2!+cm2>P|Z{pOcERmJQIcCSuPGSwe`|E|8{$Lmy(=}<-+bc`o5QG64Dq<|cflwRAm zPLPMZk&2DRbF^S`&tRkF*s=S0nofLe0YCCU+4<##w}t~|CZ1Uo1&tTN6&Q5xJi76^ zp=j9?mq*u>8=rJ)Y|^^`c5{cGJuUc7a_}x=*798uH5-_iOk%@j7+C=m^ptc7S?1xo z$+vEeXSduzj{%w2MJ|)R^f*l8^c_Hw=1xwb34@aiS>+$SelPU9?AGW}*@i&hmnLYR z13r~U`V?YMeY1b*nLxO@BLpY8z2&slifGxrzg1kXZe5p1d`)zd9lLIQI)z2f0t;nm zPNK2|*B^guGiOeztx0=@Vs;x)z1M1A=dHVTr6e}_N66W`deNurVCAhqNN(N+?fzEi znL;Nep4FWNJ~~-#=+mhbbpHHJDGe9y@k3u%euL{tEY(WpkU7#FCahX<}1R zbM*&W|8i_%xRqD!WvZ|C)?J-{+gpEpUv-W5U;N81`3nz>OH#k@zk>exu?rP($Jsp3 zw5z$*YjJ#`BB|3rol1xI&-T4(!1mla=vh)+E&4px^X8QqzRd%#Gw|cxyiGNE&;2e3 zeCK|9a!%e-`C+EMxZr>*W^t_OE8X7Hcm4@XV)WXz zD@F*=%xKXu2!V1e`HC6T98a0VmNw+9YD)&zsM@Tl3eu^n3HT|FT|qlcfp%y6Tao3ind#n>;qL0$_{5fPU>37;Px$B229T9 z&KnTkR9gcABFUDfDuPKnHro$x@jg8Bxze3BeR>B212ffYCfY`*kd$x}_hk3-VLKXD zZO3Buc5MES?U++gPYca&@Y!ouuG|nT#~Qyfwfhl@Ka~W&hZFqa5-3)yx#CSXT++FYFVktp@U@-?xZ-tys2mFgg1h4N+=A&Y}n z1j-d_2T%?%!{W`lIXy?aZl#rl+>#Va8;}0aZDM?Iq<}7cR8SDivzE); z)%BNODX>C0CPQv6QfqUta!)$11&<#$ZvNq`@WAx66P#e#+qW6m_4S1Jqkfn)emxm0 zcjJ-j3+sPfVf+;fUKuv&rx<>*iAh{F(Bo$opu?4du=upM54wA;sG+{@J`Mk)eg9Xn zzb`(adeSz-`jcjqe&=g@pX}-Tqgg=Ct=v|;UD6;@X9FzyjxGGA>fQC~YArP7cB8{~ zDy_=eZX;T>IyH<0N=UpxLbSB82_Z#xvN<-8oqG#?VVl64A@c00jJNFGy_>%qO6N%C z`ckf38H=R|ipK%8c}7VhL>Xa;kV7yuR$5yj$xDofsz1C%V=B6vO(rdA#i|dtiTW9^ zmC=088BpZQmp6Ho78Vvk$XA`9^jH;X^h@=WWNO)HVhxcq|GKd_kkBd%1L&Hmn65rZ zM?7CmStJGgQu~r&R)SY7VV9hxr^GGeS9a8G1gT1M*zDU-bxLzcx@}vwv{9aX>k|p2 zYsPSpgJ+>$1;<_AgitQM(UF9Ux(u*S%vimqdR$Y$kcJv1=Mf<@*barz(B`C47gy@j zZQPQQ(!9izaAL)uu5h4^plyzR>+EnQginhPY-XZbMx6f$4VHoi#^V9R}*hqAP>8SE-o3&Hl%!oDBE*W|v;VURC zJOWuDm!wKetaP_8g-@V-@c>!()@H2^{&y^?%#Db3ex~u9}(|9(J*tX9jIQzN#)y4Rl6iH zoO!7Vq;%xWJt)X1V_+p?iR=_6LCaV_A zvc5#c76W4>1?Y$oBhq*bQZorF0=Rj9+2oU?0x98SSCijka2MwxUhV9xz8c?X{qZ%| zL%u&nXCchEls(^lx1Yz0D9#iMjD2EggJ;Yjb}U~r4cD9#Og5t264vwsSdz&MfXlcOG_~2^5x+|LO>7cCUbtY z%7PXlOn9T_8ITRwn>Fh;SwQ9ZhiBDX1^NOr+Q`DC>nwRONXdC}dmlY_nEt7bqR%pu z{yNrHB$DHu-8}!wOjZ0%!bqbF<5hx$K*{>>THFJ%0%s?6;5x6~IgIlyaNq`0Z$}Ut zXJ+NxyXO)aZsi;`cmDkOV&EAL488r~)60ppRRUk89dBNLm2Ede^{TOR*xBAuj>N_m zXqA*naC}F=1*#xQqc8}JwBX9JMVx8}K9#MQ(Q`JL+z@Xog3wb+$FokAG+YhX+N#%- z3t=H4PixIk@6v&p{;%ahIcvGzwSP+o+}bqEqTFFW)qybxKz_V4}q=hT@5Szq_6q zj_3jw3y1yQsH@Bo(Q;$-Z(eulmAx-PqmRM0x4(a3LaLJgq?~PM8@{TMT#4zxSPDgi z8%wj-Hc<(y;(GVkAV{)1)Z-Kz7qUINRNb+fXdq7EtbsF{Pj)F&iLX(EOK4cS^1`yOv0jqlh%Z=AVQula9)~R1Xt$l3Nvu(Fo_Z zk)JSu8i|lbDMPb7TSHXl(`L*tQ;F@7&}KN?1P$O2=?^0ts6VIO?+w6yK4~g=usBs29RyZNu!b%U!&2GY+OLGLyX}6Zy}t z>`~kwp)SUL9SrAj^jme+1KVY}Ikor3YO8|iPiK3KGd0IRTUW{9&@7T=kl^}C&!-1g zf9iR&gGI=GHkGDxF;={6`a@KlLZX8!(~xlO7xt454kWMzM62tLMhd%@>N4y8W{X@kPNP{E8~BQg71%H z4A25CrX_BABJotJ;`^I5lpB>4^RTIw&q3C;G)c(udP*S?rMt@($lUm=-I4AKCh%;~ z?%zKAJiBGa7cIb;mMd4RIA*D?3VfR<_3|}&0FFdqW{HJA%>LHI#6$*%0wJV|nSZtb z&hn414o|rqIcX9DSeESUf(H){&;S#aYQ}=f*O|t1F7{N!|KinxEAzFz=vYzFv`(M; z>h7nH*J_P>nsv3*#G-7csylDJFu3%5i_g84|FEa|*RJ^Qy_Fa3{=$k$hI7|yt9Jv9 z{HT)%>ZG2b<8kVtJ0u|Y*PuLshZRC2Sq@m=xA<{WJzZV9g?7?T0U1YrIGvaMnJpPV zMZGHT7wz$^y^5&;NLu%t4BAF?LYy$Zly)&j9JGXgqA%qwtY+k5rWf8k6%MVu5wLorE>e*inn%gueXa2M1$29PbR^P4~$QdcMX0maPPSOMF&u?xh5kuG!q zG}$5a3iMF)VFKw>tAeQO+d~-g=al5+aV!vc6)E5{9K1%Mo=*H!wmnh*r=LW2V9q0p z!6gt>36QZStKNoOnAUygR#G?5v2dHU!%a{Q;1luq0brLCi1a-MA)ToI@Z$?X%lZ|c zigUo-#fDYLcjV~tBG){IJk9CD2V^v!_<-mt5%Jy@3XyOF(F2Nf@MMNaPbn-CdYc6Z zLflKTEIF(LX~brd`??K2r*=fq6eeSh=yK-8ui_C+0y6_I_VaulBBdWXs+oizxgiJy zBM>$UX@GM6j{U-%V7wfkVk0WOTPl7`d+3X{rnsGjOiG?B#rz>|@e$T=6`4~5VCzA1 zRg%2eB<}r3lIMHH@zhQLWwY~ec{`aQ>?CJh@?|MHMrAx{bafB!( zj=W`+-J}(xrv$0ofQ8jnA~`(X$oMl8^Fs#JtGUVo+g4ebBo8afa@5;;N}l>H)CWS_ z0?MD%veHn@?o<^^UPvry_upd4bIpb?&oonK?8-402_O#f@!OByyysP#Fk1lbG?t~*)+H4zWM{Uzc+~PVL#gMAcj&2? z>$L4=nqB2=WYlSUjlHa3sfji@0dE>MXkbMNRgz&Mi8`^OWKO@i4_5%34+I72D$`hp zk%UgWxh(>3+BkjRnyQQxg6Jdn%mm7NS-AXd$bujicaD38J+1ID@qfN+giQ3gZ=O9=uk^^Jyy}$t;GA_15YM z!*lPP!sF5#8fgke?AqTaW=;5}osgCY4Qyz=-(>L_Qcg*W{!N48RbMri$0qrH5q^BG z(@qZ6HSe#`zc#I|7T9t;1_jTws{|i$-`YKFwNE~r z=>NYl!+$eGsUPe9X5gfC_8Oc}Y*H()+P)^Hf1!qqt4BN>*K2kAnR@v)rIVSr52J8d zi9VHv8(=oVs+uLUC>Wb@j1Bl$#nsjNIDv;I!nh<|ofCHZEmpyXO;7O;3)~B3dIv%TulB`tnRXZh4-vK?5+X;_~ zHg9^=T~ecTXNVQrecT4J97M^(8*D{e!dQl_@W$aj7@AS!EVjKF%aRJD-X6@R}tSfcgm9xutVXuklfiAlf(CbVs#O6P!gCpF^RjY*9#&&s+w2_g) z4Iv?HrIUo0%mMC7d)l_dh~QiwnvJ^#c%LTxl{2_jMD$d3Ze)dSb3gk5_&fw``p-VL z^u~pXaCF=+`~;IL$=TT-Hil~}s)(z@xaZsS_$o}y*)zUzSJ?H}bS_L)*k-q<7f%7A zRMGr%E!mz;2sp&v(QGcmfB@mK=H@ES>T~8<%)|~wFjRzRB_Js^Bg0ztni-JHEXJK8 zx!+-w0jeM*QD}NiRjl81ylx~9rd+#rhd%8Ijy5lUp7UY4e>FwK?yBB%2kAXG{R)Fw z&*-(=%yr?~YQvycIGCXe*T)3>IJjuhn!SDX>eaNK4*kB(fBbm5bKydnQVC+ z*9Ndxizn1oKlBvc!;5OE>sDr$Q4?r$XfchmDDBcd#5DMRSmOxiKZJO|ATXw)0@k1N zZah_=d>#Zc$?+bXt*2ya(~=In?h&Mh@5$u4NFqk10aEoO7D=_IM$Z3*+CCU$rk-e^ zT-tiKX%$s4VjegKkD;5QmZF=^EU8ks6uMcVRQl34MzNY<{1_h%O*(>f$x^|g00If; zL1x)u0R%K}X2)pN--fOhbwPT#t+lG>;d9`&G4Ftm;-|zujB3Zy$$EU4BRwo(* zsj&eKQbp@2+05HHT7b1yez@DA>(3YGysFlsPIkWBwR;_E6_;M0e1D^yl_2XI#QJ*e z<#8~vC5v@@qbY9wt+T8-+|(|aWb%$QkaNWWD=MbG4Oh*O-kJEZKvxL=>;IlBteY+gfLggx*UpObJc}sLy zMClMH2uC^tz{sMLz^2l=f~FJ39ZI>7)fQ_%p2~qj$e~NZYZdv~{Na0a5qGD2ZjjaO zY@LMSm<+#7>RsVq5NCn8q*Hq>z$8*4DC-5PTqY`B!|#MbYYW|iZ(|_4yFD#tI8KLR+7heU$j52Qaa6SE3fnOJ+D382ZI46MTER{#N}K-LxFg0_R7 zZhFYL!8FxXLAb1s;Fsy0;5UFrd<@@bYP2sgXTwp%l3Bmxu_>T}dTenZ-z6j=2LH08tAy0aM z&n8!(qAl@3e2JX#)8P!E-W9%Sr>1J0KYxBO^opgO-5!Z~N^>F&t4?%;42|kTuSnA* zcJ^KoKERfdJv;D20j^)#>ZB(9UQ1y>IQ=3LX0<|4{>x7-%S-bnft~Ik(V7-BG4g|_ zdX1;_+ivu8Yx{n~#Kc)-S}F_X*WFp+>~rDd3>mf-lrF>&eF|CaOOd3US3}AFN)qPtJP9BXg-Tqgpjr`{BX=n=W}Hl!7r%4$p5aE{j0M4 zTdA_pD|ZR`9>qnr<%g1-cxwrV!b+JvkN2ruD*G8dS)5*eP6+3txR>TutS#+bf6_wO z)Y;n+-0mOwH$M}*HOIq|5wDp!r=36i`U&e(TX=@p-aqxnUF850U3n$7zw26i7#ch} z6vAk8R)&6?HiH3G=}@5VKoE%%76D<@=0t}QC zZn8sBRM7{2ih=sAE{PuxvNlp!PLP8!7o%v03#CTRMM`~2v}Wui03iED(vDE0VlnTH zAxdy=wo9*K`eps3>H~NuB_$cWNoX+x{zPJv?8tuovK%U_bh&+M#+M{%(P|SoEn1Y% zZ9N?5UtejnNm?!vLO>`InvLH-4fMn?Go8+zDBoIozxJb}9!^xkzrtG;B+I;4FDrJd zkePH2H9?zvhe87D#Is9I;iMv(o)F@MD(K7^BT>`I&jrPyDg%>?KtQM)={h3cu>HgO z^jqq1(f5);4cfMSJm!PL#vYW}RBZ4v!@_5(Vaowf+-XX=HRrPw zA|Vm1%U$fOq!$eG&~(SEcdZf_N%mCC5g z(RN>jsNUCF)zyL#SBbVoN;T|)EZFVTHajO*0MSG{`hgh+gt8@#Ip56pgi7-x?1}!WIJw0+#-kpUa z^jk3M?b&A&KTHh2)7-v_<4n)OSPrxIM^AJ>RA~5m2n|{eVo09ZGX>}#s=n0sfBw1c?zGRiue(iVt+D6WpO`+h zk{uzfZ8pdW99sG}Tb}9SPq$Y)p`BY;lyQSEa)Pgh3huVrx9q`8QOKj7bwW6$3PReD z-M6(m&ZP9Js}+n{dxw{fReA|W6A!OiRZ&vSDmEw@VbZ)R=Y$!QG}ZcR%pm`ps_N*g zSekEb+Tsry`3I0C4uZ`D>zY#nmT3Plb|epq5x6k!gsBSkR{b9oOzF>0Zx#3>f}mdkE4IsV5VbSh#l(XFvnm*|`?{Mc<% zA^~o|raA2lT42+vDo5xqYOZ3YMUGalRsY!hAp@0AvauVR_Bdo?%bUAXpTpid5tbt` zExEI6`}P^$oEmmCGc)rJ0tZ{*(5$O}{Bh#a<;%95Vl+L51<4Ia>4$W=|A5+C?Gd=% z;d|Ij@+vVbrVV-2syIA3Pk!RV^b2ntL(omVBeRqqJG4{Qrj;#%x4%R5K0d^xOlxC@ zZryJ76Sot7b6?;2m$H}y9I)xZf)S}x_#$04Ztk<0ei=dER3&Slx)J&>+L`n z;>!rNOn~310G}aUTq%o&6 zh7Z;_zw9Hkq*P`nD>xd=lB|}p@q3Nn{@9p{EJHf<7#nQ>r%FkvWRx$RGZ@>;p+un* zV?s-IrT+QxgGb;^{pG*!2gt<#QGMDbMC}xYla7{5fAy9Hwgt*IytJU3Z+7qrBmW?f znUwK#2KsUC!Q%R>9}|7(<}1^f`Uu4jX8rXkefFl6-DM{7dVcnr8IFCr8YH^at9|>} zFN4WzMak^~ZUbH#wP;~?G;K-nnS@-AB%RqqKdEjVTMdAg?h|407TP#TCvYlFiU;eQ z|6b@W0?!d9#}9Ir!nr%wo^;!jT>7dH&QbC?+FA{Q$VuA?#*kEw$BbPgw7Cy9`Sv-Q)C%S&|f95lPOkg4z( zyvCGi$eb-O3y0uJX60NI2q2nOUCw$R-_QH~35MF|q~=HpMj!_57XF%kUx)tvZCTxL z1Zwn4EETQ2>I4&ip!#dG=W7I+%@{qWtq(fV=Lv5eI50l?u2$0Co;adu8+%%$1wT10 z))Ypo2B(r2@jVWv)pFyF^S5l)%&q?F%dBPKW!@r1G>IfA6apeVM7!~(K^^_J(A+-w zZ^;CaC}LKvq8=C{s%?gbP~~b%SRqY9Xvvph>_NCFAXi98h$+tByExoYnjC1w!%!5x z+1EH_4cRAwqlekOJ47-9iszib;T*HT-InPE8qh*(3-$-4oOo;T;M|DRe0K9pk#~?2 zC}Mb^9l1vvj8>h0kL>6O)s$$PPM(yBi*!BcEsC?YolCn$B=b;TXC!{Iq=h#%>J9l(`pRoJZIW}Z%F%$y zJ|mVN8y}=l#PwR}C6bAheVoQI3Hr#IauObYj`Q`Zma1#-Ve}A1v><7i0#by1(z%x; z3ZO->VnH{nBuoC8NWBiB@oSEKo5_HHjHx_g#)8%6%;1M)OWA3t!KKY9fVSk5M13A) z@WF$v`Bup+2HsR8s-g~Kuuy4E6IslHMg|cCzPca{4ItMQ5hjs z8WF;lk>D2)=CU2G@9l5RF!XH-D6}>|N$T3ucRc7>v}M%aB=FV})3~{bKUR)ZDEer6 zIC4wkIpTYdI0Z}yzvEm~%J)EkP{{Eqx9HKtplA?64GD!x7b6sB$Y_DHP$%tZYs0Z+ zWdREhQGzH%HHYNl;pB^A&83gu3xoc5C^$8VMKY6!=*XNtaQLEDO3`Y-_)P2BxH8dP z;G#rUhT@!Pw+jB2)`>C=5SWf~4vDm!F&Szb8zHS)Ff>eTF(@sArF)pGZI7%6zs|7$ z9sFr4XOgc)xkrkfv*f0R!fo=iB)+J}O`@n!1~|wi9{Pj=W2?FZm!6sK5ej_}u%E85 z=LnR9u)`X10RT^AoRmF&3kr8OLWnDL00dWuh7C7O&OaO!6hfj==@P{aXWz4746eOl zV-f;4J39{*`7Ktd$X5VBVF=eI9&=oVdDT(!1x$bA-K%Lfuwkq8ffQ3x_AsDP43+;L z^{l9mxV;kBozRiXlq(NK+RYg!Agtn=iDENpHV0IOW2|ie>q`X%^1ql{SuwjhrM&s1 zNx8qjyzGqFLV_sL$AdOrK-k|N4xN=j(B69Ym$?(b5!=v?$u%unq~EdQ$oD?aa=h$r zPs^=*`L^r86?4Cv>SCDDJvFXTrx=T^nwvsmjQx zhH2kSJZ^91bN}=aZQZ&vN5zlJh^fA>Pslg(zPs>e#e;saHT2uA_FHV+a+6NOC%cNh zuDW~wI``ur<)_Z7DEXi*BWTyfxMo{1)te!!31d&s@bka_K1f0$N8`q1*dn2mZQ>2( z|2{kK(;;al_Cxb9vHJ*yMFvKrx!EAyBS3}n__ICu(Gq#B@h?xftd4l0p&j=2oj(i) zn%%cuS01hk^mfQ;;ebDnpck8vGLFSAIV6-uUCbMI505(_W;v(EESnCMJD51fBimI* zq?Z1f%9Ji9vJ*;|8+%dt0mVkB;67eMyb7BxlMCrGU-~-DjJe{Y88!;ViJt1!c-4uM z!bD7_r=+wJI#$MerP5-JmFpaa{DK+xmoa3S6aw3VTFZ@aDk392E4F_!%?c z{nIugsLrrI&%~ZIU|G>L?DT09ur;y7iz)EE(Jo-g@!W12iv4=sb`*QfYs`k?cK0~& zSv+iG`~ypoS5lE*lKKw{t4%ekQ2jAa-OpN+^ft>h9*ljL`b7(%{>7s@b*f5r$8uB0 zs=pdGyqdZIQ2gg_`~Sbyzul|rXHOy8?Hve<^S+>qPkY*+;ofwQU=42Jaa9iamXMn- ztgO%bTLBA5WFusDgf;eBNO?@MGZH?DaE*SCZeeUZkbIB4Gs0Q_o9vURH|*_o1S_tGW~SemvNnDh9juFw&@zb|2m6+HtL59hM>66OYx`3+n_EFztI^ytB4QiU0?Ruv=^FQ#bBblt?!BM?Y8LPf@J z4Ob3GBSIX=(h{JI8LEtg(4CqId#4JLQbuS=?w_djxb49j)E zkgVXs^!H+H3MJ~FkdVM(Yvo8Hi?O-P_;HNUxG87Ei5*&e^MbxF-FR$H{u~3nAU5z&oE6c#n*p> zJW`Opqn(_9K>sq@$YcKeThg5`@hWesg+<7F%C&T6c~f7>IH25Qxag-cc`>4o)`nr6 z7GkF2*msMHd6IW~5pOimY^XdqEtE76mzFHB zv@~;c06w?)R*_zNxp^|jE1cZ%pR`3E|^DtA&s1 z=;>N@=+Ptk)M0+y=>RgCtHylbjT)woHc_iS4fNl;w-e((wX~yyoi^II_FZ7E>KT#R zW!G!>IgL%M%ZDE=+GLX+5Po^>3KNCmB`XIIHH_j2l@pd*rRtc5@@N*aHKeP;NdPnzvFdFE2|E$#v;Ez(^l?R@sMiO z2L_@ky}|a3gte|ntbALE&wa;)cMsR8I=5?0lP*}~=k!A4*~-w8Ql`8{j96P>g2|K2 z9Kd{^xiuftyG0GElvSxJu2;JT>cAxrY0PHYX`2 z?gz5W;v2E}ZgGTAPq>*10P$_2lng>5mK~##VpT$7v+h&ZNVgS!(^N&N-jn0Vgd$^B z;SM{w)0EY#Ryomuz#2hC#DwQIh|XMc^-tJbp@oYW4ADKpf*_!cXNxmE{u&KI%GB;k zm8k8JK;G7x!U_!KzA(%0j&zc?;}y{$%LFiLh$PjHxGd_iNZ^;SktmmmdjiWA^j-E6 zSCV@()%17I>d$|!c4DkVRQME#=W_ntQ+4X{sr07sRBl2OOxI!a>9 zI6?$IP0d9)icp{Rm(J&6P=mg6cJ~x{e7ow_slR`vC^)KBtEQ`zMWUNb9#Dyf0DJx- zzu`JjMgVq^?1hDfju4p4`fut^9~sZT#xT}6!*;q#nV+KODCab9DWOlllAfhh zr5JHFyIw&qOBc=wQ*)=1NlRX?B#4&>*uZM}CP?@WVS+0<3fhdfpnZ&xieVaK zS-fy1{{+q`GCYj3OmH?Jy9oQcm?v`ljG!0@1s0uk%oBmh#sqf%%Xqv`Oz?~mQ43yx zPVZhkIE_&6QBl#vlG6sS>esK2wr4MgIhE62dPq{FR;ntTB38mrLIyHD(w_TnMNz4fpNtF=L~Qj#3I=p?{>W|MYY| zkjXB3#Uu8q03?z6rjFidAX9GAIF=b%`WnvbqpQEtd~MT8mi`y}b;G2ERd z!GLu@Zj&fR{k~=0y434BZ>tdK{cr8(L-cY!xLxH3MEcQ_njF*9?W$}1@x}R~+=O`D zWOgToUCh$JGa^!W_s%BwE~BeQc($TB2hFN`eM-Q%ZEL$Ih}Kr4hk;Cr6A@QAXTnV0ite3vCBZkR%1cj(k9 z;qfBqu88Or7hHN77#Z0iKRfDo<_c&AC-j5DHr#o1Q7TDV+Cskhp9Gy33bhLpdK(K@ zk@SNL{2LCiZjSN|`?&qU7lE|zp|zMpvI_W4L*YAY-n1FV)z%^G)vz~BTMZa+_NIdu z42t|(xk=?K6Pd?nyk*Ol zauH_2we!yqLybAq0po`^iB9TmNwm)&j>RuL8Y#N$(c zBfw3b<@54l6PXDQi;J^Czvd4`KQpswnRLAg-N=2%#&q*351XX6QWA=Zu7wyt`S-2weCL}}8^@%dnjJkLxeldHXR7Y(L2BXtQrZ+iA;)!T4KPZ!#0PbSz_!ki#%whXs{DQ4%g?AVRGpXD}XGT%6A| zb#x-6;|FdQ^$dy)2B~i2?p$&=YOq8(rY{BBaQEbavo}O(x2spkzccRuNd~;;Xz4>clGGVO@ZGUIpoiL=bje_@^2e3{LO(a zs%FzG_-pB(21QffIbAQ!jJ{mK;^@tHv#_J>|WB&w)Xmn05Cml(iG>70b#a{qC!n&@yYDjw}@CqmYMN;PIu7=^VzO z3KBabG?7HZ8ys}-IzF#rG^fCcq&BQ%m@zFOq?B#3i%iQ<2}L|r+oDA;X&j+69#-o=iLUEB7Yo|@YyQZg*Jeo%zp?y zW-97&Dlml7!6YlQJV;-&ys*o0X)=_u|4=~ggsaSP(XDjKIUR!xPz-2fa^+G^I;WuI zH=Fi=`Y9u4)+ypMENTjM9gLSJ(z|k|A#L~h@F!L*!|b-ujzz5In2kM`N&U zF%8&^xHc#?c0v6*bqKIy5z*-NsK;gBzhqzh+{xODy)04zZ=nO9yO~dza0EA7dt_fn z(PoWzZRgHf;!pF(zl!fZo%`0zWn@xgzoZ)l=Rd%o*aRk=+*RT0>+7`BzHr{-@{|XI ztcz(_aXN$32~axbEU(*6mxAd2TI9?#$pA6A_9ae4J5QFPdVC330U;R(&VMaoi-3Nb zMyAY-h zZ|=J)&BH$_O75gW@f9r@R}bOAz#l0tU7lkqK_nbIoa z+f6%q8BNZ5akTv*cfE@CpCw@bMuXmAu%QyX*GTylPl4qq;hV{M+J4|i8 zN6muv^t)v0n?G~@nEfkT|MkwZJ02QUv9kSNb$XOoxZOR&eR6`tKOP&KQLoFH{S3^A zrx3VS#l4hbbNKMf{UYfaBv1p%x@N+g$(3VuW;9gzx~k!+OBNl#nCVxW2@?#6N#|ZN ze;SH3P?&1dNB$wH&fP~K1MB|NTZjZ7-etB;ZBPyUW`$& zmMjjq?zu!oie6+#w8Z^3BqMb&!~yG zCYp7p3K&#F*bb!MkFEQ%gy$femhSCBT3RmQm75>}tNm0XFi z!$EQ^KuCO&GkDbEWSs?f&=3yU&K4z zbZ1#(;K$qnwbIVzcNjK#)zzU}4;)y%E_uplPR00C#sSWD0dzRGmKs&o1M*HG)k0kw zoUQtXfFea$ahla%m=@u=h4Ew}Q{YI*V)OBt*69ty7hl)-cI+|;pt{^L*z8l34%T_z z$-6GuJzd~my_s#>)2B;TEu9h;KH^Lmuvvz2$JsI;gg&X^ls_~#JS5b%z*cm3?b=)W zrGZGGK&8DWgQ(Is4JZM8jE#!2sFRc7m?KC);#B$t?8i1Q^iTD~3+j?;JoWX`ZPZXh zl~RNQ7o9ruFsHF)-8P?It;xQh*00>Mbt}+jkIrFxHOCDR|TrhpFe+uyL~O4 zeTLWt?r*(%-dfXw)0^&adYVH~K;J5|A7+b&d-=S1^N^ryVl|F96J#1UVS?y(BM|3T zn#72ZOe{G)Z09oGyY%XY^iO$!I32h00HzGvvAv)fWad)VjxXQP zl~6*6`11l=mTvw8D(TeXOK zcW1lvb^c{Y`L@a{d|2~r)`JAcWM|elX`~tQ5+i^=$&159?i*$0*K;l`1iy~U_>*8f z%3VY6r>308yyA2`Y3D2INe*xwca*KhUqy-yX_W^X?7&=6IZ24cNzt8>w$sD| zAu@(Vs$H=LI1B5ik|yp{sRaR^1Rard`1EN#YwPWAJH5`ia^(Z`eQr$vV}^`&EzIfx zTmc#mOo9lk1@)>vFx~df2X|??aUcrS&W#o^J7L07BXw%s_U#YdE%$BB8<- z1e!-(J7)X)f;G8uD*_{-QVte9$>k_nt~QHU^@>b$+HJha=9{9dTrxf58_cKu+L#hE ztQE)%IC3{6LTT4h($CKsez?ZwQc0@@uVNOhr;X)8oE5l*n#5$5H3~8}1Xq|M80$}` z=ES&4s-M-(ac>_N@s&hd5PIN%3#DT~4A1JIu~IYc98R7>`0CN&h_!X8|8YO__P_G# z#+Paw|CS*(z!J@Rem9 zFvDUG&{*JFG7Pp>dVBtJd21DcRCs&hA;;oa0Y6t?Yc75qCT4te_+egJEm%x#yT^~4cs$kaD4dSbVy5M#@4kA_b&!n;rGq@`KY8>=~EPmtJTohjr1Txb`wu@$CWE z+qP*#&YJ7?jKsw5TqBMLaKMNM`IfRLhC|J>iiq821KZo!?61E3?LOB1ANOTMd(_H^ zOZ5l$_B{ewa{iq%6}%^*3e|QF(;x+zLR0cjeOt%KUnH+BXgBy^M`=8WDdAnr3HCh;tkv0>>_1-upb25K82PU zu?PJY*7)`P0_mG4Oqj4BhPuIt&2d2izDo{;OGoBg-u6!GY58h@U+G3_tFajN*~HnS zJ#W|9^F{vtZqJ~`Dow7+qjKzbUd(bnjqKOA?@FXKYCaV)L#*BbrBo)M72`ZmdH`zM z>>D@wT(5vCN|%^V%}j(1U`x%fuzq%yl6J$d8&%J8g#akX2CIW|Vuh^!pWA~<1hgp@ z)6{zMy&y5eJSqycGXGi($aZewJOG@9mZJW`;^$}w<$Kd7t9$wa!6mev*`IjdR(D6Z9oci`!?Uv{);qFPeXL&) zx*@PNPvL5T2>}K5d=*D1m<~mkh!8kADd;`tkI#?TSrx>}@>)JV&3}Kbq1_szy51JC zMlZN?IMU2TJlQku&T~~l8<$;vI9M~D0IBCSG5v!Dd{n9w(HJ89C`p_668BbPH} zTs@_Kz&0qEnqwvvpp7@OQ3nrdWQ9s%?x0Oy<$05fKuKH1&dpAxFK8`6_?xU>t*g4h zOVGJ4WS-=&tQ}qr*szOY=7^Lu9!J_m`z5*+8;=6}Mz}NQ<8QRa;?1s2Ex#@51FwyB zj{_AJ6eNG$)5)nO*p|u6Qbm3t1u3HUF4a&uR{Y0?>c!b}4L&VneeN4$#(Nb^+(l!R znOodgu~@f%`fbnB9=jiJP8&VpJrjz`u((@ZN7M!Fv~Ka#W+gxQuS2QUAN|8YbNSY- zZFH#?RV-J6KRoC4r?X+z&*MZjH>h@F_@D67eaDKSQiCDq*!%NaGxUUfy52q7-2LI@ z;Xo5Z=bl#vqNl?%aRC^AF{s4NVG|FT_s z-Ohf^;8$}@@j9OO+w_w+uC09y5eAEA!sXZ!)&utSf?-;Y{)8tqf)o$f&3a&HVrEw~ zb{8!Af9!zFgx;1dQ|2=yC*%WOxDfuKrDpN-tp08FU>*cviVF(R8>9?-*J5P;BBD86 z|BV|rDozL2r}+P&VjeEscS@By_~R#(>g#lxPk~~{7w!#bqD>iz?-p1YVaXb@!k5Cy z5NNW0)*;LmG5gl>C$C-|OewFnF)=Zba%&kp&_`kKAR}71qX?2J0UdBMups8B_Fd9c zi-EMDL-0gSp^HlIZ$X`4TGhQE5f5E6D&G-mc*?JfJ3XM}7QYIT;yE{)Zo<*BDFj~O z6b1<%@z$`0F46YN4KkAT*N=CiY?i-45&@n%1ECJ%vy|Dwq?H%P%$}P4I7*6jDSvb- z3sDFegCW^@;1!R+(j#$x(9JCVXKz-YQDL|5kXQr^Z@Z#xm{J;yR2%(^=Fj@t#nMW1 zq-4C)*h&rJ$br3kFOy_W?hN)438zBaj&2@kWs((%g*X9Sgd7J^89sryT6%f9HmypX zW?KHyC(f_DWM9Xwh|oMo)v~-U1845be=HVUx%R#6?cFA~V`-9vw_ch%btlLFLjGpTsn&5f@B)BO z8DE|;8d`b zDM3`~BA_y*7NL<}V$AS;UMQ4ZWHx8J+?je@fHVL0xY!RFvPlcEc)Eam29PN_lrSM% znV=E?7w4@SyUg;N`m`@oA@wc_0%j2Q4v6F2bEHd=0^E7v7%687S5--wW`Rm+x& z%Fr>O_*q&)4axu#b3`=Y6y_qEhe&VqlDZij9AuQldbn6oN4`^Pd_ed3qo1@h_<3&l z;n$K9`6G`p8^Pa2n&WzvSsi!=U5WTZ8gwd;ZZumURKy4aM~Q|o80=g4 zxVglE5Ljk|km9havHL8VB+6%Em5Upg7mwB6V}}oKwB3(ng3?p}mXkB$=F}ncFM{^B zxO&2{rxNT#d4?-(BIyr4d+q|aT^nM!gBgqx2>GhYgH+Yjr}Gc z9e|UiqiI|C;4EmF0|z-v{Oku^Xa$~s`S$Tmm|j}}ER!eiHNL=O zfn-Iu=UUD>XCG%e%@Y{TSyyat0Wr?UxV#HSLrF=TWV{KN2g{#VYT4|8A42&e0P9=T zbwKmZlWX&B-&d)Sx)ZGe^{^*`RUv~24=^gUr1{BFR$ejQ`u~T@yO`buID9jMdgaIx z@40qLPVkiFvs1EjiX;w*kM~S}bB}$Ce(!gDyLZ;7e~=k7SD0g?XQ`Pi?fdBaqPq)y zg}<|7P%5On3S09 z$J>i!%C|gWL~aH+m@o8Cy7Ygx2Jl&5>2&{LtNz_(aG}LRo~g*2w|^UzdUNa^)3vc} z1<&5Dv;NJ!MM($qFAuF|a2xdpN^3b(9A9Z8#QDoG%sIqKvl;%mSYJl@?1^y;oG8gV z&cfMZ;XqH@H0K#@RBuGU+!JoZ9}NQVs?_9?U{Ol)twC&kBPiFE3W=*9-fSDpEOM)q z*0<$s)9zJsNpLX1M1Bc_-m-1d7xjw3Y^0~tQCTFk1?e669j6E)f)J~U8Yx4EVLAf^ zuNWP6>Aw~BAI3N1MkXI4H)!h&yod2_jhZHW#c{|WhG8Wck-bS~5|+x%%;!Z>iq&1Z z^j!Mc*9_;brnH=JxAt2NQkGDy-pUL0efA?WDggT!%PsDfI#r|61WL8ze`KiP0UJ@} zp})=TVdo~dm<@vlWv|4*%rf^6B)`HcnB1Cvuc`}Ovv~qqF_LM=K%a>nXbIo{1MkaEfU7US<3alF^ z^)EGc)TH7r{*TVvwh1vK_JBp!3wKNly?^!%+X2td&Io&0EB(UytPS6;XP~?>p-xb$u4828fg|kO-sPoXj|;E z(cWP`+22!n408e6In5o{aG^Xr;)sX!8uG8Imq)m&NG4GJ^mIqE+v%sgR&SL^*_z(nP}|??KZOSaCFN63mh3XqFi#(G=0$-?-;a z|DD*g;;>CATO#{Nq>MagDN9k*HF)OWO{>BcZeYAA225%(>Lyc}aEF9GA{>y|G1%>U znVcvPaSbf!6YOCb!lX)x5wGL0aA8B_W^Vo);!KpyHkLQY4qEEI5UYq_M&Gt-Hh0&q z&m~7irYL2E5O9V49f+c4O!GKq%~!%REHZ%!G`KL**8t2`!9gcLx=>GejHmK?jNYy> zEaD1uKVFQE`h5AaWh(&-N410WoD9u&jrMIfgJylHAFokSHcRVQvHz%WMY~N#ZwaFe zG(36tWH^8$CtRj(gVfl2;6QriGGzwKq03KxRBwD<_(gpW1iQY!d)KZ_@1*Zt(r2aT z!ekVK2^vwxd%S>$|M`Uj3{$)hBQmTi^7h9_Gp|9t1$LTWVNeXx`jGGrpzn z!LyruF(D%Tr~K=CqKxeqQ1t$}CvEoRqtN(p6xS*AlwlXv^%>xg%_K#ECvw%x2~00G z;z2E^{z*Ps0gi0^rGxP5tsJmcb@tOZE)dUDc3F!&i1i{s=VxP9rB{~M&$Q_P|0EN z$CJa=iq=EG!v^B}{h?1`To2zcu!D_?!8A8@i!~r1vHY{UyqY>!&SX2xhD$p)Cpr0! zyYpc0P1`rMM$*@PZJRoRrJT4P>>fv_+6b>THu4&aXtE#DH@3y0trlbgtdYcJ4q{vX z1znql#Gt}ghML)2hIS~JfXJMeUGfXn+lxi3oe%xSlq)mK*DsvYJE4!Y7d=h1YX5;Knlk^9f2=Z#Li$>LphVCFJo2jyQ9WYCXz(3C4ZyT-uJuhqL(w z8WmP9wBDUj*1c_QXCc}A^yau=r{TlvSZ>0#v6Yhhr=tv)&JKxt+L>~g>l8KgzXhYZ zHVNB2Ho06V&($xeP$)LcK`(mzBOw8Ogn04rg{CAOKmKE%2zcNv924fyvCZ!F0WX2d50x6`z`*ogJVKnuf{#oe+t;TzHj=jUQIi1 zc%xFbscFS-OH#HBulB>gg5NH(I~RTB{dZpb?59SD+D}`(rFO;nua6&dX-963cS5W2 ziET5VroDZ=z~kcYV*9F$Z{rJbLYk=0G zc?o+zE9~omYZk60NMU(+b0 zuXuqky!Ud-XCuo{&e-j@(xQ5xmlCpORADHn0qpOm@&hM*38cK-s5q%ZPc!bw@>KZH z_xVHXv^Jo3Ql^TWDIRK8_7W(2dMO5?c-`bs&~O0itdDseQX+?=6~jQ;;Qk|S>@Bt>yJSb!=&6$J})F8dP@XbMDxoP-<^N`m%%mQ_vu|G=W@i-P3J zr#Iq2t=MwRdLAU_PcljvVz=AJ8ZJjFpO?j<%sF6$ebXNAQX=uSaGsrUA2;$yTDu*o zWiOAdhSf(Hr$_bGJTM`A2O?67LtnW}ptL>FObN4{cl0xo2+)p*-t!;N=jt3`LyiMk z2ajeI07KvZ?c*|rM=HwBQKQLtDP9G&fq5+vj0*jsr!ij`sl<9s7ZW+A@ne*1k^x}y zA+@xx9y`@eXzfC1IQNaqp1XE>L0>rKjO2{hKRr&1KC(gEXDd-^h*=@eNuC%bVe-Q@ zLX&e^9vcyXmCli?WxYECu|gL!Km~=z7C4JuF11$IZre5^xqA9y`rNviZ`Js0M`EY$ zT1KwDf2qTrTQuSEy&=={lbv7WJcS~})KFVihw^c0y6d21?}X{&6r&ERT&Ef7j3z{* zE;fML3fGorGy-d*-~eKEr@)X8NNsFruM&!D;6S~scrK9DO8((#dFHUhSoOy+vsHFC zQbYu4436xLI*AB6>p9}bxr?cp_qG1P#=Lgz!0e}P9ez!I{7dlt#g5mTICOV(a5$0P zqr~feY0B8ns`ja3%OvC%k%NatdMjXMB%z}(*wRoG`<|$Vg^3CofnC$~OcYGL1Jw+- zPwKRU;O}e}^+mvUQRZCI2NoLZ;e*zkv^ac>CKD zv7-vFw{4`5HVSKr>$K!4pqD?4+~0%}$PG&GYT+jbH-VRJIpU@$4PKp$6}6qYnW=yG^+ z%hj{9^E-#RJig$gXe9RbTsAs+>H78SXVszfdQHldfh{X;Vf@JEo?HWEM*t=DR-Z&d;V;TWlRH* z{qVEL?hb_bto*$8h)d&VCiuV90vy;&{evK0p65{HD2eYQ6X3EE1r~c;?9B%se99ng zr3uL@7y7kSydl~$FMlJBTjjd*Y(ci^y(hg>%q$k~sQgpx?;{U?A6)*IZTHo0YO=pI z|6rJSy3I(>nR}+T96W61o_7s?iyOVbH!8fGdhRx zdOvXZ3E3yg3a^)O*Laik&J?hBO~>qP)>vkc2!(jC^>T9$3=gEe)YHmCB{c=OX#{J= z;R&7hem7)FU(3=eH!5Uo3Xr5jKvvT8}vt@0WGiS~_g(ndd z$zdIOD#?(3oCk6b8Yh{S6fg>eZ8Y&PpW)l9ym_4PBV|Bp@)dzXN`e7K4vgHdsqQ4U zaz2Q6f*zaO`6g%7?H|a?JWP45Ty~WMyDu2!v0okvpMX7S09D)W`e)6NLyX)%;qLs+`Uw_pJgy;jN(E zaMDVM8+vr1!n?)(Bn?D2KOmq`FqalP{@beERy6ujk(r!;axDRim=tx1jyGRPP3ZQY zUq~XyZ4KYRfmBCzhN@w7&wE&7CaF=RE(__gu3fgNo?(D>JI>qzW+Z90h1XgfI!*Xl zJ_yY)-i?$z#ya)t#rLegEUcW>=S`Z7zIZz#dx%+!4BupL`_d-@IF*9e4gHud$}8>R z+zv0xvY3!Xj%*h1U8fk`RCEO`T`1eA$$5+S>>qxxB$RS?*V>=az$CZ>;YbioSF@4l z?Zq25G?7%W0AYdhi0#a1&^B}YXiEoOzz}`VXp$0*mlo60snYYC(u4Ed=?TRAEAmtW z(2a$pQKq5?QXsGC3^l&RQAXLTq$7o80rNmKh|)b&yCG4On!~NFL5cMKNhQZTm0AQ7 zOHZXR2*RZIO27BW)7Mk(RYCl0^dguP(IlYv z=#I+Q6=|&T+*c-94=8*yGAO76eV$S@=))`DA}UFzIMN@l%;$6J<9R`)5A#vdS;e-e5BjJQvl1a?|cON7AfV3*ZX z6_ru$a-ss{EzzsHJr+>F%SS9Hp*aXmq>el)T`BmoR`cJuu&bUSk>5CAz*l8>R#9Ku z`1j=@ML)KMYS+nYHfPUAoIMp1zb@TY#@(FfSp2szylm%5cWv@@95)XG2UCrRZcWQM z?Mdapgw5EX13Zua;>+A0$o6EjRTly;YO+E9;;}mzoStFyF;^aO=cMGtDSn!;iJ#=u zWM3!yzfFi5Chj-{I z*H-6*m`4}}gzael6F-ypa5e)0My$R?}e+mq{5(x1Jp#yK{xaed}@?K$hef9YfTr-E#%8}F`{`P!iV zccK>@{_yKHA5g~iF8y+9*%j5Y`kG~l%#J=t37#uL6D3OoxYvupYCQY$?t;}!h`--s zZJXdL#H5ilZ2qjIA-0fal$pfV@Pc<@o`%7+Jm4t45xOJh3}L8}i2;Uk5_d(MdtgW< zt}bKMU9CF4$A_~FO-t`b!V5<`ak4dd+NrK!SB7JTvJ5P)Lpti%X>_SD@O?uT;M9>R zZdt_^eDqmcga|~TFlvk&I!AM^Fi494lH_J^ID?f_wuhcPd9r`xm8>ksPc{%McJ^$& zyxsG$`!AZ-%WyAe+OOx-YW1GE+QITM8nQRg#GR2JiD1HCyrOI_?JppTBSQtp3rQ?u z0SH+@r zO*4&?n5aKsmJ`$3)0naWp9Wu{&TT&& z1c7M!udO;+I+-t$|NY*3wK&5$YRm~9_3js>*FF{|5?s14u5=U?+Af-IdFRC3l5 zY(Og$TJ-Sb#N&v#ex7;3P&P|3!43K0ErVYdWPA8;m&0n93U z#*1@(=AQmNSIr)EF#GB38nc`njC#`_H>&%yIC}j2ZE;ZVx-b$3dfb7Dgc>lfBWWhN zi+s6F)A!Ln-DNSgazc^2?t_g-wJX-_%T5d3}HKXF%dO&S9lzWNavTY6NZ+M!*6 zO|U}(ycHzeFs6ra0fSdVJ}D1PGN=93^W|~zX^I1to`|1qPM#~Tdkh(}Y|5RBHO_DZ zPE8y8Tl9rA$3A`R66+c!d$${BdGtfR>R!i<7mF$&IYK3j0|!>+r!$_WWrpxPDNv9O zjHSEni zm5$7s32hJPbbf9;Cl|n*$>}>}WTps{*oC_w<(Wyz_&_|B^Tcuk-C0d9VYuvNt1-#Z z5tN{<2JUd#tFhQ;$W{vi>e~vS#tWRmwKnGLeFZ6UL58zayqM#oC-%g@v}t^$7rERT zOfE%nw_xO3@d_(KH|wC1F3` z>+qlPh2T7nx&a@*dek)*&=4KL4k>OFFP@MjU3+e@CmW>F?|=_9eYS7?fWXLbY`hwD z3&I3OPfg3) zG@rN+RX}8d)KjQndPX0mgr#5PNKK|1H#v~yp^oQ2o`ECw)7DW6GggU%Vlia5{oXZP z)KbOg@OIcHOGGD{Qh9D^&!w+&KW<00Bk9%I`>=uEMD(kYrsg*e48wJnN{HC)@_sl&d(dP0on8D+N-u(10fUDth0S<+nNqIOD zI1`vhW)C1K;BRE}Kvv{t@f$ep*sN(MmKqu2h|~9L|NA?txK?HFOw5}TXyVbD{mP>? zoQ1yfY2V^Y2RphR7`L-o;-?!FAH85eh0vPLFsyz9=NCE~=`zuXK_zO`qf%ou*VxxR*TlC@0Hb0d& z_}%-<3$+L*Ls-ro4h~uOyr#GyYrO=O&G6KXWj4NUh&^Z@yVcOX#rX?}hH-2$I4<|Q z8@%XbBRMMc`bQMospxO2ZFX%P9tKYC%H2StHl|uSh2#U=)H~C zB0?J7o4C7fjhD|2u)&nwxi($wR&U$(b0YYMh$=Aq^D`Igpghe~Nrlgt_n;Hy;|9!S zEC1>w=dn7PVm@HPdp@}C2UID?hSv>6Y)|lcy!o{`#cA(bUD`jRprht06tj3g8T3#Qi)9e?yD=_-_MpqX9Ub z#;}3-y=MDnbh4X90Cn%H;Es^Be=P5Zv6kieqS4^*|6J?+=Wn9C^9#)Cd$$GW>~ah{ zp@$n)zUR!)z5lczM zMya6e4Y3PcPoZP=%f)v~S_rSte)vsPnr0s}0B2vBjqDt4U+f%MV};KGL!JfBSg*5z zAR>Ee9oiwqQUfpD-F%%?0}b`25@H<4LYb|g-+K=Ru^m&oc+2S0wJT;tY45+;eL>+v z)cfMr%#7`APbgn&{~nz17yzoI@lZ+_%hF}MJ8s-%bX7MCSqxs5Qv0&Xzs;;;r6)3w z(=vXD2-7(8Vs>39wELT{Pn|TWc*Kq_$5?C z6f51i?6}UBv+v;Q)s;2+iqYMtl!-{GJL%3vT9ApJ$j|v*(rMxE;rZRs*!`HD*)lMO zYK^dy2B|}zJbBXkPz|q|ZQHJCFufnA70~%mif#l^y{P|W z|Kf6>PK+pKh2h~%0KwTYYKBNSj__bvb zEzm%R*|X7KCWTA=rg$x0yCg-v>UfR=p$Wp{C=gKjGc)x1=De`}QQftn@ts2CmBjw} zK1!JW8!W44*3rS*VW}+KGwD$kV{CGO!j)f)hFJGZ!Sg&V$@D`&yR8%7M#>M;SUFA8 zikUl5!$e^fRo{pG%=Or<99@ID^BC51!DY%LM~3~H9LA0f=c!iuCaRU=BO+S+hxgm= zzsYI{xaq1aZzvZ^1)unqKn=d8N73B}4XVC(o8fm#H}suWn=Eg3ATNY9HjK(df|&l1 zBkSbHWOyw%9=xAugES~vqmh7Ib?_*cA1#@c2@MF7Lo}Fp^yA5DEzf?~V#(UI-5b>V zqp6v{^em#(&mTI*^5(rp^E^PkuY?PyKazhTAn6$a$?z)zYN_NEG0LGx_uo5s`WuGWDtaM$vh3DAfa~PE+U@n9 zPQK!8V|{x3_fL*J(7kWc?B!vXIs_q}lN((2G^%{1d}>nl=wa@VM?)?>`vygT=4MZJ zBD2{mXUq}%D*hF6a(KSBnpb=~c8zD!?1gjZj=NP4&k)u+O_?HD0u89$z9)7OsDdk9 zT!BjgY2O}>Z-=LNMP)wFq=cGFj-M`>G#9NJS2?A&o6DAM*AEjA*NMY=sj<`Dt8flb z9Yvc+G0i+RiyCNU#~%SfFYWC0!_cFdYeRk;IoSErhC^!TPm4}oKvOLI5A9=X&lWw& zkNS*U$NVu(ER~f_Q>8-}NZ|q%-!Nh!uhFhX+raa7zjV2vKk*5R19yGWs$q+%-TAE1 zPsfy=KQ7wm`MV~WCNyUPhx`g~!q<&o1Vh&E^LdLl;nX<1h#T!NQNoh5nP$+4A)$<85JJIZ@!xp`4kd06x2g3_{?KSlJ^&5sJ7U&aYdm? zxqd#1L0sxfmd}ZDF&IiGC>H=8L)tx#emrjL(Ihy_sZKZyg(BlJ#-H_3ZskJXXPQj8 zeZCjVDWJAnh4Y+2tt&S^mpq6~AeAw7#c0|1TkHpG#e}tX(YLwnWW&s^_Tw{iFh3^TfPnu9v z*-lfYJQ-DG;Vz(2bNnA*>kA^1dq&-+UOwma#@QgsxW5A&?xf{GiE4^H)Qvh~hS&E3 z(b1#NXpDQ2?UGVcQ-_D|-wlB`uJ0f9&d{LMn}dRu?p>#QHfAaum#6_^_v)Hn;&hCD z{zPN`1V{#jcCllRO^zPxzWfNv2AiwJg@?vQp)Zw%+TE2k_W-`H9-&ci8}>N;xFgs8 z{W~<9X7|^U`v#nJ2nP$@qtV39cZQBY5^-c9-RU~+L{^HlT{bOgrjcIEeh%*Gt=(B~+EaD zqLRt`GrXdzk@sEogAXoWol`dv$;d5s;`$K{>(@US1Hz92PU0vM`Y_%c;>d%tN+}N(^lW@%c)hs(ZD0JzirEw=VN$^Ew93?qyjKYr@=dB z?9~g=bLZju3XeN-#5MQKD^?7vhH!%4n8tvmp?A`v8fKQCdSUXFr-P$LXY5OkVGye2;~|Z2N19?x z!(}Jegx#6=UHG@-8*ScNVZLeg!n^bD?>*dSj+L<_FIO91rKN6AQYM?;{+1ypmS7k{ zbYo`*X4(RnuQh~=M~{b^2J{EDv9qIUla|sLj0;2hYYtrc+uM(N=k}}A``bV2N7s(o zHNc^j%EhEMctuJj0n{h%OlY-q+45b{{RdW^@o12>YmAFob0?v77vGt8BC*TZv3EA~OJ_L+X@qxq$7S9YH}_vaNy9#=?exz1+(%;>jf zeG69NJllQ{)Iu0kfx_gaSfD?23~4=ScT8qt(f&HVWmhf?3D_b|TQCB#P!ZgD^^z>M z2Hx}c|C+625WNxNqDDZ-`WN0Vn*KQHK+vX5cc(vgrFB*SB1s}nbq7v-9uxfG#?KJ}7B}`*M04WJfv)V@g^k`AlC8;O6yIj$$a~Tl zd97T&<4{pQ1-c^#P~{ra@Okc&1HVmQOyw(<+++r`03To(m11@OQoOpqJbI;K&&JWm z(pw#Ck~lGH;Y^NPJG3vFoMAeHAPVIcv}o4Vt#yyFtw(?um>fJjHY?>2z}V)=b2!hf z;K9lvj(J`*6)k%X<);=lgyA``B0<<@OVlXUxgc*O`N8_@h>amR>_anug$Ag4aqsvo z9PJv^rMXTtr9OSzk4`0`nF#pmH*;29!W|r&caJE{EmmCuZcr#}%U5{=R$lC52w=+B+USK!DOkCcGhf;d5 zu|7zkrkW2s^@Zm7X)-sMKUkoWye4=%$6MjM#tSM`9PmT~w1xCisFXD?q}NnxBS(+4 zMIbTrn%GC~x>ttZ!rjkRDTu5>{tbjTsoU@E8g#hic4!z-!ZEkz&6#rwHZ}dpQwOJk znpKK-I;tp!h|3X-FjX1^d%tvS0B_FcGt&qbJ#no;>O-5K&&Rf^;MnWEW5&d|Nyi?i z531imJu+Vvs&=un{N*2UVIAmvxxGTEy8Vk2a|x?WTD-UzItd{4XIGnlRHh6{*qifz z^U}0HAlRFjz8gjyqSI#W?$i6l(WcqXWTBKT!|%6^%dg_s1c`uK1r`K}(Fe108lA|N zwhd&fDnTs{Pw|-PKozWoHDyZnGUuk4<_5$}yU=m*Ylh4HTrx|voOdCag0C5Ble64E zB!#sJBZ?~$K0-*LzpJNyH;NB=^l83jzLTUyS-13RuvbpB`&sGwZ`JGw_bkcM=mt|D zTC9O4OO~oAxk%=Wr`t6usO-ld4`b25UL4H2znpOmRDP-qAKW+%qlY>2J8nE6fwpLq z;%256tCW;(8mP_=Kore3JSqJS&n~@V{_{XOBWst;y)myR!a*Bmea;m_huzg@ z|3Q~HTyM%W5Q~F z;d}1UV_P^bXPop@_#!Yxq3Xm|!g<_fTQf5y&mEXNe)6QHMQ3xev#m_>AA&1rXp|(M zsNT)8=Ewjk$q@M~=+bIPC`Xm2-d{39xvMC+6r-T#n=~e|EU;qEl<{FMd+)_BdEfBG zn^{j1^Q(+036$1oDG$75GA*K`6W-HUtQJ_pGDr|Ye>$q2B}S=Qxdm!2z(ObBxzUI@ z5kwQbyLp1>Xp*X2K;`Cb>h+WbtNz-`#53xQyoa#^i$8a7Ot4ym20kNqaH%wc7TB*D z(^=HkYuJKRKcrE~fYqx#`;ep}fVcK$xG{86TL2I=Ji{_g0WOVRL_|8=VX6pwWMnsB zgX%kzRXFC&usf{tp%l)sW7N8!wI0q|L(aiTv+2_FB-a`ovhmZV=apZZn?&;5QUo1c zyLJsO|cJNXE)?<(S6a4`pO3^;SL| zgeQe&cFt_}bJ;rs6P$AHE5?GnA|+Ow)l?Fh22Vy)Xuqbn_pQxQ6d1F{BUwAn^cW6O z^RJJF9QBH-hw4lkUawrF!BHGhXi{6L(6E%Y44C=CjPS%UC-FuqN8Lg9eK}x0@)ofE zx1Dqsrb{u#?K>HMhh9bF(wS-BOfnQSc%ZAKWTm$#vYb!|Elm!neWaS@@xR%__sJ6W z@h#*@Aj;d~G79pnu1jW@tS9fMx6@(?;ypbSSdOa!S%Dl2fQ&bs4?S5%@erTU9Z;+p zQ^>SBv||Guar`wS84pC~I9>)MTzZb;+Z+Nd)`p;oP>2hLv7|R`o;8r#xhpL~s^&e> z#ogdOASZN!K+I;kwQ?>HpNJ9F)DT$hn?s*|zz#0C7z>TtE7qK?MAH=GN*zHL-ShTp z+AKcB^U|pkFBhN@JIEuTFTZ(+j4glr-;y7{W3h$4Z}w^sG&8SsW4(^4WA_eWU%7qv F{{UoWVmSZ+ literal 0 HcmV?d00001 From d0bca6091e747ba0677e8303180bfdc6f8d55d5a Mon Sep 17 00:00:00 2001 From: QueensGambit Date: Mon, 3 Dec 2018 18:53:51 +0100 Subject: [PATCH 3/3] added comments for can_claim_threefold_repetition --- .../src/domain/agent/player/MCTSAgent.py | 14 ++++++++++++++ .../src/preprocessing/analyze_train_data.ipynb | 16 +++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py index a0e698b1..63bc52da 100644 --- a/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py +++ b/DeepCrazyhouse/src/domain/agent/player/MCTSAgent.py @@ -665,11 +665,24 @@ def _run_single_playout(self, state: GameState, parent_node: Node, pipe_id=0, de return -value, depth, chosen_nodes def can_claim_threefold_repetition(self, transposition_key, chosen_nodes): + """ + Checks if a three fold repetition event can be claimed in the current search path. + This method makes use of the class transposition table and checks for board occurrences in the local search path + of the current thread as well. + + :param transposition_key: Transposition key which defines the board state by all it's pieces and pocket state. + The move counter is disregarded. + :param chosen_nodes: List of integer indices which correspond to the child node indices chosen from the + root node downwards. + :return: True, if threefold repetition can be claimed, else False + """ + # set the number of occurrences by default to 0 search_occurrence_counter = 0 node = self.root_node.child_nodes[chosen_nodes[0]] + # iterate over all accessed nodes during the current search of the thread and check for same transposition key for node_idx in chosen_nodes[1:-1]: if node.transposition_key == transposition_key: search_occurrence_counter += 1 @@ -677,6 +690,7 @@ def can_claim_threefold_repetition(self, transposition_key, chosen_nodes): if node is None: break + # use all occurrences in the class transposition table as well as the locally found equalities return self.transposition_table[transposition_key] + search_occurrence_counter >= 2 def _select_node(self, parent_node: Node): diff --git a/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb b/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb index a5550261..76ad8e2d 100644 --- a/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb +++ b/DeepCrazyhouse/src/preprocessing/analyze_train_data.ipynb @@ -294,7 +294,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame.from_csv('crazyara_lichess_dataset_stats.csv')" + "df = pd.DataFrame.from_csv('data/crazyara_lichess_dataset_stats.csv')" ] }, { @@ -303,7 +303,7 @@ "metadata": {}, "outputs": [], "source": [ - "df['White'].value_counts()[:10][::-1].plot('barh')" + "df_full = pd.concat([df['White'], df['Black']])" ] }, { @@ -312,16 +312,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_white = df['White'].value_counts().reset_index().rename(columns={'index': 'Name', 0: 'White'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_black = df['Black'].value_counts().reset_index().rename(columns={'index': 'Name', 0: 'Black'})" + "((df_full.value_counts()[:10] / len(df)) * 100).round(2)" ] }, { @@ -378,7 +369,6 @@ "plt.suptitle(\"CrazyAra's Traing Data\\n569,537 Games total (%.2f\" % cum_perc + \"% \" + \"by %d players)\" % top_x, y=1.05, size=20)\n", "\n", "#ax = (df_full.value_counts()[:20][::-1] / len(df) * 100).plot('barh', title=\"CrazyAra's Traing Data\")\n", - "df_full = pd.concat([df['White'], df['Black']])\n", "ax = (df_full.value_counts()[:top_x][::-1]).plot('barh', title=\"\\nTop %d Active Crazyhouse-Players with Matches >= 2,000 elo for both Players\\nfrom January 2016 to June 2018 (database.lichess.org/)\" % top_x, ax=ax1)\n", "ax.set_xlabel(\"Number of Games\")\n", "#ax.set_ylabel(\"Crazyhouse Players on lichess.org\")\n",