Review fixes

Signed-off-by: ArtObr <[email protected]>

scripts/client_connections/README.md

@@ -0,0 +1,38 @@
+# Test for checking that all simultaneous clients have a chance to connect to the pool with enabled client stack restart
+
+**Steps to reproduce:**
+ - Using scrips from environment/docker/pool start up and configure the pool so that client stack restart is disabled:
+    - ``$ cd environment/docker/pool``
+    - ``$ ./pool_start.sh``
+    - ``$ for i in {1..4} ; do docker exec -u root -it node${i} setup_iptables 970$(( i * 2 )) 100 ; done``
+    - ``$ for i in {1..4} ; do docker exec -u root -it node${i} bash -c "echo -e '\nTRACK_CONNECTED_CLIENTS_NUM_ENABLED = False\nCLIENT_STACK_RESTART_ENABLED = False\nMAX_CONNECTED_CLIENTS_NUM = 100\nMIN_STACK_RESTART_TIMEOUT = 15\nMAX_STACK_RESTART_TIME_DEVIATION = 2' >> /etc/indy/indy_config.py && systemctl restart indy-node" ; done``
+
+ - Prepare another docker container, which contains script for creating N simultaneous client connection, in our test we use build 618 of indysdk version 1.5.0 that keeps connection infinitely:
+    - ``docker run -itd --privileged --name indy-cli --net=pool-network node1 bash``
+    - ``docker cp ../../../scripts/client_connections/just_connect_N_times.py indy-cli:/home/indy``
+    - ``docker cp node1:/var/lib/indy/sandbox/pool_transactions_genesis /tmp``
+    - ``docker cp /tmp/pool_transactions_genesis indy-cli:/home/indy``
+    - ``docker exec -it -u root indy-cli apt update``
+    - ``docker exec -it -u root indy-cli apt install -y --allow-downgrades libindy=1.5.0~618``
+    - ``docker exec -it -u root indy-cli pip install --upgrade python3-indy==1.5.0.dev618``
+    - the `just_connect_N_times.py` script will try to create 150 simultaneous connections to pool.
+
+ - Then run script, test should fail as client stack restart is disabled, press `ctrl-C` et the end to terminate parallel processes:
+    - ``docker exec -u root -it indy-cli python3 /home/indy/just_connect_N_times.py -g /home/indy/pool_transactions_genesis -c 150``
+
+ - Enable client stack restart on all nodes of the pool:
+    - ``for i in {1..4} ; do docker exec -u root -it node${i} bash -c "echo -e '\nTRACK_CONNECTED_CLIENTS_NUM_ENABLED = True\nCLIENT_STACK_RESTART_ENABLED = True\n' >> /etc/indy/indy_config.py && systemctl restart indy-node" ; done``
+
+ - Then run script again, test should pass as client stack restart is enabled and all clients have a chance to connect:
+    - ``docker exec -u root -it indy-cli python3 /home/indy/just_connect_N_times.py -g /home/indy/pool_transactions_genesis -c 150``
+
+**Some notes**
+
+ As default, indy-sdk has a connection timeout about 50 seconds. In that case, we expect, that limited count of client will be connected to the pool and 
+ other not. When 50 second is left, process with client connection will return error 307 (PoolLedgerTimeout).
+ Each client is run in a different process.
+
+ NOTE: for now, new indy-sdk client is marked as connected to a pool if it is connected to n-f pool nodes. In that case, max possible connected clients can be evaluated as:
+
+ `max_connected_clients = limit * n / (n-f)`, and in this test with n=4 and limit=100, maximum number of successfully connected clients without stack restart can be between 100 and 133.
+ We consider the test as passed if the number of `finally` connected clients is greater than described `max_connected_clients`.

scripts/client_connections/just_connect_N_times.py

@@ -0,0 +1,121 @@
+import argparse
+import json
+import multiprocessing
+import time
+import asyncio
+
+from multiprocessing import Process
+
+import os
+
+import functools
+
+import sys
+from indy import pool
+
+count_of_connected = 0
+count_of_not_connected = 0
+client_connections_node_limit = 100
+pool_size = 4
+max_failure_tolerance = 1
+
+
+def run_client(genesis_path, pipe_conn, client_number):
+
+    async def run_test(genesis_path, loop, pipe_conn):
+        try:
+            pool_cfg = json.dumps({"genesis_txn": genesis_path})
+
+            # TODO: remove after latest changes committed
+            pool_name = "pool_{}_{}".format(int(time.time()), os.getpid())
+            await pool.set_protocol_version(2)
+
+            await pool.create_pool_ledger_config(pool_name, pool_cfg)
+            await pool.open_pool_ledger(pool_name, None)
+            pipe_conn.send((0, client_number))
+            time.sleep(100000)
+        except Exception:
+            pipe_conn.send((1, client_number))
+            loop.call_soon(loop.stop)
+            return
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    try:
+        loop.run_until_complete(asyncio.gather(
+            run_test(genesis_path, loop, pipe_conn)
+        ))
+    except Exception as e:
+        pipe_conn.send(e)
+
+
+def get_max_connected_clients_without_stack_restart(limit, N, F):
+    return int(limit * (N / (N - F)))
+
+
+def read_cb(pipe_conn):
+    global count_of_connected
+    global count_of_not_connected
+    global max_connected_clients_without_stack_restart
+    global arg_clients_num
+    res = pipe_conn.recv()
+    if isinstance(res, tuple):
+        code, cl_number = res
+        if code == 0:
+            print("Client with number {} is connected".format(cl_number))
+            count_of_connected += 1
+        elif code == 1:
+            print("Client with number {} is not connected".format(cl_number))
+            count_of_not_connected += 1
+        print("===============================================")
+        print("Count of connected clients:     {}".format(count_of_connected))
+        print("Count of not connected clients: {}".format(count_of_not_connected))
+        print("===============================================")
+        if count_of_connected + count_of_not_connected == arg_clients_num:
+            result_str = "\n===== TEST {}: connected clients {}, not connected clients {}, max(limit, N, F) {} =====".\
+                format("PASSED" if count_of_connected > max_connected_clients_without_stack_restart else "FAILED",
+                       count_of_connected,
+                       count_of_not_connected,
+                       max_connected_clients_without_stack_restart)
+            print(result_str)
+
+    else:
+        print(res)
+
+
+async def start_all_procs(args, wr):
+    global client_connections_node_limit
+    processes = []
+    for client_number in range(args.clients):
+        if client_number == client_connections_node_limit:
+            # Give a chance all clients that fit the limit to connect
+            time.sleep(10)
+        process = Process(target=run_client, args=(args.genesis_path, wr, client_number))
+        processes.append(process)
+        process.start()
+
+
+parser = argparse.ArgumentParser(description="Create N simultaneous connection to pool ")
+parser.add_argument('-c', '--clients', default=100, type=int, required=False, dest='clients',
+                    help='Number of client you want to create. ')
+parser.add_argument('-g', '--genesis', required=True, dest='genesis_path', type=str,
+                    help='Path to genesis txns file. '
+                         'Default value is ~/.indy-cli/networks/sandbox/pool_transactions_genesis')
+
+args = parser.parse_args()
+arg_clients_num = args.clients
+
+count_failed_clients = 0
+max_connected_clients_without_stack_restart = \
+    get_max_connected_clients_without_stack_restart(
+        client_connections_node_limit, pool_size, max_failure_tolerance)
+
+rd, wr = multiprocessing.Pipe()
+main_loop = asyncio.get_event_loop()
+main_loop.add_reader(rd, functools.partial(read_cb, rd))
+print("Connecting clients...")
+asyncio.run_coroutine_threadsafe(start_all_procs(args, wr), loop=main_loop)
+try:
+    main_loop.run_forever()
+except KeyboardInterrupt:
+    sys.exit(0)