Merge remote-tracking branch 'base/master' into indy_1919

indy_node/server/upgrader.py

@@ -375,7 +375,6 @@ async def _sendUpgradeRequest(self, when, version, upgrade_id, failTimeout, pkg_
                                 upgrade_id=upgrade_id,
                                 reason="problems in communication with node control service")
             self._unscheduleAction()
-            self._actionFailedCallback()
         else:
             logger.info("Waiting {} minutes for upgrade to be performed".format(failTimeout))
             timesUp = partial(self._declareTimeoutExceeded, when, version, upgrade_id)

indy_node/test/catchup/test_batch_rejected_and_later_ordered_on_catchup.py

@@ -53,7 +53,7 @@ def test_batch_rejected_on_catchup_start_can_be_ordered_before_ledgers_sync(
         disable_transport_batching):
     """
     Verifies that a batch rejected due to catch-up start can be successfully
-    re-applied and ordered later before ledgers synchronization without any
+    ordered later before ledgers synchronization without any
     warnings.
 
     In the test we perform stashing / unstashing messages and patching /
@@ -75,8 +75,7 @@ def test_batch_rejected_on_catchup_start_can_be_ordered_before_ledgers_sync(
     since it is not in participating mode.
     11. The slow node receives LEDGER_STATUS messages from the other nodes.
     12. Prior to ledgers synchronization the slow node processes the stashed
-    ORDER messages. When it is processing the ORDER message from the master
-    replica, it re-applies the batch and orders it.
+    Commits messages and orders batch.
     13. The slow node synchronizes its ledgers and completes the catch-up.
     """
 
@@ -86,8 +85,6 @@ def test_batch_rejected_on_catchup_start_can_be_ordered_before_ledgers_sync(
     slow_node.nodeIbStasher.delay(cDelay(300))
     slow_node.start_catchup = MethodType(patched_start_catchup, slow_node)
 
-    send_random_requests(looper, sdk_pool_handle, sdk_wallet_trust_anchor, 1)
-
     no_more_catchups_needed_call_times_before = \
         slow_node.spylog.count(Node.no_more_catchups_needed.__name__)
     on_batch_rejected_call_times_before = \
@@ -97,6 +94,8 @@ def test_batch_rejected_on_catchup_start_can_be_ordered_before_ledgers_sync(
     process_ordered_call_times_before = \
         slow_node.spylog.count(Node.processOrdered.__name__)
 
+    send_random_requests(looper, sdk_pool_handle, sdk_wallet_trust_anchor, 1)
+
     slow_node.start_catchup()
 
     def check_catchup_done():
@@ -119,7 +118,7 @@ def check_catchup_done():
            - on_batch_created_call_times_before == 1
 
     assert process_ordered_call_times_after \
-           - process_ordered_call_times_before == 2  # one per replica
+           - process_ordered_call_times_before == 1  # one per replica
     last_2_process_ordered_results = \
         [call.result for call
          in slow_node.spylog.getAll(Node.processOrdered.__name__)[-2:]]

indy_node/test/helper.py

@@ -21,7 +21,6 @@
 
 logger = getlogger()
 
-
 @spyable(methods=[Upgrader.processLedger])
 class TestUpgrader(Upgrader):
     pass
@@ -50,7 +49,9 @@ def __init__(self, *args, **kwargs):
 
     def init_upgrader(self):
         return TestUpgrader(self.id, self.name, self.dataLocation, self.config,
-                            self.configLedger)
+                            self.configLedger,
+                            actionFailedCallback=self.postConfigLedgerCaughtUp,
+                            action_start_callback=self.notify_upgrade_start)
 
     def init_domain_req_handler(self):
         return Node.init_domain_req_handler(self)

indy_node/test/pool_config/test_pool_config.py

@@ -13,7 +13,7 @@
 from plenum.common.constants import VERSION
 from plenum.test.pool_transactions.helper import sdk_add_new_nym
 
-from indy_node.test.upgrade.conftest import validUpgrade, nodeIds
+from indy_node.test.upgrade.conftest import validUpgrade, nodeIds, pckg
 
 
 def sdk_pool_bad_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee, change_writes,

indy_node/test/txn_validation/test_pool_upgrade_validation.py

@@ -7,7 +7,7 @@
 from indy_node.test.upgrade.helper import loweredVersion, sdk_ensure_upgrade_sent
 from indy_common.constants import JUSTIFICATION, JUSTIFICATION_MAX_SIZE
 
-from indy_node.test.upgrade.conftest import validUpgrade, nodeIds
+from indy_node.test.upgrade.conftest import validUpgrade, nodeIds, pckg
 
 
 def testPoolUpgradeFailsIfVersionIsLowerThanCurrent(

indy_node/test/upgrade/conftest.py

@@ -61,7 +61,12 @@ def mock_get_info_from_package_manager(package):
 
 
 @pytest.fixture(scope='function', params=[(EXT_PKT_NAME, EXT_PKT_VERSION), (APP_NAME, None)])
-def validUpgrade(nodeIds, tconf, monkeypatch, request):
+def pckg(request):
+    return request.param
+
+
+@pytest.fixture(scope='function')
+def validUpgrade(nodeIds, tconf, monkeypatch, pckg):
     schedule = {}
     unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
     startAt = unow + timedelta(seconds=100)
@@ -70,10 +75,10 @@ def validUpgrade(nodeIds, tconf, monkeypatch, request):
         schedule[i] = datetime.isoformat(startAt)
         startAt = startAt + timedelta(seconds=acceptableDiff + 3)
 
-    patch_packet_mgr_output(monkeypatch, request.param[0], request.param[1])
+    patch_packet_mgr_output(monkeypatch, pckg[0], pckg[1])
 
-    return dict(name='upgrade-{}'.format(randomText(3)), version=bumpedVersion(request.param[1]),
-                action=START, schedule=schedule, timeout=1, package=request.param[0],
+    return dict(name='upgrade-{}'.format(randomText(3)), version=bumpedVersion(pckg[1]),
+                action=START, schedule=schedule, timeout=1, package=pckg[0],
                 sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55')
 
 

indy_node/test/upgrade/test_broken_connection_control_tool.py

@@ -0,0 +1,66 @@
+import dateutil
+import dateutil.tz
+import pytest
+
+from datetime import datetime, timedelta
+from copy import deepcopy
+from indy_common.constants import START
+from indy_node.test.upgrade.conftest import patch_packet_mgr_output, EXT_PKT_NAME, EXT_PKT_VERSION
+
+from indy_node.server.upgrader import Upgrader
+from indy_node.test.upgrade.helper import sdk_ensure_upgrade_sent, bumpedVersion
+from plenum.test.helper import randomText
+
+from stp_core.common.log import getlogger
+
+delta = 2
+logger = getlogger()
+
+
+@pytest.fixture(scope='function')
+def pckg():
+    return (EXT_PKT_NAME, EXT_PKT_VERSION)
+
+
+@pytest.fixture(scope="module")
+def tconf(tconf):
+    old_delta = tconf.MinSepBetweenNodeUpgrades
+    tconf.MinSepBetweenNodeUpgrades = delta
+    yield tconf
+    tconf.MinSepBetweenNodeUpgrades = old_delta
+
+
+@pytest.fixture(scope='function')
+def skip_functions():
+    # Do this to prevent exceptions because of node_control_tool absence
+    old_action_failed = deepcopy(Upgrader._action_failed)
+
+    Upgrader._action_failed = \
+        lambda self, version, scheduled_on, upgrade_id, external_reason: 1
+    yield
+    Upgrader._action_failed = old_action_failed
+
+
+def test_node_doesnt_retry_upgrade(looper, nodeSet, validUpgrade, nodeIds,
+                                   sdk_pool_handle, sdk_wallet_trustee, tconf,
+                                   skip_functions):
+    schedule = {}
+    unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
+    startAt = unow + timedelta(seconds=delta)
+    for i in nodeIds:
+        schedule[i] = datetime.isoformat(startAt)
+        startAt = startAt + timedelta(seconds=delta)
+    validUpgrade['schedule'] = schedule
+
+    # Emulating connection problems
+    for node in nodeSet:
+        node.upgrader.retry_limit = 0
+
+    # Send upgrade
+    sdk_ensure_upgrade_sent(looper, sdk_pool_handle,
+                            sdk_wallet_trustee, validUpgrade)
+    looper.runFor(len(nodeIds) * delta)
+
+    # Every node, including bad_node, tried to upgrade only once
+    for node in nodeSet:
+        assert node.upgrader.spylog.count(Upgrader.processLedger.__name__) == 1

setup.py

@@ -56,7 +56,7 @@
     data_files=[(
         (BASE_DIR, ['data/nssm_original.exe'])
     )],
-    install_requires=['indy-plenum-dev==1.6.646',
+    install_requires=['indy-plenum-dev==1.6.652',
                       'python-dateutil',
                       'timeout-decorator==0.4.0'],
     setup_requires=['pytest-runner'],