Merge pull request #663 from guardicore/release/1.8.2

Release/1.8.2

.gitignore

@@ -82,5 +82,11 @@ MonkeyZoo/*
 !MonkeyZoo/config.tf
 !MonkeyZoo/MonkeyZooDocs.pdf
 
+# Exported monkey telemetries
+/monkey/telem_sample/
+
+# Profiling logs
+profiler_logs/
+
 # vim swap files
 *.swp

.travis.yml

@@ -1,10 +1,15 @@
 # Infection Monkey travis.yml. See Travis documentation for information about this file structure.
 
+# If you change this file, you can validate using Travis CI's Build Config Explorer https://config.travis-ci.com/explore
+
 group: travis_latest
 
 language: python
 
-cache: pip
+cache:
+  - pip
+  - directories:
+    - "$HOME/.npm"
 
 python:
 - 3.7
@@ -18,6 +23,16 @@ install:
 - pip install coverage  # for code coverage
 - pip install -r monkey/infection_monkey/requirements.txt  # for unit tests
 
+# node + npm + eslint
+- node --version
+- npm --version
+- nvm --version
+- nvm install node
+- nvm use node
+- npm i -g eslint
+- node --version
+- npm --version
+
 before_script:
 # Set the server config to `testing`. This is required for for the UTs to pass.
 - python monkey/monkey_island/cc/set_server_config.py testing
@@ -36,7 +51,7 @@ script:
 ## Display the linter issues
 - cat flake8_warnings.txt
 ## Make sure that we haven't increased the amount of warnings.
-- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=190
+- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=120
 - if [ $(tail -n 1 flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi
 
 ## Run unit tests
@@ -48,13 +63,10 @@ script:
 
 # Check JS code. The npm install must happen AFTER the flake8 because the node_modules folder will cause a lot of errors.
 - cd monkey_island/cc/ui
-- npm i
-- npm i -g eslint
-- cd -
-- cd monkey_island/cc/ui
-- eslint ./src --quiet
-- JS_WARNINGS_AMOUNT_UPPER_LIMIT=37
-- eslint ./src --max-warnings $JS_WARNINGS_AMOUNT_UPPER_LIMIT
+- npm ci  # See https://docs.npmjs.com/cli/ci.html
+- eslint ./src --quiet  # Test for errors
+- JS_WARNINGS_AMOUNT_UPPER_LIMIT=490
+- eslint ./src --max-warnings $JS_WARNINGS_AMOUNT_UPPER_LIMIT  # Test for max warnings
 
 after_success:
   # Upload code coverage results to codecov.io, see https://github.com/codecov/codecov-bash for more information

envs/monkey_zoo/.gitignore

@@ -1 +1,2 @@
 logs/
+/blackbox/tests/performance/telem_sample

envs/monkey_zoo/blackbox/README.md

@@ -10,10 +10,37 @@ In order to execute the entire test suite, you must know the external IP of the
 this information in the GCP Console `Compute Engine/VM Instances` under _External IP_. 
 
 #### Running in command line
-Run the following command:
+Blackbox tests have following parameters:
+- `--island=IP` Sets island's IP
+- `--no-gcp` (Optional) Use for no interaction with the cloud (local test).
+- `--quick-performance-tests` (Optional) If enabled performance tests won't reset island and won't send telemetries, 
+instead will just test performance of endpoints in already present island state.
 
-`monkey\envs\monkey_zoo\blackbox>python -m pytest --island=35.207.152.72:5000 test_blackbox.py`
+Example run command:
+
+`monkey\envs\monkey_zoo\blackbox>python -m pytest -s --island=35.207.152.72:5000 test_blackbox.py`
 
 #### Running in PyCharm
-Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the 
+Configure a PyTest configuration with the additional arguments `-s --island=35.207.152.72` on the 
 `monkey\envs\monkey_zoo\blackbox`.
+
+### Running telemetry performance test
+
+**Before running performance test make sure browser is not sending requests to island!** 
+
+To run telemetry performance test follow these steps:
+1. Gather monkey telemetries.
+    1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have 
+    exported telemetries already.
+    2. Run monkey and wait until infection is done.
+    3. All telemetries are gathered in `monkey/telem_sample`
+2. Run telemetry performance test.
+    1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems`
+    2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply 
+    telemetries gathered.
+        1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox`
+        2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate
+        telemetries 4 times.
+        3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion.
+    3. Performance test will run as part of BlackBox tests or you can run it separately by adding 
+    `-k 'test_telem_performance'` option.

envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py

@@ -1,59 +1,48 @@
 import logging
 from datetime import timedelta
+from typing import Dict
 
 from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer
-from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
+from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 
-MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2)
-MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5)
-
-REPORT_URLS = [
-    "api/report/security",
-    "api/attack/report",
-    "api/report/zero_trust/findings",
-    "api/report/zero_trust/principles",
-    "api/report/zero_trust/pillars"
-]
-
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 
 class PerformanceAnalyzer(Analyzer):
 
-    def __init__(self, island_client: MonkeyIslandClient, break_if_took_too_long=False):
-        self.break_if_took_too_long = break_if_took_too_long
-        self.island_client = island_client
-
-    def analyze_test_results(self) -> bool:
-        if not self.island_client.is_all_monkeys_dead():
-            raise RuntimeError("Can't test report times since not all Monkeys have died.")
+    def __init__(self, performance_test_config: PerformanceTestConfig, endpoint_timings: Dict[str, timedelta]):
+        self.performance_test_config = performance_test_config
+        self.endpoint_timings = endpoint_timings
 
-        # Collect timings for all pages
-        self.island_client.clear_caches()
-        report_resource_to_response_time = {}
-        for url in REPORT_URLS:
-            report_resource_to_response_time[url] = self.island_client.get_elapsed_for_get_request(url)
-
-        # Calculate total time and check each page
+    def analyze_test_results(self):
+        # Calculate total time and check each endpoint
         single_page_time_less_then_max = True
         total_time = timedelta()
-        for page, elapsed in report_resource_to_response_time.items():
-            logger.info(f"page {page} took {str(elapsed)}")
+        for endpoint, elapsed in self.endpoint_timings.items():
             total_time += elapsed
-            if elapsed > MAX_ALLOWED_SINGLE_PAGE_TIME:
+            if elapsed > self.performance_test_config.max_allowed_single_page_time:
                 single_page_time_less_then_max = False
 
-        total_time_less_then_max = total_time < MAX_ALLOWED_TOTAL_TIME
+        total_time_less_then_max = total_time < self.performance_test_config.max_allowed_total_time
 
-        logger.info(f"total time is {str(total_time)}")
+        PerformanceAnalyzer.log_slowest_endpoints(self.endpoint_timings)
+        LOGGER.info(f"Total time is {str(total_time)}")
 
         performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max
 
-        if self.break_if_took_too_long and not performance_is_good_enough:
-            logger.warning(
+        if self.performance_test_config.break_on_timeout and not performance_is_good_enough:
+            LOGGER.warning(
                 "Calling breakpoint - pausing to enable investigation of island. Type 'c' to continue once you're done "
                 "investigating. Type 'p timings' and 'p total_time' to see performance information."
             )
             breakpoint()
 
         return performance_is_good_enough
+
+    @staticmethod
+    def log_slowest_endpoints(endpoint_timings, max_endpoints_to_display=100):
+        slow_endpoint_list = list(endpoint_timings.items())
+        slow_endpoint_list.sort(key=lambda x: x[1], reverse=True)
+        slow_endpoint_list = slow_endpoint_list[:max_endpoints_to_display]
+        for endpoint in slow_endpoint_list:
+            LOGGER.info(f"{endpoint[0]} took {str(endpoint[1])}")

envs/monkey_zoo/blackbox/conftest.py

@@ -4,8 +4,23 @@
 def pytest_addoption(parser):
     parser.addoption("--island", action="store", default="",
                      help="Specify the Monkey Island address (host+port).")
+    parser.addoption("--no-gcp", action="store_true", default=False,
+                     help="Use for no interaction with the cloud.")
+    parser.addoption("--quick-performance-tests", action="store_true", default=False,
+                     help="If enabled performance tests won't reset island and won't send telemetries, "
+                          "instead will just test performance of already present island state.")
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope='session')
 def island(request):
     return request.config.getoption("--island")
+
+
+@pytest.fixture(scope='session')
+def no_gcp(request):
+    return request.config.getoption("--no-gcp")
+
+
+@pytest.fixture(scope='session')
+def quick_performance_tests(request):
+    return request.config.getoption("--quick-performance-tests")

envs/monkey_zoo/blackbox/island_client/monkey_island_client.py

@@ -1,8 +1,8 @@
-from datetime import timedelta
-from time import sleep
 import json
 
 import logging
+from time import sleep
+
 from bson import json_util
 
 from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests
@@ -31,7 +31,7 @@ def import_config(self, config_contents):
 
     @avoid_race_condition
     def run_monkey_local(self):
-        response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"})
+        response = self.requests.post_json("api/local-monkey", data={"action": "run"})
         if MonkeyIslandClient.monkey_ran_successfully(response):
             LOGGER.info("Running the monkey.")
         else:
@@ -96,13 +96,3 @@ def clear_caches(self):
         response = self.requests.get("api/test/clear_caches")
         response.raise_for_status()
         return response
-
-    def get_elapsed_for_get_request(self, url):
-        response = self.requests.get(url)
-        if response.ok:
-            LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}")
-            return response.elapsed
-        else:
-            LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}")
-            # instead of raising for status, mark failed responses as maxtime
-            return timedelta.max()

envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py

@@ -1,9 +1,15 @@
+from typing import Dict
+from datetime import timedelta
+
+
 import requests
 import functools
 
-# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
+from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod
+
 import logging
 
+# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
 NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
                 '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
 LOGGER = logging.getLogger(__name__)
@@ -14,6 +20,26 @@ class MonkeyIslandRequests(object):
     def __init__(self, server_address):
         self.addr = "https://{IP}/".format(IP=server_address)
         self.token = self.try_get_jwt_from_server()
+        self.supported_request_methods = {SupportedRequestMethod.GET: self.get,
+                                          SupportedRequestMethod.POST: self.post,
+                                          SupportedRequestMethod.PATCH: self.patch,
+                                          SupportedRequestMethod.DELETE: self.delete}
+
+    def get_request_time(self, url, method: SupportedRequestMethod, data=None):
+        response = self.send_request_by_method(url, method, data)
+        if response.ok:
+            LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}")
+            return response.elapsed
+        else:
+            LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}")
+            # instead of raising for status, mark failed responses as maxtime
+            return timedelta.max
+
+    def send_request_by_method(self, url, method=SupportedRequestMethod.GET, data=None):
+        if data:
+            return self.supported_request_methods[method](url, data)
+        else:
+            return self.supported_request_methods[method](url)
 
     def try_get_jwt_from_server(self):
         try:
@@ -55,12 +81,20 @@ def post(self, url, data):
                              verify=False)
 
     @_Decorators.refresh_jwt_token
-    def post_json(self, url, dict_data):
+    def post_json(self, url, data: Dict):
         return requests.post(self.addr + url,  # noqa: DUO123
-                             json=dict_data,
+                             json=data,
                              headers=self.get_jwt_header(),
                              verify=False)
 
+    @_Decorators.refresh_jwt_token
+    def patch(self, url, data: Dict):
+        return requests.patch(self.addr + url,  # noqa: DUO123
+                              data=data,
+                              headers=self.get_jwt_header(),
+                              verify=False
+        )
+
     @_Decorators.refresh_jwt_token
     def delete(self, url):
         return requests.delete(  # noqa: DOU123

envs/monkey_zoo/blackbox/island_client/supported_request_method.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class SupportedRequestMethod(Enum):
+    GET = "GET"
+    POST = "POST"
+    PATCH = "PATCH"
+    DELETE = "DELETE"