Merge pull request #221 from aristanetworks/release-1.2.2

Release 1.2.2

MANIFEST.in

@@ -1,4 +1,4 @@
-include README.rst
+include README.md
 include Makefile
 include *.spec
 include *.txt

README.md

@@ -372,6 +372,7 @@ It is strongly recommended to use one from the below:
 - ```Feat```: Create a capability e.g. feature, test, dependency
 - ```Fix```: Fix an issue e.g. bug, typo, accident, misstatement
 - ```Doc```: Refactor of documentation, e.g. help files
+- ```Example```: Add a new example or modify an [existing one](docs/labs/)
 - ```Test```: Add or refactor anything regarding test, e.g add a new testCases or missing testCases
 - ```Refactor```: A code change that MUST be just a refactoring
 - ```Bump```: Increase the version of something e.g. dependency

VERSION

@@ -1 +1 @@
-1.2.0
+1.2.2

cvprac/__init__.py

@@ -32,5 +32,5 @@
 ''' RESTful API Client class for Cloudvision(R) Portal
 '''
 
-__version__ = '1.2.0'
+__version__ = '1.2.2'
 __author__ = 'Arista Networks, Inc.'

cvprac/cvp_client.py

@@ -100,7 +100,7 @@
 
 import requests
 from requests.exceptions import ConnectionError, HTTPError, Timeout, \
-    ReadTimeout, TooManyRedirects
+    ReadTimeout, TooManyRedirects, JSONDecodeError
 
 from cvprac.cvp_api import CvpApi
 from cvprac.cvp_client_errors import CvpApiError, CvpLoginError, \
@@ -602,6 +602,11 @@ def _make_request(self, req_type, url, timeout, data=None,
                     or delete request failed and no session could be
                     established to a CVP node.  Destroy the class and
                     re-instantiate.
+                JSONDecodeError: A JSONDecodeError is raised when the response
+                    content contains invalid JSON. Potentially in the case of
+                    Resource APIs that will return Stream JSON format with
+                    multiple object or in the case where the response contains
+                    incomplete JSON.
         '''
         # pylint: disable=too-many-branches
         # pylint: disable=too-many-statements
@@ -656,25 +661,51 @@ def _make_request(self, req_type, url, timeout, data=None,
                     raise error
                 continue
             break
-        resp_data = None
-        if response:
-            try:
-                resp_data = response.json()
-            except ValueError as error:
-                self.log.debug('Error trying to decode request response %s',
-                               error)
-                if 'Extra data' in str(error):
-                    self.log.debug('Found multiple objects in response data.'
-                                   'Attempt to decode')
-                    decoded_data = json_decoder(response.text)
-                    resp_data = dict(data=decoded_data)
-                else:
-                    self.log.debug('Attempt to return response text')
-                    resp_data = dict(data=response.text)
-        else:
+
+        if not response:
             self.log.debug('Received no response for request %s %s',
                            req_type, url)
-        return resp_data
+            return None
+
+        # Added check for response.content being 'null' because of the
+        # service account APIs being a special case /services/ API that
+        # returns a null string for no objects instead of an empty string.
+        if not response.content or response.content == b'null':
+            return {'data': []}
+
+        try:
+            resp_data = response.json()
+            if (resp_data is not None and 'result' in resp_data
+                    and '/resources/' in full_url):
+                # Resource APIs use JSON streaming and will return
+                # multiple JSON objects during GetAll type API
+                # calls. We are wrapping the multiple objects into
+                # a key "data" and we also return a dictionary with
+                # key "data" as an empty dict for no data. This
+                # checks and keeps consistent the "data" key wrapper
+                # for a Resource API GetAll that returns a single
+                # object.
+                return {'data': [resp_data]}
+            return resp_data
+        except JSONDecodeError as error:
+            # Truncate long error messages
+            err_str = str(error)
+            if len(err_str) > 700:
+                err_str = f"{err_str[:300]}[... truncated ...]" \
+                          f" {err_str[-300:]}"
+            self.log.debug('Error trying to decode request response - %s',
+                           err_str)
+            if 'Extra data' in str(error):
+                self.log.debug('Found multiple objects or NO objects in'
+                               'response data. Attempt to decode')
+                decoded_data = json_decoder(response.text)
+                return {'data': decoded_data}
+            else:
+                self.log.error('Unknown format for JSONDecodeError - %s',
+                               err_str)
+                # Suppressing context as per
+                # https://peps.python.org/pep-0409/
+                raise JSONDecodeError(err_str) from None
 
     def _send_request(self, req_type, full_url, timeout, data=None,
                       files=None):

dev-requirements.txt

@@ -6,3 +6,4 @@ pep8
 pyflakes
 pylint
 pyyaml
+twine

docs/labs/lab02-inventory-operations/get_running_configs_by_time.py

@@ -23,8 +23,12 @@
         }
     }
     try:
-        resultRunningConfig = clnt.post(uri, data=data)[0]['config']
+        resultRunningConfig = clnt.post(uri, data=data)
+        for idx in resultRunningConfig:
+            if 'config' in idx:
+                result = idx['config']
+                break
         with open(device['hostname']+'.cfg','w') as f:
-            f.write(resultRunningConfig)
+            f.write(result)
     except Exception as e:
         print("Not able to get configuration for device {} - exception {}".format(device['fqdn'], e))

docs/labs/lab02-inventory-operations/remove_all_devices_legacy.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2021 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+from cvprac.cvp_client import CvpClient
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+# Create connection to CloudVision
+clnt = CvpClient()
+clnt.connect(nodes=['cvp1'], username="username",password="password")
+
+inventory = clnt.api.get_inventory()
+
+devices = []
+for netelement in inventory:
+   devices.append(netelement['systemMacAddress'])
+
+# Remove devices from provisioning
+# This is a legacy API call that removes the devices from Network Provisioning 
+# in CVP versions older than 2021.3.0, however it does not remove them from 
+# the Device Inventory as that requires the streaming agent (TerminAttr) to be shutdown,
+# which this API does not support.
+# To fully decommission a device the device_decommissioning() API can be used, which is
+# supported from 2021.3.0+.
+# Note that using the delete_devices() function post CVP 2021.3.0 the device will be 
+# automatically added back to the Undefined container.
+clnt.api.delete_devices(devices)

docs/labs/lab02-inventory-operations/remove_and_decommission_device.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2022 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+from cvprac.cvp_client import CvpClient
+import ssl
+import uuid
+import time
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+# Create connection to CloudVision
+clnt = CvpClient()
+clnt.connect(nodes=['cvp1'], username="username", password="password")
+
+device_id = input("Serial number of the device to be decommissioned: ")
+request_id = str(uuid.uuid4())
+clnt.api.device_decommissioning(device_id, request_id)
+
+# This API call will fully decommission the device, ie remove it from both
+# Network Provisioning and Device Inventory (telemetry). It send an eAPI request
+# to EOS to shutdown the TerminAttr daemon, waits for streaming to stop and then removes
+# the device from provisioning and finally decommissions it. This operation can take a few minutes.
+# Supported from CVP 2021.3.0+ and CVaaS.
+decomm_status = "DECOMMISSIONING_STATUS_SUCCESS"
+decomm_result = ""
+while decomm_result != decomm_status:
+    decomm_result = clnt.api.device_decommissioning_status_get_one(request_id)['value']['status']
+    time.sleep(10)
+
+print(decomm_result)

docs/labs/lab02-inventory-operations/remove_devices_from_container.py → docs/labs/lab02-inventory-operations/remove_devices_from_container_legacy.py

@@ -15,8 +15,18 @@
 # Get devices in a specific container
 inventory = clnt.api.get_devices_in_container("Undefined")
 
+# Create device list
 devices = []
 for netelement in inventory:
    devices.append(netelement['systemMacAddress'])
 
+# Remove devices from provisioning
+# This is a legacy API call that removes the devices from Network Provisioning 
+# in CVP versions older than 2021.3.0, however it does not remove them from 
+# the Device Inventory as that requires the streaming agent (TerminAttr) to be shutdown,
+# which this API does not support.
+# To fully decommission a device the device_decommissioning() API can be used, which is
+# supported from 2021.3.0+.
+# Note that using the delete_devices() function post CVP 2021.3.0 the device will be 
+# automatically added back to the Undefined container.
 clnt.api.delete_devices(devices)

docs/labs/lab02-inventory-operations/remove_devices.py → docs/labs/lab02-inventory-operations/remove_devices_legacy.py

@@ -18,4 +18,13 @@
 devices = ["50:08:00:a7:ca:c3","50:08:00:b1:5b:0b","50:08:00:60:c6:76",
            "50:08:00:25:9d:36","50:08:00:8b:ee:b1","50:08:00:8c:22:49"]
 
+# Remove devices from provisioning
+# This is a legacy API call that removes the devices from Network Provisioning 
+# in CVP versions older than 2021.3.0, however it does not remove them from 
+# the Device Inventory as that requires the streaming agent (TerminAttr) to be shutdown,
+# which this API does not support.
+# To fully decommission a device the device_decommissioning() API can be used, which is
+# supported from 2021.3.0+.
+# Note that using the delete_devices() function post CVP 2021.3.0 the device will be 
+# automatically added back to the Undefined container.
 clnt.api.delete_devices(devices)

docs/labs/lab03-configlet-management/config_search.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2022 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+from cvprac.cvp_client import CvpClient
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+# Create connection to CloudVision
+clnt = CvpClient()
+clnt.connect(nodes=['cvp1'], username="username",password="password")
+
+def main():
+
+   print('Retrieving configlets ...')
+
+   inventory = clnt.api.get_inventory()
+   data = clnt.api.get_configlets_and_mappers()['data']
+   print(data)
+
+   print('Number of configlets:', len(data['configlets']))
+
+   searchAgain = True
+   while searchAgain:
+        try:
+            search = input( "\nEnter Config Line: " )
+            print(f"\n\n\'{search}\' has been found in following configlets:\n\n")
+            print(f"{'Hostname':<30}{'Serial number':<50}{'MAC address':<30}{'Configlets':<40}")
+            print("=" * 150)
+            for i in inventory:
+                device = i['hostname']
+                device_sn = i['serialNumber']
+                device_mac = i['systemMacAddress']
+                configlet_list = []
+                for c in data['configlets']:
+                    for g in data['generatedConfigletMappers']:
+                        if device_mac == g['netElementId'] and c['key'] == g['configletBuilderId'] and search in c['config']:
+                            configlet_list.append(c['name'])
+                    for k in data['configletMappers']:
+                        if device_mac == k['objectId'] and c['key'] == k['configletId'] and search in c['config']:
+                            configlet_list.append(c['name'])
+                configlet_list_final = ",".join(configlet_list)
+                if len(configlet_list) > 0:                
+                    print(f"{device:<30}{device_sn:<50}{device_mac:<30}{configlet_list_final:<30}")
+
+        except KeyboardInterrupt:
+            print('\nExiting... \n')
+            return
+
+if __name__ == '__main__':
+   main()
+

docs/labs/lab06-provisioning/auto_reconcile_on_rc_change.py

@@ -0,0 +1,64 @@
+# Copyright (c) 2022 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+# This script can be run as a cronjob to periodically reconcile all devices
+# that are out of configuration compliance in environments where the running-config
+# is still modified via the CLI often.
+from cvprac.cvp_client import CvpClient
+import ssl
+from datetime import datetime
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+clnt = CvpClient()
+clnt.set_log_level(log_level='WARNING')
+
+# Reading the service account token from a file
+with open("token.tok") as f:
+    token = f.read().strip('\n')
+
+clnt = CvpClient()
+clnt.connect(nodes=['cvp1'], username='',password='',api_token=token)
+
+inventory = clnt.api.get_inventory()
+
+compliance = {"0001": "Config is out of sync",
+        "0003": "Config & image out of sync",
+        "0004": "Config, Image and Device time are in sync",
+        "0005": "Device is not reachable",
+        "0008": "Config, Image and Extensions are out of sync",
+        "0009": "Config and Extensions are out of sync",
+        "0012": "Config, Image, Extension and Device time are out of sync",
+        "0013": "Config, Image and Device time are out of sync",
+        "0014": "Config, Extensions and Device time are out of sync",
+        "0016": "Config and Device time are out of sync"
+    }
+
+non_compliants = []
+taskIds = []
+for device in inventory:
+    if device['complianceCode'] in compliance.keys():
+        # create a list of non-compliant devices for reporting purposes
+        non_compliants.append(device['hostname'])
+        dev_mac = device['systemMacAddress']
+        # check if device already has reconciled config and save the key if it does
+        try:
+            configlets = clnt.api.get_configlets_by_device_id(dev_mac)
+            for configlet in configlets:
+                if configlet['reconciled'] == True:
+                    configlet_key = configlet['key']
+                    break
+                else:
+                    configlet_key = ""
+            rc = clnt.api.get_device_configuration(dev_mac)
+            name = 'RECONCILE_' + device['serialNumber']
+            update = clnt.api.update_reconcile_configlet(dev_mac, rc, configlet_key, name, True)
+            # if the device had no reconciled config, it means we need to append the reconciled
+            # configlet to the list of applied configlets on the device
+            if configlet_key == "":
+                addcfg = clnt.api.apply_configlets_to_device("auto-reconciling",device,[update['data']])
+                clnt.api.cancel_task(addcfg['data']['taskIds'][0])            
+        except Exception as e:
+            continue
+print(f"The non compliant devices were: {str(non_compliants)}")

docs/labs/lab07-aaa/create_svc_account.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2021 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+from cvprac.cvp_client import CvpClient
+from cvprac.cvp_client_errors import CvpApiError
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+# Create connection to CloudVision using user/password (on-prem only)
+clnt = CvpClient()
+clnt.connect(['cvp1'],'username', 'password')
+
+username = "cvprac2"
+description = "test cvprac"
+roles = ["network-admin", "clouddeploy"] # both role names and role IDs are supported
+status = 1 # 1 is equivalent to "ACCOUNT_STATUS_ENABLED"
+clnt.api.svc_account_set(username, description, roles, status)

docs/labs/lab07-aaa/create_svc_account_token.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2021 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the COPYING file.
+
+from cvprac.cvp_client import CvpClient
+from cvprac.cvp_client_errors import CvpApiError
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+# Create connection to CloudVision using user/password (on-prem only)
+clnt = CvpClient()
+clnt.connect(['cvp1'],'username', 'password')
+
+username = "cvprac2"
+duration = "31536000s" # 1 year validity
+description = "test cvprac"
+svc_token = clnt.api.svc_account_token_set(username, duration, description)
+
+# Write the token to file in <username>.tok format
+with open(svc_token[0]['value']['user'] + ".tok", "w") as f:
+    f.write(svc_token[0]['value']['token'])