phm

#!/usr/bin/env python 
#
# Copyright (c) 2014, Arista Networks, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#  - Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#  - Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#  - Neither the name of Arista Networks nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ARISTA NETWORKS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Port Health Monitor
#
#    Version 3.0 9/4/2013
#    Written by: 
#       Andrei Dvornic, Arista Networks
#
#    Revision history:
#       1.0 - initial release
#       2.0 - turn off interface if error condition persists
#           - syslog on degradation
#           - removed email notifications
#       3.0 - use eAPI for polling interface counters
#           - configuration at the top of the script
#       4.0 - emailing option
#           - bug fixes

'''
   DESCRIPTION
      The Port Health Monitor script generates a syslog notifications
      whenever the FCS/symbol errors counters on an interface exceed
      some pre-configured levels.

   INSTALLATION
      In order to install Port Health Monitor, copy 'phm' to
      /mnt/flash.

      Port Health Monitor can then be started using:
         (bash:root)# /mnt/flash/phm [<options>] &
      and you can use 'nohup' utility in order to make this persistent
      over ssh:
         (bash:root)# nohup /mnt/flash/phm [<options>] &      

      See: 
         (bash:root)# /mnt/flash/phm --help
      for details about the command-line options.

      In order to run Port Health Monitor as a daemon (persistent
      after reboot), add the following to the startup-config:

         event-handler phm
            trigger on-boot
            action bash sudo /usr/bin/daemonize 
               /mnt/flash/phm [<options>] &
               
      Port Health Monitor process name is 'phm', so standard Linux
      tools can be used to stop and restart it with different
      command-line options:

      e.g.
         (bash:root)# pkill phm
         (bash:root)# /mnt/flash/phm [<new-options>] &

      Note that in order to make sure the Port Health Monitor does not
      restart on reboot / starts with a different config on reboot,
      the startup-config has to be changed accordingly.

      In order to uninstall Port Health Monitor, use:
         (bash:root)# pkill phm
         (bash:root)# rm /mnt/flash/phm

   CONFIGURATION/DEBUGGING
      The configuration of the script can be found in the highlighted
      section below.

      The base values for the interface error counters are the values
      of the counters when the script is started or when an interface
      becomes connected. Each base value is reset after any of the
      warning levels is exceeded and a syslog message is generated.
      The base values are also reset whenever the counters are
      cleared.

      If the failure warning levels are continuously being exceeded,
      then the corresponding interface is automatically turned
      off. This check is performed by verifying whether the increase
      in the error counters exceed the failure threshold for several
      polling intervals.
      
      In order to receive email alerts, use the '--email' or '-e' 
      command line option.

      In order to enable debugging output to stdout, use the '--debug'
      command line option.
      e.g.
         (bash:root)# /mnt/flash/phm -d &
      Note that the output can be quite verbose so it is recommended
      that this option is used with caution, and only for debugging
      purposes.

   COMPATIBILITY
      Version 4.0 has been developed and tested against
      EOS-4.14.1F. Please reach out to support@aristanetworks.com if
      you want to run this against a different EOS release.

  LIMITATIONS
      None known.
'''

import jsonrpclib
import optparse
import socket
import syslog
import time
import subprocess

from ctypes import cdll, byref, create_string_buffer

#-------------------Configuration------------------------
EAPI_USERNAME = 'admin'
EAPI_PASSWORD = ''
EAPI_ENABLE_PASSWORD = ''

# http/https
EAPI_METHOD = 'http'

# How often to poll for information (seconds)
EAPI_POLL_INTERVAL = 5       # in seconds

# Optional Emailing recipient
EMAIL_TO = "change.me.recipient@domain"

# - - - - - - - - - - - - - - - - - - - - - - - - - - - -

# Increase in FCS errors which triggers a degradation alert
FCS_DEGRADE_TOLERANCE = 100

# Increase in FCS errors which triggers a failure alert
FCS_ERROR_TOLERANCE = 200

# Increase in symbol errors which triggers a degradation alert
SYMBOL_DEGRADE_TOLERANCE = 100

# Increase in symbol errors which triggers a failure alert
SYMBOL_ERROR_TOLERANCE = 200

# number of failure intervals for which an error condition 
# needs to be preserved before being turned off
FAIL_INTERVAL_COUNT = 3
#--------------------------------------------------------

errors = {}

def trace( msg ):
    if debug:
        print msg
        
def email( msg ):
    if email_active:
        # send msg to the email body, get the hostname+first word of msg as subject
        bashCommand = 'echo ' + msg + ' |  email -i -s \"[$HOSTNAME] ' + \
                      msg.split(' ')[0] + '\" ' + EMAIL_TO
        p = subprocess.Popen( bashCommand , 
                              shell=True, 
                              stdout=subprocess.PIPE, 
                              stderr=subprocess.PIPE )
        p.wait()
        

def log_fcs_degradation( intf, start, end ):
    msg = '%%INTERFACES-4-FCS_DEGRADATION_WARNING: ' \
          'FCS degradation detected on %s - error count ' \
          'increased from %s to %s' % ( intf, start, end )
    syslog.syslog( msg )
    email( msg )
    trace( msg )

def log_fcs_errors( intf, start, end ):
    msg = '%%INTERFACES-4-FCS_ERRORS_WARNING: ' \
          'FCS errors detected on %s - error count ' \
          'increased from %s to %s' % ( intf, start, end ) 
    syslog.syslog( msg )
    email( msg )
    trace( msg )

def log_fcs_failure( intf ):
    msg = '%%INTERFACES-4-FCS_FAILURE_WARNING: ' \
          'Interface %s was turned off because a constant ' \
          'increase in the number of FCS errors was detected' % intf 
    syslog.syslog( msg )
    email( msg )
    trace( msg )

def log_symbol_degradation( intf, start, end ):
    msg = '%%INTERFACES-4-SYMBOL_DEGRADATION_WARNING: ' \
          'SYMBOL degradation detected on %s - error count ' \
          'increased from %s to %s' % ( intf, start, end )
    syslog.syslog( msg )
    email( msg )
    trace( msg )

def log_symbol_errors( intf, start, end ):
    msg = '%%INTERFACES-4-SYMBOL_ERRORS_WARNING: ' \
         'SYMBOL errors detected on %s - error count ' \
         'increased from %s to %s' % ( intf, start, end ) 
    syslog.syslog( msg )
    email( msg )
    trace( msg )

def log_symbol_failure( intf ):
    msg = '%%INTERFACES-4-SYMBOL_FAILURE_WARNING: ' \
          'Interface %s was turned off because a constant ' \
          'increase in the number of SYMBOL errors was detected' % intf 
    syslog.syslog( msg )
    email( msg )
    trace( msg )


def setProcName(newname):
    libc = cdll.LoadLibrary( 'libc.so.6' )
    buff = create_string_buffer( len( newname ) + 1 )
    buff.value = newname    
    libc.prctl( 15, byref( buff ), 0, 0, 0)

class Error( Exception ):
    pass

class ConnectionError( Error ):
    '''
    Raised when connection to a eAPI server cannot
    be established.
    '''
    pass

class EApiClient( object ):

    def __init__( self ):
        url = '%s://%s:%s@localhost/command-api' % \
              ( EAPI_METHOD, EAPI_USERNAME, EAPI_PASSWORD )
        self.client = jsonrpclib.Server( url  )

        try:
            self.runEnableCmds( [] )
        except socket.error:
            raise ConnectionError( url )

    def runEnableCmds( self, cmds, mode='json' ):
        result = self.client.runCmds( 
            1, [ { 'cmd': 'enable', 
                   'input': EAPI_ENABLE_PASSWORD } ] +
            cmds, mode)[ 1: ]

        if mode == 'text':
            return [ x.values()[ 0 ] for x in result if x.values() ]
        else:
            return result
        
    def connectedInterfaces( self ):
        # Using text mode in order to be able to match interface shortname
        # in interfaceErrorCounters
        output = self.runEnableCmds( 
            [ 'show interfaces status connected' ], 
            mode='text' )[ 0 ].split( '\n' )[ 1 : -1 ]
        return [ x.split()[ 0 ] for x in output if x.split() ]

    def interfaceErrorCounters( self, interfaces ):
        result = {}
        output = self.runEnableCmds( 
            [ 'show interfaces counters errors' ],
            # Not yet converted
            mode='text' )[ 0 ].split( '\n' )[ 1 : -1 ]
        for line in output:
            tokens = line.split()
            if tokens[ 0 ] in interfaces:
                result[ tokens[ 0 ] ] = { 'fcs': int( tokens[ 1 ] ),
                                          'symbol': int( tokens[ 3 ] ) }
        return result

    def connectedInterfacesCounters( self ):
        return self.interfaceErrorCounters( self.connectedInterfaces() )
    
    def turnOffInterface( self, interface ):
        self.runEnableCmds( [ 'configure',
                              'interface %s' % interface,
                              'shutdown' ] )

def checkInterfaces():
    eapi = EApiClient()
    interface_counters = {}

    while True:
        trace( 'Polling Sysdb' )
        counters = eapi.connectedInterfacesCounters()
        trace( 'Counters: %s' % counters )
        for intf, details in counters.iteritems():
            if intf not in interface_counters:
                errors[ intf ] = {}
                errors[ intf ][ 'fcs' ] = details[ 'fcs' ]
                errors[ intf ][ 'symbol' ] = details[ 'symbol' ]
                errors[ intf ][ 'fcs_count' ] = 0
                errors[ intf ][ 'symbol_count' ] = 0
            else:
                fcsBase = errors[ intf ][ 'fcs' ]
                if details[ 'fcs' ] < fcsBase:
                    # Counters cleared
                    trace( 'Resetting FCS base for %s' % intf )
                    errors[ intf ][ 'fcs' ] = details[ 'fcs' ]
                    errors[ intf ][ 'fcs_count' ] = 0
                elif details[ 'fcs' ] > fcsBase + FCS_ERROR_TOLERANCE:
                    log_fcs_errors( intf, fcsBase, details[ 'fcs' ] )
                    errors[ intf ][ 'fcs' ] = details[ 'fcs' ]
                    errors[ intf ][ 'fcs_count' ] += 1
                elif details[ 'fcs' ] > fcsBase + FCS_DEGRADE_TOLERANCE:
                    log_fcs_degradation( intf, fcsBase, details[ 'fcs' ] )
                    errors[ intf ][ 'fcs' ] = details[ 'fcs' ]
                    errors[ intf ][ 'fcs_count' ] = 0
                else:
                    trace( 'FCS counters for %s OK' % intf )
                    errors[ intf ][ 'fcs_count' ] = 0
                    
                if errors[ intf ][ 'fcs_count' ] == FAIL_INTERVAL_COUNT:
                    log_fcs_failure( intf )
                    eapi.turnOffInterface( intf )


                symbolBase = errors[ intf ][ 'symbol' ]
                if details[ 'symbol' ] < symbolBase:
                    # Counters cleared
                    trace( 'Resetting SYMBOL base for %s' % intf )
                    errors[ intf ][ 'symbol' ] = details[ 'symbol' ]
                    errors[ intf ][ 'symbol_count' ] = 0                    
                elif details[ 'symbol' ] > symbolBase + SYMBOL_ERROR_TOLERANCE:
                    log_symbol_errors( intf, symbolBase, details[ 'symbol' ] )
                    errors[ intf ][ 'symbol' ] = details[ 'symbol' ]
                    errors[ intf ][ 'symbol_count' ] += 1
                elif details[ 'symbol' ] > symbolBase + \
                        SYMBOL_DEGRADE_TOLERANCE:
                    log_symbol_degradation( intf, symbolBase, 
                                            details[ 'symbol' ] )
                    errors[ intf ][ 'symbol' ] = details[ 'symbol' ]
                    errors[ intf ][ 'symbol_count' ] = 0
                else:
                    trace( 'SYMBOL counters for %s OK' % intf )
                    errors[ intf ][ 'symbol_count' ] = 0
                    
                if errors[ intf ][ 'symbol_count' ] == FAIL_INTERVAL_COUNT:
                    log_symbol_failure( intf )
                    eapi.turnOffInterface( intf )

        interface_counters = counters
        for intf in [ x for x in errors if x not in counters ]:
            del errors[ intf ]

        time.sleep(EAPI_POLL_INTERVAL)

debug = None
email_active = None
def main():
    global debug
    global email_active

    setProcName( 'phm' )

    # Create help string and parse cmd line
    usage = 'usage: %prog [options]'
    op = optparse.OptionParser(usage=usage)
    op.add_option( '-d', '--debug', dest='debug', action='store_true',
                   help='print debug info' )
    op.add_option( '-e', '--email', dest='email', action='store_true',
                   help='sends email alerts' )
    opts, _ = op.parse_args()
    
    debug = opts.debug
    email_active = opts.email
    
    syslog.openlog( 'phm', 0, syslog.LOG_LOCAL4 )

    checkInterfaces()

if __name__ == '__main__':
   main()