Skip to content

Commit

Permalink
Added new metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@aroraharsh23
aroraharsh23 committed Oct 15, 2019
1 parent 71157cb commit 7d785a1
Show file tree
Hide file tree
Showing 7 changed files with 952 additions and 392 deletions.
66 changes: 32 additions & 34 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,15 @@ The user can then access the exported metrics directly thorugh port 8888 on the
<summary>Usage as a Container</summary>
<br>

In order to use the exporter as a container, the image ```quay.io/citrix/citrix-adc-metrics-exporter:1.2``` will need to be pulled using;
In order to use the exporter as a container, the image ```quay.io/citrix/citrix-adc-metrics-exporter:1.3``` will need to be pulled using;
```
docker pull quay.io/citrix/citrix-adc-metrics-exporter:1.2
docker pull quay.io/citrix/citrix-adc-metrics-exporter:1.3
```
**NOTE:** It can also be build locally using ```docker build -f Dockerfile -t <image_name>:<tag> ./```

Now, the exporter can be run using:
```
docker run -dt -p <host_port>:<container_port> quay.io/citrix/citrix-adc-metrics-exporter:1.2 [flags]
docker run -dt -p <host_port>:<container_port> quay.io/citrix/citrix-adc-metrics-exporter:1.3 [flags]
```
where the flags are:

Expand All @@ -102,7 +102,7 @@ flag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbs

To setup the exporter as given in the diagram, the following command can be used:
```
docker run -dt -p 8888:8888 --name citrix-adc-exporter quay.io/citrix/citrix-adc-metrics-exporter:1.2 --target-nsip=10.0.0.1:80 --target-nsip=10.0.0.2:80 --target-nsip=172.17.0.2:80 --port 8888
docker run -dt -p 8888:8888 --name citrix-adc-exporter quay.io/citrix/citrix-adc-metrics-exporter:1.3 --target-nsip=10.0.0.1:80 --target-nsip=10.0.0.2:80 --target-nsip=172.17.0.2:80 --port 8888
```
This directs the exporter container to scrape the 10.0.0.1, 10.0.0.2, and 172.17.0.2, IPs on port 80, and the expose the stats it collects on port 8888. The user can then access the exported metrics directly thorugh port 8888 on the machine where the exporter is running, or Prometheus and Grafana can be setup to view the exported metrics though their GUI.

Expand All @@ -128,7 +128,7 @@ metadata:
spec:
containers:
- name: exporter
image: quay.io/citrix/citrix-adc-metrics-exporter:1.2
image: quay.io/citrix/citrix-adc-metrics-exporter:1.3
args:
- "--target-nsip=10.0.0.1:80"
- "--target-nsip=10.0.0.2:80"
Expand Down Expand Up @@ -238,73 +238,71 @@ For example, to export ```aaa``` stats, the lines given between ```-.-.-.-``` c
{
"system": {
"counters": [
["numcpus", "cpu_number"]
["numcpus", "citrixadc_cpu_number"]
],
"gauges": [
["cpuusagepcnt", "cpu_usage_percent"],
["mgmtcpuusagepcnt", "cpu_management_cpu_usage_percent"],
["pktcpuusagepcnt", "cpu_packet_cpu_usage_percent"],
["rescpuusagepcnt", "cpu_res_cpu_usage_percent"]
["cpuusagepcnt", "citrixadc_cpu_usage_percent"],
["mgmtcpuusagepcnt", "citrixadc_cpu_management_cpu_usage_percent"],
["pktcpuusagepcnt", "citrixadc_cpu_packet_cpu_usage_percent"],
["rescpuusagepcnt", "citrixadc_cpu_res_cpu_usage_percent"]
]
},
-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.
"aaa": {
"counters": [
["aaatotsessions", "aaa_tot_sessions"],
["aaatotsessiontimeout", "aaa_tot_session_timeout"]
["aaatotsessions", "citrixadc_aaa_tot_sessions"],
["aaatotsessiontimeout", "citrixadc_aaa_tot_session_timeout"]
],
"gauges": [
["aaasessionsrate', 'aaa_sessions_rate"],
["aaasessiontimeoutrate ', 'aaa_session_timeout_rate"]
["aaasessionsrate', 'citrixadc_aaa_sessions_rate"],
["aaasessiontimeoutrate ', 'citrixadc_aaa_session_timeout_rate"]
]
},
-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.
"protocolhttp": {
"counters": [
["httptotrequests", "http_tot_requests"],
["httptotresponses", "http_tot_responses"],
["httptotposts", "http_tot_posts"],
["httptotgets", "http_tot_gets"],
["httptotrequests", "citrixadc_http_tot_requests"],
["httptotresponses", "citrixadc_http_tot_responses"],
["httptotposts", "citrixadc_http_tot_posts"],
["httptotgets", "citrixadc_http_tot_gets"],
...
...
["httptotchunkedrequests", "http_tot_chunked_requests"]
["httptotchunkedrequests", "citrixadc_http_tot_chunked_requests"]
],
"gauges": [
["httprequestsrate", "http_requests_rate"],
["spdystreamsrate", "http_spdy_streams_rate"],
["httprequestsrate", "citrixadc_http_requests_rate"],
["spdystreamsrate", "citrixadc_http_spdy_streams_rate"],
...
...
["http11responsesrate", "http_11_responses_rate"]
["http11responsesrate", "citrixadc_http_11_responses_rate"]
]
},
"lbvserver": {
"counters": [
["totalpktssent", "lb_packets_sent_total"],
["tothits", "lb_hits_total"],
["totalrequestbytes", "lb_request_bytes_total"],
["totalpktssent", "citrixadc_lb_packets_sent_total"],
["tothits", "citrixadc_lb_hits_total"],
["totalrequestbytes", "citrixadc_lb_request_bytes_total"],
...
...
["totalresponsebytes", "lb_response_bytes_received_total"]
["totalresponsebytes", "citrixadc_lb_response_bytes_received_total"]
],
"gauges": [
["requestbytesrate", "lb_request_rate_bytes"],
["requestsrate", "lb_request_rate"],
["requestbytesrate", "citrixadc_lb_request_rate_bytes"],
["requestsrate", "citrixadc_lb_request_rate"],
...
...
["inactsvcs", "lb_inactive_services_count"]
["inactsvcs", "citrixadc_lb_inactive_services_count"]
],
["name", "lb_ingress_name"],
["type", "lb_type"],
["name", "lb_service_name"],
["name", "lb_name"]
["name", "citrixadc_lb_name"],
["type", "citrixadc_lb_type"],
]
},
Expand Down Expand Up @@ -444,7 +442,7 @@ The steps bellow can be followed to setup up a Grafana container with a sample d
<img src="images/grafana-datasource-1.png" width="200">
<img src="images/grafana-datasource-2.png" width="300">

5. Usage of Dashboard: For K8s CIC set up(providing K8sCICPrefix), with "k8s_ingress_services_stats.json",dashboard shows service requests, surque queue, RPS and Invalid Request/Response. User can select any of the configured services and then apply filter for a given ingress from the drop down menu for ingress. And with "sample_system_stats.json", dashboard shows CPU utilization, Memory Utilization and bandwidth capacity utilization where user can also set an alert. The dashboard can be expanded to include graphs of any other stats which the exporter is collecting. For more information on modifying the Grafana dashboard, please take a look at their [documentation](http://docs.grafana.org/) or demo [videos](https://www.youtube.com/watch?v=mgcJPREl3CU).
5. Usage of Dashboard: For K8s CIC set up(providing K8sCICPrefix), with "k8s_cic_ingress_services_stats.json",dashboard shows service requests, surque queue, RPS and Invalid Request/Response. User can select any of the configured ingress and then apply filter for a given ingress from the drop down menu for ingress.Fro non-k8s setup, one can use "sample_lb_stats.json" for all lbvservers configured on ADC for some counters. And with "sample_system_stats.json", dashboard shows CPU utilization, Memory Utilization and bandwidth capacity utilization where user can also set an alert. The dashboard can be expanded to include graphs of any other stats which the exporter is collecting. For more information on modifying the Grafana dashboard, please take a look at their [documentation](http://docs.grafana.org/) or demo [videos](https://www.youtube.com/watch?v=mgcJPREl3CU).

<img src="images/k8s-service-stats-dashboard.png" width="400"> <img src="images/system-stats-dashboard.png" width="400">

Expand Down
80 changes: 56 additions & 24 deletions exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def validate_ns_session(protocol, nsip, username, password):
try:
response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
if (response.status_code == requests.status_codes.codes.unauthorized):
logger.error('Invalid username or password!, Unaurthorized Err : {}'.format(response.status_code))
logger.error('Invalid username or password for Citrix Adc!, Unaurthorized Err : {}'.format(response.status_code))
return False
except requests.exceptions.RequestException as err:
logger.error('{}'.format(err))
Expand All @@ -44,6 +44,8 @@ def collect_data(nsip, entity, username, password, protocol, nitro_timeout):

# Login credentials
headers = {'X-NITRO-USER': username, 'X-NITRO-PASS': password}

# nitro call for all entities lbvserver bindings
if (entity == 'lbvserver_binding'):
url_lbvserver = '%s://%s/nitro/v1/config/lbvserver' % (protocol, nsip)

Expand All @@ -68,7 +70,7 @@ def collect_data(nsip, entity, username, password, protocol, nitro_timeout):
else:
total_down += 1
if total != 0:
percentup = (total_up/total) * 100
percentup = (total_up/float(total)) * 100
else:
percentup = 0

Expand All @@ -78,6 +80,10 @@ def collect_data(nsip, entity, username, password, protocol, nitro_timeout):
data = json.loads(teste)
return data['lbvserver_binding']

# this is to fetch lb status for ingress/services in k8s enviroment
if(entity == 'k8s_ingress_lbvs'):
entity = 'lbvserver'

# nitro call for all entities except 'services' (ie. servicegroups)
if (entity != 'services'):
if(entity != 'nscapacity' and entity != 'sslcertkey'):
Expand Down Expand Up @@ -113,18 +119,37 @@ def collect_data(nsip, entity, username, password, protocol, nitro_timeout):


def update_lbvs_label(k8s_cic_prefix, label_values, ns_metric_name, log_prefix_match):
'''Updates lbvserver lables for ingress and services'''
if (str(label_values).find("_svc") != -1):
cur_prefix = str(label_values[0].split("_")[0].split("-", 1)[0])
if cur_prefix == k8s_cic_prefix:
label_values[0] = label_values[0].split("_")[0].split("-", 1)[1]
label_values[2] = label_values[2].split("_")[3].split("-", 1)[1]
return True
'''Updates lbvserver lables for ingress and services for k8s_cic_ingress_service_stat dashboard'''
try:
# If lbvs name ends with expected _svc, then label values are updated with ingress/service info.
if (str(label_values).find("_svc") != -1):
cur_prefix = str(label_values[0].split("_")[0].split("-", 1)[0])
# update lables only if prefix provided is same as CIC prefix used
if cur_prefix == k8s_cic_prefix:
# return if ingress name as a service
if label_values[0].split("_")[3] == 'svc':
if log_prefix_match:
logger.info('k8s_ingress_service_stat Ingress dashboard cannot be used without ingress')
return False
# update label "citrixadc_k8s_ing_lb_ingress_name" with ingress name
label_values[0] = label_values[0].split("_")[0].split("-", 1)[1]
# update label "citrixadc_k8s_ing_lb_ingress_port" with ingress port
label_values[1] = label_values[1].split("_")[2]
# update label "citrixadc_k8s_ing_lb_service_name" with service name
label_values[2] = label_values[2].split("_")[3].split("-", 1)[1]
# update label "citrixadc_k8s_ing_lb_ingress_port" with service port
label_values[3] = label_values[3].split("_")[5]
return True
else:
if log_prefix_match:
logger.info('k8s_cic_ingress_service_stat Ingress dashboard cannot be used for CIC prefix "%s"', cur_prefix)
return False
else:
if log_prefix_match:
logger.info('k8s_ingress_service_stat Ingress dashboard cannot be used for CIC prefix "%s"', cur_prefix)
logger.info('k8s_cic_ingress_service_stat Ingress dashboard cannot be used for non-CIC(or < CIC 1.2) ingress/lb"',)
return False
else:
except Exception as e:
logger.error('Unable to update k8s label: (%s)', e)
return False


Expand Down Expand Up @@ -174,17 +199,21 @@ def collect(self):
continue

if ns_metric_name not in data_item.keys():
logger.warning('Counter stats for %s not enabled in netscalar %s, so could not add to %s' % (ns_metric_name, nsip, entity_name))
logger.warning('Counter stats for %s not enabled in adc %s, so could not add to %s' % (ns_metric_name, nsip, entity_name))
break

if('labels' in entity.keys()):
label_values = [data_item[key] for key in [v[0] for v in entity['labels']]]
if os.environ.get('KUBERNETES_SERVICE_HOST') is not None:
if entity_name == "lbvserver":

# populate and update k8s_ingress_lbvs metrics if in k8s-CIC enviroment
if entity_name == "k8s_ingress_lbvs":
if os.environ.get('KUBERNETES_SERVICE_HOST') is not None:
prefix_match = update_lbvs_label(self.k8s_cic_prefix, label_values, ns_metric_name, log_prefix_match)
if not prefix_match:
log_prefix_match = False

continue
else:
continue
label_values.append(nsip)
else:
label_values = [nsip]
Expand All @@ -207,17 +236,21 @@ def collect(self):
if not data_item:
continue
if ns_metric_name not in data_item.keys():
logger.warning('Gauge stats for %s not enabled in netscalar %s, so could not add to %s' % (ns_metric_name, nsip, entity_name))
logger.warning('Gauge stats for %s not enabled in adc %s, so could not add to %s' % (ns_metric_name, nsip, entity_name))
break

if('labels' in entity.keys()):
label_values = [data_item[key] for key in [v[0] for v in entity['labels']]]

if os.environ.get('KUBERNETES_SERVICE_HOST') is not None:
if entity_name == "lbvserver":
# populate and update k8s_ingress_lbvs metrics if in k8s-CIC enviroment
if entity_name == "k8s_ingress_lbvs":
if os.environ.get('KUBERNETES_SERVICE_HOST') is not None:
prefix_match = update_lbvs_label(self.k8s_cic_prefix, label_values, ns_metric_name, log_prefix_match)
if not prefix_match:
log_prefix_match = False
continue
else:
continue

label_values.append(nsip)
else:
Expand Down Expand Up @@ -307,7 +340,7 @@ def collect(self):
except Exception as e:
logger.error('Error while loading metrics::%s', e)

# validating netscalar access
# validating ADC access
secure = args.secure.lower()
if secure == 'yes':
ns_protocol = 'https'
Expand All @@ -318,12 +351,12 @@ def collect(self):
for nsip in args.target_nsip:
res = validate_ns_session(ns_protocol, nsip, ns_user, ns_password)
if res is False:
logger.error('Exiting since NS access test failed for nsip {}'.format(nsip))
sys.exit()
logger.error('Citrix adc access test failed for ip {}'.format(nsip))
else:
logger.info('Exporter connected to adc {}'.format(nsip))

if not args.k8sCICprefix.isalnum():
logger.error('Invalid k8sCICprefix : non-alphanumeric not accepted')
sys.exit()

# Register the exporter as a stat collector
logger.info('Registering collector for %s' % args.target_nsip)
Expand All @@ -332,8 +365,7 @@ def collect(self):
REGISTRY.register(CitrixAdcCollector(nsips=args.target_nsip, metrics=metrics_json, username=ns_user,
password=ns_password, protocol=ns_protocol, nitro_timeout=args.timeout, k8s_cic_prefix=args.k8sCICprefix))
except Exception as e:
logger.error('Exiting: invalid arguments! could not register collector for {}::{}'.format(args.target_nsip, e))
sys.exit()
logger.error('Exiting: Invalid arguments! could not register collector for {}::{}'.format(args.target_nsip, e))

# Forever
while True:
Expand Down
Loading

0 comments on commit 7d785a1

Please sign in to comment.