Skip to content
This repository has been archived by the owner on Mar 27, 2018. It is now read-only.

Commit

Permalink
Merge pull request #6 from oscp/feature/healthy-status
Browse files Browse the repository at this point in the history 
The cli now generates a HEALTHY event if everything is OK.
  • Loading branch information
@ReToCode
ReToCode authored Mar 13, 2018
2 parents 8f1cd63 + 2f32d2c commit aebd8fd
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 112 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# dirs
out/

# Binaries for programs and plugins
*.exe
*.dll
Expand Down
171 changes: 60 additions & 111 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright © 2017 Maciej Raciborski <[email protected]>
// Copyright © 2017 SBB Cloud Stack Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -21,15 +21,16 @@ import (
"path/filepath"
"runtime"
"strings"
"errors"

"github.com/op/go-logging"
"github.com/oscp/openshift-monitoring-checks/checks"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

var pretty bool
var debug bool
var pretty bool
var debug bool

var log = logging.MustGetLogger("openshift-monitoring-cli")

Expand All @@ -44,6 +45,13 @@ type IntegrationData struct {
Events []EventData `json:"events"`
}

var data = IntegrationData{
Name: "ch.sbb.openshift-integration",
ProtocolVersion: "1",
IntegrationVersion: "1.0.0",
Events: make([]EventData, 0),
}

var rootCmd = &cobra.Command{
Use: "openshift-monitoring-cli",
Short: "This cli tool runs monitoring checks for OpenShift installations.",
Expand Down Expand Up @@ -120,30 +128,34 @@ func createEvent(err error) map[string]interface{} {
return event
}

func createMajorEvent(err error) map[string]interface{} {
func createHealthyEvent(err error) map[string]interface{} {
var event = createEvent(err)
event["category"] = "MAJOR"
log.Error("MAJOR:", err.Error())
event["category"] = "HEALTHY"
log.Error("HEALTHY:", err.Error())
return event
}

func createMinorEvent(err error) map[string]interface{} {
var event = createEvent(err)
event["category"] = "MINOR"
log.Error("MINOR:", err.Error())
return event
func evalMajor(fn func() error) {
if err := fn(); err != nil {
var event= createEvent(err)
event["category"] = "MAJOR"
log.Error("MAJOR:", err.Error())
data.Events = append(data.Events, event)
}
}

func evalMinor(fn func() error) {
if err := fn(); err != nil {
var event = createEvent(err)
event["category"] = "MINOR"
log.Error("MINOR:", err.Error())
data.Events = append(data.Events, event)
}
}

func runChecks(cmd *cobra.Command, args []string) {
log.Info("Running", viper.GetString("node.type"), "checks for OpenShift.")

var data = IntegrationData{
Name: "ch.sbb.openshift-integration",
ProtocolVersion: "1",
IntegrationVersion: "1.0.0",
Events: make([]EventData, 0),
}

if viper.GetString("node.type") == "master" {
if len(viper.GetString("etcd.ips")) == 0 || len(viper.GetString("router.ips")) == 0 {
log.Fatal("Can't read service IPs from configuration file.")
Expand All @@ -158,75 +170,40 @@ func runChecks(cmd *cobra.Command, args []string) {
// majors on storage
if viper.GetString("node.type") == "storage" {
log.Debug("Running major checks for storage.")
if err := checks.CheckIfGlusterdIsRunning(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckMountPointSizes(90); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckLVPoolSizes(90); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckVGSizes(5); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
evalMajor(func() error { return checks.CheckIfGlusterdIsRunning() })
evalMajor(func() error { return checks.CheckMountPointSizes(90) })
evalMajor(func() error { return checks.CheckLVPoolSizes(90) })
evalMajor(func() error { return checks.CheckVGSizes(5) })
}

// majors on node
if viper.GetString("node.type") == "node" {
log.Debug("Running major checks for node.")

if err := checks.CheckDockerPool(90); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckDnsNslookupOnKubernetes(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckDnsServiceNode(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
evalMajor(func() error { return checks.CheckDockerPool(90) })
evalMajor(func() error { return checks.CheckDnsNslookupOnKubernetes() })
evalMajor(func() error { return checks.CheckDnsServiceNode() })
}

// majors on master
if viper.GetString("node.type") == "master" {
log.Debug("Running major checks for master.")

if err := checks.CheckOcGetNodes(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckEtcdHealth(viper.GetString("etcd.ips"), ""); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
evalMajor(func() error { return checks.CheckOcGetNodes() })
evalMajor(func() error { return checks.CheckEtcdHealth(viper.GetString("etcd.ips"), "") })

if len(viper.GetString("registry.ip")) > 0 {
if err := checks.CheckRegistryHealth(viper.GetString("registry.ip")); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
evalMajor(func() error { return checks.CheckRegistryHealth(viper.GetString("registry.ip")) })
}

for _, rip := range strings.Split(viper.GetString("router.ips"), ",") {
if err := checks.CheckRouterHealth(rip); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
}

if err := checks.CheckMasterApis("https://localhost:8443/api"); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
evalMajor(func() error { return checks.CheckRouterHealth(rip) })
}

if err := checks.CheckDnsNslookupOnKubernetes(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}

if err := checks.CheckDnsServiceNode(); err != nil {
data.Events = append(data.Events, createMajorEvent(err))
}
evalMajor(func() error { return checks.CheckMasterApis("https://localhost:8443/api") })
evalMajor(func() error { return checks.CheckDnsNslookupOnKubernetes() })
evalMajor(func() error { return checks.CheckDnsServiceNode() })
}

/////////////////
Expand All @@ -237,67 +214,39 @@ func runChecks(cmd *cobra.Command, args []string) {
// minors on storage
if viper.GetString("node.type") == "storage" {
log.Debug("Running minor checks for storage.")
if err := checks.CheckOpenFileCount(); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckMountPointSizes(85); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckLVPoolSizes(80); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckVGSizes(10); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}
evalMinor(func() error { return checks.CheckOpenFileCount() })
evalMinor(func() error { return checks.CheckMountPointSizes(85) })
evalMinor(func() error { return checks.CheckLVPoolSizes(80) })
evalMinor(func() error { return checks.CheckVGSizes(10) })
}

// minors on node
if viper.GetString("node.type") == "node" {
log.Debug("Running minor checks for node.")
if err := checks.CheckDockerPool(80); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckHttpService(false); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}
evalMinor(func() error { return checks.CheckDockerPool(80) })
evalMinor(func() error { return checks.CheckHttpService(false) })
}

// minors on master
if viper.GetString("node.type") == "master" {
log.Debug("Running minor checks for master.")
if err := checks.CheckExternalSystem(viper.GetString("externalSystemUrl")); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckHawcularHealth(viper.GetString("hawcularIP")); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckRouterRestartCount(); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckLimitsAndQuotas(viper.GetInt("projectsWithoutLimits")); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckHttpService(false); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}

if err := checks.CheckLoggingRestartsCount(); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
}
evalMinor(func() error { return checks.CheckExternalSystem(viper.GetString("externalSystemUrl")) })
evalMinor(func() error { return checks.CheckHawcularHealth(viper.GetString("hawcularIP")) })
evalMinor(func() error { return checks.CheckRouterRestartCount() })
evalMinor(func() error { return checks.CheckLimitsAndQuotas(viper.GetInt("projectsWithoutLimits")) })
evalMinor(func() error { return checks.CheckHttpService(false) })
evalMinor(func() error { return checks.CheckLoggingRestartsCount() })
}

log.Debug("Running minor checks for all node types.")
// minor for all server types
if err := checks.CheckNtpd(); err != nil {
data.Events = append(data.Events, createMinorEvent(err))
evalMinor(func() error { return checks.CheckNtpd() })

if len(data.Events) == 0 {
data.Events = append(data.Events, createHealthyEvent(errors.New("System healthy, nothing to do.")));
}

OutputJSON(data)
Expand Down
2 changes: 1 addition & 1 deletion main.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright © 2017 Maciej Raciborski <[email protected]>
// Copyright © 2017 SBB Cloud Stack Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down

0 comments on commit aebd8fd

Please sign in to comment.