From 52e3ac0ccbb3875927323ee3d07cc3101e6c73a6 Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Tue, 1 Dec 2020 15:29:10 -0500 Subject: [PATCH 1/3] Updating plan --- command/deploy.go | 2 +- command/plan.go | 31 +++- levant/plan.go | 310 ++++++++++++++++++++++++++++++--------- levant/structs/config.go | 21 ++- 4 files changed, 286 insertions(+), 78 deletions(-) diff --git a/command/deploy.go b/command/deploy.go index c3817d226..91ae4f926 100644 --- a/command/deploy.go +++ b/command/deploy.go @@ -180,7 +180,7 @@ func (c *DeployCommand) Run(args []string) int { } if !config.Deploy.Force { - p := levant.PlanConfig{ + p := structs.LevantPlanConfig{ Client: config.Client, Plan: config.Plan, Template: config.Template, diff --git a/command/plan.go b/command/plan.go index caee6d907..200770f54 100644 --- a/command/plan.go +++ b/command/plan.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/levant/levant" "github.com/hashicorp/levant/levant/structs" "github.com/hashicorp/levant/logging" + "github.com/hashicorp/levant/output" "github.com/hashicorp/levant/template" ) @@ -48,7 +49,15 @@ General Options: -force-count Use the taskgroup count from the Nomad jobfile instead of the count that - is currently set in a running job. + is currently set in a running job. + + -output-format + Specify the format of plan to be emitted. Valid values include JSON, DIFF, + and NDJSON. The default is DIFF. + + -output-to + Specify the destination for the diff. Valid values include CLI, LOG, STDOUT, + and STDERR. The default is LOG. -ignore-no-changes By default if no changes are detected when running a plan Levant will @@ -81,11 +90,14 @@ func (c *PlanCommand) Synopsis() string { func (c *PlanCommand) Run(args []string) int { var err error - var level, format string - config := &levant.PlanConfig{ + var logLevel, logFormat, outFormat, outDest string + config := &structs.LevantPlanConfig{ Client: &structs.ClientConfig{}, Plan: &structs.PlanConfig{}, Template: &structs.TemplateConfig{}, + Output: &structs.DiffOutputConfig{ + UI: &c.UI, + }, } flags := c.Meta.FlagSet("plan", FlagSetVars) @@ -95,8 +107,10 @@ func (c *PlanCommand) Run(args []string) int { flags.BoolVar(&config.Client.AllowStale, "allow-stale", false, "") flags.StringVar(&config.Client.ConsulAddr, "consul-address", "", "") flags.BoolVar(&config.Plan.IgnoreNoChanges, "ignore-no-changes", false, "") - flags.StringVar(&level, "log-level", "INFO", "") - flags.StringVar(&format, "log-format", "HUMAN", "") + flags.StringVar(&logLevel, "log-level", "INFO", "") + flags.StringVar(&logFormat, "log-format", "HUMAN", "") + flags.StringVar(&outFormat, "output-format", "DIFF", "") + flags.StringVar(&outDest, "output-to", "CLI", "") flags.Var((*helper.FlagStringSlice)(&config.Template.VariableFiles), "var-file", "") if err = flags.Parse(args); err != nil { @@ -105,7 +119,12 @@ func (c *PlanCommand) Run(args []string) int { args = flags.Args() - if err = logging.SetupLogger(level, format); err != nil { + if err = logging.SetupLogger(logLevel, logFormat); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + if err = output.ConfigureOutputSettings(config, &outFormat, &outDest); err != nil { c.UI.Error(err.Error()) return 1 } diff --git a/levant/plan.go b/levant/plan.go index 3d4a5f1e0..712200a41 100644 --- a/levant/plan.go +++ b/levant/plan.go @@ -1,33 +1,43 @@ package levant import ( + "encoding/json" "fmt" + "os" + "strings" "github.com/hashicorp/levant/client" "github.com/hashicorp/levant/levant/structs" + "github.com/hashicorp/levant/output" nomad "github.com/hashicorp/nomad/api" "github.com/rs/zerolog/log" ) const ( - diffTypeAdded = "Added" - diffTypeEdited = "Edited" - diffTypeNone = "None" + diffTypeAdded = "Added" + diffTypeEdited = "Edited" + diffTypeDeleted = "Deleted" + diffTypeNone = "None" + + typeJob = "job" + typeTaskGroup = "taskgroup" + typeTask = "task" + typeObject = "object" + typeEndObject = "end_object" + typeField = "field" + + operatorAdded = " + " + operatorDeleted = " - " + operatorEdited = "+/-" + operatorNone = " " ) type levantPlan struct { nomad *nomad.Client - config *PlanConfig + config *structs.LevantPlanConfig } -// PlanConfig is the set of config structs required to run a Levant plan. -type PlanConfig struct { - Client *structs.ClientConfig - Plan *structs.PlanConfig - Template *structs.TemplateConfig -} - -func newPlan(config *PlanConfig) (*levantPlan, error) { +func newPlan(config *structs.LevantPlanConfig) (*levantPlan, error) { var err error @@ -42,7 +52,7 @@ func newPlan(config *PlanConfig) (*levantPlan, error) { } // TriggerPlan initiates a Levant plan run. -func TriggerPlan(config *PlanConfig) (bool, bool) { +func TriggerPlan(config *structs.LevantPlanConfig) (bool, bool) { lp, err := newPlan(config) if err != nil { @@ -97,87 +107,247 @@ func (lp *levantPlan) plan() (bool, error) { // If there are changes, run the planDiff function which is responsible for // iterating through the plan and logging all the planned changes. case diffTypeEdited: - planDiff(resp.Diff) + outputDiff(lp.config.Output, resp.Diff) } return true, nil } -func planDiff(plan *nomad.JobDiff) { +func outputDiff(config *structs.DiffOutputConfig, diff *nomad.JobDiff) { + var outFun func(string) - // Iterate through each TaskGroup. - for _, tg := range plan.TaskGroups { - if tg.Type != diffTypeEdited { - continue + switch *config.Destination { + case output.OutLog: + outFun = func(out string) { + log.Info().Msgf("levant/plan: %s", out) } - for _, tgo := range tg.Objects { - recurseObjDiff(tg.Name, "", tgo) + case output.OutCLI: + outFun = func(out string) { + (*config.UI).Output(out) } - - // Iterate through each Task. - for _, t := range tg.Tasks { - if t.Type != diffTypeEdited { - continue - } - if len(t.Objects) == 0 { - return - } - for _, o := range t.Objects { - recurseObjDiff(tg.Name, t.Name, o) - } + case output.OutSTDOUT: + outFun = func(out string) { + fmt.Println(out) } + case output.OutSTDERR: + outFun = func(out string) { + fmt.Fprintln(os.Stderr, out) + } + } + switch *config.Format { + case output.FmtJSON: + printDiffAsJSON(diff, outFun) + case output.FmtDiff: + parsedPlan := parsePlanJSON(diff) + printLinesAsText(parsedPlan, outFun) + case output.FmtNDJSON: + parsedPlan := parsePlanJSON(diff) + printLinesAsJSON(parsedPlan, outFun) } + return } -func recurseObjDiff(g, t string, objDiff *nomad.ObjectDiff) { +func printDiffAsJSON(diff *nomad.JobDiff, outFun func(string)) { + out, err := json.Marshal(diff) + if err != nil { + log.Error().Msgf("levant/plan: failed to marshall plan to JSON: %s", err.Error) + return + } + outFun(string(out)) + return +} - // If we have reached the end of the object tree, and have an edited type - // with field information then we can interate on the fields to find those - // which have changed. - if len(objDiff.Objects) == 0 && len(objDiff.Fields) > 0 && objDiff.Type == diffTypeEdited { - for _, f := range objDiff.Fields { - if f.Type != diffTypeEdited { - continue - } - logDiffObj(g, t, objDiff.Name, f.Name, f.Old, f.New) - continue - } +// JSONLine contains a single JSON element generated by the formatPlanJSON +// function. This forms the basis of the json log line and is converted to +// plaintext for some output options. This could also be converted to html +// or other output alternatives. +type JSONLine struct { + // The item that has changed + Item string `json:"item"` + // What sort of object that item is (job, taskgroup, task, object) + Type string `json:"type"` + // The depth, necessary for padding. The shallowest object (the job) is at depth=0 + Depth int `json:"depth"` + // The operation that is happening to the object: add, remove, etc. + Operation string `json:"operation"` + // Previous value of fields that are being edited or deleted + Before string `json:"before"` + // New value for fields being added or edited + After string `json:"after"` + // List of string annotations from the scheduler + Annotations []string `json:"annotations"` +} - } else { - // Continue to interate through the object diff objects until such time - // the above is triggered. - for _, o := range objDiff.Objects { - recurseObjDiff(g, t, o) - } +func createJSONLine(item string, inType string, depth int, oper string, before string, after string, ann []string) *JSONLine { + outJSON := &JSONLine{ + Item: item, + Type: inType, + Depth: depth, + Operation: oper, + Before: before, + After: after, + Annotations: ann, + } + return outJSON +} + +func createJobLine(item string, depth int, oper string) *JSONLine { + return createJSONLine(item, typeJob, depth, oper, "", "", nil) +} + +func createTaskgroupLine(item string, depth int, oper string, updates map[string]uint64) *JSONLine { + var ann []string + for updateType, count := range updates { + ann = append(ann, fmt.Sprintf("%d %s", count, updateType)) + } + return createJSONLine(item, typeTaskGroup, depth, oper, "", "", ann) +} + +func createTaskLine(item string, depth int, oper string, ann []string) *JSONLine { + return createJSONLine(item, typeTask, depth, oper, "", "", ann) +} + +func createObjectLine(item string, depth int, oper string) *JSONLine { + return createJSONLine(item, typeObject, depth, oper, "", "", nil) +} + +func createEndObjectLine(item string, depth int, oper string) *JSONLine { + return createJSONLine(item, typeEndObject, depth, oper, "", "", nil) +} + +func createFieldLine(item string, depth int, oper string, before string, after string, ann []string) *JSONLine { + return createJSONLine(item, typeField, depth, oper, before, after, ann) +} + +// parsePlanJSON builds a newline delimited JSON which can +// then be formatted before emitting to the log. +func parsePlanJSON(plan *nomad.JobDiff) []*JSONLine { + var lines []*JSONLine + var depth int = 0 + lines = append(lines, createJobLine(plan.ID, 0, plan.Type)) + depth = depth + 1 + lines = append(lines, parseFieldDiffs(plan.Fields, depth)...) + lines = append(lines, parseObjectDiffs(plan.Objects, depth)...) + for _, group := range plan.TaskGroups { + lines = append(lines, parseTaskGroupDiff(group, depth)...) } + return lines } -// logDiffObj is a helper function so Levant can log the most accurate and -// useful plan output messages. -func logDiffObj(g, t, objName, fName, fOld, fNew string) { +func parseTaskGroupDiff(tg *nomad.TaskGroupDiff, depth int) []*JSONLine { + var lines []*JSONLine - var lStart, l string + lines = append(lines, createTaskgroupLine(tg.Name, depth, tg.Type, tg.Updates)) + depth = depth + 1 + lines = append(lines, parseFieldDiffs(tg.Fields, depth)...) + lines = append(lines, parseObjectDiffs(tg.Objects, depth)...) + for _, task := range tg.Tasks { + lines = append(lines, parseTaskDiff(task, depth)...) + } + return lines +} - // We will always have at least this information to log. - lEnd := fmt.Sprintf("plan indicates change of %s:%s from %s to %s", - objName, fName, fOld, fNew) +func parseTaskDiff(task *nomad.TaskDiff, depth int) []*JSONLine { + var lines []*JSONLine + lines = append(lines, createTaskLine(task.Name, depth, task.Type, task.Annotations)) + depth = depth + 1 + lines = append(lines, parseFieldDiffs(task.Fields, depth)...) + lines = append(lines, parseObjectDiffs(task.Objects, depth)...) + return lines +} - // If we have been passed a group name, use this to start the log line. - if g != "" { - lStart = fmt.Sprintf("group %s ", g) +func parseObjectDiffs(objects []*nomad.ObjectDiff, depth int) []*JSONLine { + var lines []*JSONLine + + for _, object := range objects { + lines = append(lines, createObjectLine(object.Name, depth, object.Type)) + depth = depth + 1 + lines = append(lines, parseFieldDiffs(object.Fields, depth)...) + lines = append(lines, parseObjectDiffs(object.Objects, depth)...) + depth = depth - 1 + lines = append(lines, createEndObjectLine(object.Name, depth, object.Type)) } + return lines +} + +func parseFieldDiffs(fields []*nomad.FieldDiff, depth int) []*JSONLine { + var lines []*JSONLine - // If we have been passed a task name, append this to the group name. - if t != "" { - lStart = lStart + fmt.Sprintf("and task %s ", t) + for _, field := range fields { + lines = append(lines, createFieldLine(field.Name, depth, field.Type, field.Old, field.New, field.Annotations)) } + return lines +} + +func printLinesAsText(lines []*JSONLine, outFun func(string)) { + for _, line := range lines { + var operator, padding, body, annotations, out string + padding = strings.Repeat(" ", line.Depth) - // Build the final log message. - if lStart != "" { - l = lStart + lEnd - } else { - l = lEnd + if line.Annotations != nil { + annotations = " (" + strings.Join(line.Annotations, ",") + ")" + } + + switch line.Operation { + case diffTypeAdded: + operator = operatorAdded + body = fmt.Sprintf("%s", formatValue(line.After)) + case diffTypeDeleted: + operator = operatorDeleted + body = fmt.Sprintf("%s", formatValue(line.Before)) + case diffTypeEdited: + operator = operatorEdited + body = fmt.Sprintf("%s => %s", formatValue(line.Before), formatValue(line.After)) + case diffTypeNone: + operator = operatorNone + body = fmt.Sprintf("%s", formatValue(line.Before)) + default: + panic(fmt.Sprintf("reached default condition unexpectedly: %s", line.Type)) + } + + switch line.Type { + case typeJob: + out = fmt.Sprintf("%s%s Job \"%s\": %s", padding, operator, line.Item, annotations) + case typeTaskGroup: + out = fmt.Sprintf("%s%s Task Group \"%s\": %s", padding, operator, line.Item, annotations) + case typeTask: + out = fmt.Sprintf("%s%s Task \"%s\" %s", padding, operator, line.Item, annotations) + case typeObject: + out = fmt.Sprintf("%s%s %s {", padding, operator, line.Item) + case typeEndObject: + // override the incoming operator so the closing mark doesn't have + // a sign + operator = operatorNone + out = fmt.Sprintf("%s%s }", padding, operator) + case typeField: + out = fmt.Sprintf("%s%s %s: %s", padding, operator, line.Item, body) + default: + panic(fmt.Sprintf("Unexpected line type: %s", line.Type)) + } + + outFun(out) + } +} + +func printLinesAsJSON(lines []*JSONLine, outFun func(string)) { + var out string + for _, line := range lines { + switch line.Type { + case typeEndObject: + continue + default: + out = formatValue(line) + } + outFun(out) } +} - log.Info().Msgf("levant/plan: %s", l) +// formatValue formats the incoming value as JSON. If that isn't +// possible, it returns error text as the value +func formatValue(in interface{}) string { + out, err := json.Marshal(in) + if err == nil { + return string(out) + } + return fmt.Sprintf("failed to marshal. err: %s", err.Error()) } diff --git a/levant/structs/config.go b/levant/structs/config.go index dba4fde32..375cccd80 100644 --- a/levant/structs/config.go +++ b/levant/structs/config.go @@ -1,6 +1,9 @@ package structs -import nomad "github.com/hashicorp/nomad/api" +import ( + nomad "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" +) const ( // JobIDContextField is the logging context feild added when interacting @@ -70,6 +73,22 @@ type PlanConfig struct { IgnoreNoChanges bool } +// LevantPlanConfig is the set of config structs required to run a Levant plan. +type LevantPlanConfig struct { + Client *ClientConfig + Plan *PlanConfig + Template *TemplateConfig + Output *DiffOutputConfig +} + +// DiffOutputConfig controls how the output is returned to the user for +// a Levant plan +type DiffOutputConfig struct { + Format *string + Destination *string + UI *cli.Ui +} + // TemplateConfig contains all the job templating configuration options including // the rendered job. type TemplateConfig struct { From 7ef13feebff8c6d97a86ba705f16cec057d72aab Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Tue, 1 Dec 2020 17:02:24 -0500 Subject: [PATCH 2/3] Fixes for linter --- levant/plan.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/levant/plan.go b/levant/plan.go index 712200a41..c000b0536 100644 --- a/levant/plan.go +++ b/levant/plan.go @@ -150,7 +150,7 @@ func outputDiff(config *structs.DiffOutputConfig, diff *nomad.JobDiff) { func printDiffAsJSON(diff *nomad.JobDiff, outFun func(string)) { out, err := json.Marshal(diff) if err != nil { - log.Error().Msgf("levant/plan: failed to marshall plan to JSON: %s", err.Error) + log.Error().Msgf("levant/plan: failed to marshall plan to JSON: %s", err.Error()) return } outFun(string(out)) @@ -196,7 +196,8 @@ func createJobLine(item string, depth int, oper string) *JSONLine { } func createTaskgroupLine(item string, depth int, oper string, updates map[string]uint64) *JSONLine { - var ann []string + ann := make([]string, 0, 5) + for updateType, count := range updates { ann = append(ann, fmt.Sprintf("%d %s", count, updateType)) } @@ -222,7 +223,7 @@ func createFieldLine(item string, depth int, oper string, before string, after s // parsePlanJSON builds a newline delimited JSON which can // then be formatted before emitting to the log. func parsePlanJSON(plan *nomad.JobDiff) []*JSONLine { - var lines []*JSONLine + lines := make([]*JSONLine, 0, 100) var depth int = 0 lines = append(lines, createJobLine(plan.ID, 0, plan.Type)) depth = depth + 1 @@ -235,7 +236,7 @@ func parsePlanJSON(plan *nomad.JobDiff) []*JSONLine { } func parseTaskGroupDiff(tg *nomad.TaskGroupDiff, depth int) []*JSONLine { - var lines []*JSONLine + lines := make([]*JSONLine, 0, 100) lines = append(lines, createTaskgroupLine(tg.Name, depth, tg.Type, tg.Updates)) depth = depth + 1 @@ -248,7 +249,7 @@ func parseTaskGroupDiff(tg *nomad.TaskGroupDiff, depth int) []*JSONLine { } func parseTaskDiff(task *nomad.TaskDiff, depth int) []*JSONLine { - var lines []*JSONLine + lines := make([]*JSONLine, 0, 100) lines = append(lines, createTaskLine(task.Name, depth, task.Type, task.Annotations)) depth = depth + 1 lines = append(lines, parseFieldDiffs(task.Fields, depth)...) @@ -257,7 +258,7 @@ func parseTaskDiff(task *nomad.TaskDiff, depth int) []*JSONLine { } func parseObjectDiffs(objects []*nomad.ObjectDiff, depth int) []*JSONLine { - var lines []*JSONLine + lines := make([]*JSONLine, 0, 100) for _, object := range objects { lines = append(lines, createObjectLine(object.Name, depth, object.Type)) @@ -271,7 +272,7 @@ func parseObjectDiffs(objects []*nomad.ObjectDiff, depth int) []*JSONLine { } func parseFieldDiffs(fields []*nomad.FieldDiff, depth int) []*JSONLine { - var lines []*JSONLine + lines := make([]*JSONLine, 0, 20) for _, field := range fields { lines = append(lines, createFieldLine(field.Name, depth, field.Type, field.Old, field.New, field.Annotations)) From 34ff90bba00b5f51c601719fb4b2020997f0f1ba Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Tue, 1 Dec 2020 17:20:59 -0500 Subject: [PATCH 3/3] Output to log by default --- command/plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/plan.go b/command/plan.go index 200770f54..705140c4a 100644 --- a/command/plan.go +++ b/command/plan.go @@ -110,7 +110,7 @@ func (c *PlanCommand) Run(args []string) int { flags.StringVar(&logLevel, "log-level", "INFO", "") flags.StringVar(&logFormat, "log-format", "HUMAN", "") flags.StringVar(&outFormat, "output-format", "DIFF", "") - flags.StringVar(&outDest, "output-to", "CLI", "") + flags.StringVar(&outDest, "output-to", "LOG", "") flags.Var((*helper.FlagStringSlice)(&config.Template.VariableFiles), "var-file", "") if err = flags.Parse(args); err != nil {