feat: progress bar for long running operations in cli (#130)

- reorganized commands to follow same convention across cli - added spinner and verbose flags in deploy and validate steps for better UX - `optimus validate job` command is changed to `optimus job validate` Signed-off-by: Kush <[email protected]>
raystack · Jan 5, 2022 · 7645d51 · 7645d51
1 parent 52860dd
commit 7645d51
Show file tree

Hide file tree

Showing 43 changed files with 1,685 additions and 1,374 deletions.
diff --git a/README.md b/README.md
@@ -32,26 +32,28 @@ Optimus has two components, Optimus service that is the core orchestrator instal
 $ brew install odpf/taps/optimus
 $ optimus --help
 
-optimus v0.0.2-alpha.1
-
-optimus is a scaffolding tool for creating transformation job specs
+Optimus is an easy-to-use, reliable, and performant workflow orchestrator for
+data transformation, data modeling, pipelines, and data quality management.
 
 Usage:
   optimus [command]
 
 Available Commands:
+  backup      Backup a resource and its downstream
+  completion  generate the autocompletion script for the specified shell
   config      Manage optimus configuration required to deploy specifications
-  create      Create a new job/resource
-  deploy      Deploy current project to server
+  deploy      Deploy current optimus project to server
+  extension   Operate with extension
   help        Help about any command
-  render      convert raw representation of specification to consumables
-  replay      re-running jobs in order to update data for older dates/partitions
+  job         Interact with schedulable Job
+  replay      Re-running jobs in order to update data for older dates/partitions
+  resource    Interact with data resource
   serve       Starts optimus service
   version     Print the client version information
 
 Flags:
   -h, --help       help for optimus
-      --no-color   disable colored output
+      --no-color   Disable colored output
 
 Additional help topics:
   optimus validate check if specifications are valid for deployment
@@ -80,7 +82,7 @@ Run the following commands to compile `optimus` from source
 ```shell
 $ git clone [email protected]:odpf/optimus.git
 $ cd optimus
-$ make build
+$ make
 ```
 
 Use the following command to run
@@ -96,7 +98,7 @@ $ ./optimus serve
 ```
 
 `serve` command has few required configurations that needs to be set for it to start. Configuration can either be stored
-in `.optimus.yaml` file or set as environment variable. Read more about it in [getting started](https://odpf.github.io/optimus/getting-started/configuration/).
+in `.optimus.yaml` file or set as environment variable. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration).
 
 ## Compatibility
 
@@ -106,7 +108,7 @@ Optimus is currently undergoing heavy development with frequent, breaking API ch
 
 Development of Optimus happens in the open on GitHub, and we are grateful to the community for contributing bugfixes and improvements. Read below to learn how you can take part in improving Optimus.
 
-Read our [contributing guide](https://odpf.github.io/optimus/contribute/contributing/) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Optimus.
+Read our [contributing guide](https://odpf.github.io/optimus/docs/contribute/contributing) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Optimus.
 
 To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/odpf/optimus/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started.
 

diff --git a/cmd/admin.go b/cmd/admin.go
@@ -1,37 +1,28 @@
 package cmd
 
 import (
-	"github.com/odpf/optimus/models"
+	"github.com/odpf/optimus/config"
 	"github.com/odpf/salt/log"
 	cli "github.com/spf13/cobra"
 )
 
-// adminCommand requests a resource from optimus
-func adminCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command {
+// adminCommand registers internal administration commands
+func adminCommand(l log.Logger, conf config.Provider) *cli.Command {
 	cmd := &cli.Command{
-		Use:   "admin",
-		Short: "administration commands, should not be used by user",
+		Use:    "admin",
+		Short:  "Internal administration commands",
+		Hidden: true,
 	}
-	cmd.AddCommand(adminBuildCommand(l))
-	cmd.AddCommand(adminGetCommand(l, pluginRepo))
+	cmd.AddCommand(adminBuildCommand(l, conf))
 	return cmd
 }
 
-// adminBuildCommand builds a resource
-func adminBuildCommand(l log.Logger) *cli.Command {
+// adminBuildCommand builds a run instance
+func adminBuildCommand(l log.Logger, conf config.Provider) *cli.Command {
 	cmd := &cli.Command{
 		Use:   "build",
 		Short: "Register a job run and get required assets",
 	}
-	cmd.AddCommand(adminBuildInstanceCommand(l))
-	return cmd
-}
-
-// adminGetCommand gets a resource
-func adminGetCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command {
-	cmd := &cli.Command{
-		Use: "get",
-	}
-	cmd.AddCommand(adminGetStatusCommand(l))
+	cmd.AddCommand(adminBuildInstanceCommand(l, conf))
 	return cmd
 }
diff --git a/cmd/admin_build_instance.go b/cmd/admin_build_instance.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/odpf/optimus/config"
+
 	pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1"
 	"github.com/odpf/optimus/models"
 	"github.com/odpf/optimus/utils"
@@ -19,52 +21,46 @@ import (
 )
 
 var (
-	taskInputDirectory = "in"
-
+	taskInputDirectory        = "in"
 	adminBuildInstanceTimeout = time.Minute * 1
 )
 
-func adminBuildInstanceCommand(l log.Logger) *cli.Command {
+func adminBuildInstanceCommand(l log.Logger, conf config.Provider) *cli.Command {
 	var (
-		optimusHost    string
-		projectName    string
-		assetOutputDir string
-		scheduledAt    string
-		runType        string
+		optimusHost    = conf.GetHost()
+		projectName    = conf.GetProject().Name
+		assetOutputDir = "/tmp/"
+		runType        = "task"
 		runName        string
+		scheduledAt    string
+		cmd            = &cli.Command{
+			Use:     "instance",
+			Short:   "Builds a Job instance including the assets for a scheduled execution",
+			Example: "optimus admin build instance <sample_replace> --output-dir </tmp> --scheduled-at <2021-01-14T02:00:00+00:00> --type task --name <bq2bq> [--project \"project-id\"]",
+			Args:    cli.MinimumNArgs(1),
+		}
 	)
-	cmd := &cli.Command{
-		Use:     "instance",
-		Short:   "Builds a Job instance including the assets for a scheduled execution",
-		Example: "optimus admin build instance sample_replace --project \"project-id\" --output-dir /tmp",
-		Args:    cli.MinimumNArgs(1),
-	}
-
-	cmd.Flags().StringVar(&assetOutputDir, "output-dir", "", "output directory for assets")
+	cmd.Flags().StringVar(&assetOutputDir, "output-dir", assetOutputDir, "Output directory for assets")
 	cmd.MarkFlagRequired("output-dir")
-	cmd.Flags().StringVar(&scheduledAt, "scheduled-at", "", "time at which the job was scheduled for execution")
+	cmd.Flags().StringVar(&scheduledAt, "scheduled-at", "", "Time at which the job was scheduled for execution")
 	cmd.MarkFlagRequired("scheduled-at")
-	cmd.Flags().StringVar(&runType, "type", "", "type of task, could be base/hook")
+	cmd.Flags().StringVar(&runType, "type", "task", "Type of instance, could be task/hook")
 	cmd.MarkFlagRequired("type")
-	cmd.Flags().StringVar(&runName, "name", "", "name of task, could be bq2bq/transporter/predator")
+	cmd.Flags().StringVar(&runName, "name", "", "Name of running instance, e.g., bq2bq/transporter/predator")
 	cmd.MarkFlagRequired("name")
 
-	cmd.Flags().StringVar(&projectName, "project", "", "name of the tenant")
-	cmd.MarkFlagRequired("project")
-	cmd.Flags().StringVar(&optimusHost, "host", "", "optimus service endpoint url")
-	cmd.MarkFlagRequired("host")
+	cmd.Flags().StringVarP(&projectName, "project", "p", projectName, "Name of the optimus project")
+	cmd.Flags().StringVar(&optimusHost, "host", optimusHost, "Optimus service endpoint url")
 
 	cmd.RunE = func(c *cli.Command, args []string) error {
 		jobName := args[0]
-		l.Info(fmt.Sprintf("requesting resources for project %s, job %s at %s\nplease wait...", projectName, jobName, optimusHost))
-		l.Info(fmt.Sprintf("run name %s, run type %s, scheduled at %s", runName, runType, scheduledAt))
+		l.Info(fmt.Sprintf("Requesting resources for project %s, job %s at %s", projectName, jobName, optimusHost))
+		l.Info(fmt.Sprintf("Run name %s, run type %s, scheduled at %s\n", runName, runType, scheduledAt))
+		l.Info("please wait...")
+
 		// append base path to input file directory
 		inputDirectory := filepath.Join(assetOutputDir, taskInputDirectory)
-
-		if err := getInstanceBuildRequest(l, jobName, inputDirectory, optimusHost, projectName, scheduledAt, runType, runName); err != nil {
-			return err
-		}
-		return nil
+		return getInstanceBuildRequest(l, jobName, inputDirectory, optimusHost, projectName, scheduledAt, runType, runName)
 	}
 	return cmd
 }
@@ -85,7 +81,7 @@ func getInstanceBuildRequest(l log.Logger, jobName, inputDirectory, host, projec
 	var conn *grpc.ClientConn
 	if conn, err = createConnection(dialTimeoutCtx, host); err != nil {
 		if errors.Is(err, context.DeadlineExceeded) {
-			l.Info("can't reach optimus service, timing out")
+			l.Error(ErrServerNotReachable(host).Error())
 		}
 		return err
 	}