Merge pull request #470 from bruin-data/patch/reduce-response-size-fo…

…r-internal-parse-commands Patch/reduce response size for internal parse commands
bruin-data · Feb 25, 2025 · e4add6c · e4add6c
2 parents 22e74c9 + e8b122e
commit e4add6c
Show file tree

Hide file tree

Showing 13 changed files with 77 additions and 1,865 deletions.
diff --git a/cmd/internal.go b/cmd/internal.go
@@ -73,14 +73,20 @@ func ParsePipeline() *cli.Command {
 				Required:    false,
 				DefaultText: "false",
 			},
+			&cli.BoolFlag{
+				Name:        "exp-slim-response",
+				Usage:       "experimental flag to return a slim response",
+				Required:    false,
+				DefaultText: "false",
+			},
 		},
 		Action: func(c *cli.Context) error {
 			r := ParseCommand{
 				builder:      DefaultPipelineBuilder,
 				errorPrinter: errorPrinter,
 			}
 
-			return r.ParsePipeline(c.Args().Get(0), c.Bool("column-lineage"))
+			return r.ParsePipeline(c.Args().Get(0), c.Bool("column-lineage"), c.Bool("exp-slim-response"))
 		},
 		Before: telemetry.BeforeCommand,
 		After:  telemetry.AfterCommand,
@@ -111,7 +117,7 @@ type ParseCommand struct {
 	errorPrinter *color2.Color
 }
 
-func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool) error {
+func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool, slimResponse bool) error {
 	// defer RecoverFromPanic()
 	var lineageWg conc.WaitGroup
 	var sqlParser *sqlparser.SQLParser
@@ -170,7 +176,48 @@ func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool) error {
 
 	foundPipeline.WipeContentOfAssets()
 
-	js, err := json.Marshal(foundPipeline)
+	if !slimResponse {
+		js, err := json.Marshal(foundPipeline)
+		if err != nil {
+			printErrorJSON(err)
+			return cli.Exit("", 1)
+		}
+
+		fmt.Println(string(js))
+		return nil
+	}
+
+	type assetSummary struct {
+		ID             string                       `json:"id"`
+		Name           string                       `json:"name"`
+		Type           pipeline.AssetType           `json:"type"`
+		ExecutableFile *pipeline.ExecutableFile     `json:"executable_file"`
+		DefinitionFile *pipeline.TaskDefinitionFile `json:"definition_file"`
+		Upstreams      []pipeline.Upstream          `json:"upstreams"`
+	}
+
+	type pipelineSummary struct {
+		*pipeline.Pipeline
+		Assets []*assetSummary `json:"assets"`
+	}
+
+	ps := pipelineSummary{
+		Pipeline: foundPipeline,
+		Assets:   make([]*assetSummary, len(foundPipeline.Assets)),
+	}
+
+	for i, asset := range foundPipeline.Assets {
+		ps.Assets[i] = &assetSummary{
+			ID:             asset.ID,
+			Name:           asset.Name,
+			Type:           asset.Type,
+			ExecutableFile: &asset.ExecutableFile,
+			DefinitionFile: &asset.DefinitionFile,
+			Upstreams:      asset.Upstreams,
+		}
+	}
+
+	js, err := json.Marshal(ps)
 	if err != nil {
 		printErrorJSON(err)
 		return cli.Exit("", 1)
@@ -244,14 +291,22 @@ func (r *ParseCommand) Run(assetPath string, lineage bool) error {
 		}
 	}
 
+	type pipelineSummary struct {
+		Name     string            `json:"name"`
+		Schedule pipeline.Schedule `json:"schedule"`
+	}
+
 	js, err := json.Marshal(struct {
-		Asset    *pipeline.Asset    `json:"asset"`
-		Pipeline *pipeline.Pipeline `json:"pipeline"`
-		Repo     *git.Repo          `json:"repo"`
+		Asset    *pipeline.Asset `json:"asset"`
+		Pipeline pipelineSummary `json:"pipeline"`
+		Repo     *git.Repo       `json:"repo"`
 	}{
-		Asset:    asset,
-		Pipeline: foundPipeline,
-		Repo:     repoRoot,
+		Asset: asset,
+		Pipeline: pipelineSummary{
+			Name:     foundPipeline.Name,
+			Schedule: foundPipeline.Schedule,
+		},
+		Repo: repoRoot,
 	})
 	if err != nil {
 		printErrorJSON(err)

diff --git a/cmd/internal_test.go b/cmd/internal_test.go
@@ -14,7 +14,7 @@ func BenchmarkInternalParsePipeline(b *testing.B) {
 	for range [10]int{} {
 		b.ResetTimer()
 		start := time.Now()
-		if err := r.ParsePipeline("./testdata/lineage", true); err != nil {
+		if err := r.ParsePipeline("./testdata/lineage", true, false); err != nil {
 			b.Fatalf("Failed to run Internal Parse Pipeline command: %v", err)
 		}
 		b.StopTimer()
@@ -52,7 +52,7 @@ func BenchmarkInternalParsePipelineWithoutColumnLineage(b *testing.B) {
 	for range [10]int{} {
 		b.ResetTimer()
 		start := time.Now()
-		if err := r.ParsePipeline("./testdata/lineage", false); err != nil {
+		if err := r.ParsePipeline("./testdata/lineage", false, false); err != nil {
 			b.Fatalf("Failed to run Internal Parse Pipeline command: %v", err)
 		}
 		b.StopTimer()