Skip to content

Commit

Permalink
do not parse the whole pipeline for asset parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
karakanb committed Feb 28, 2025
1 parent e4add6c commit 16a503f
Showing 1 changed file with 35 additions and 4 deletions.
39 changes: 35 additions & 4 deletions cmd/internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
"path/filepath"

"github.com/bruin-data/bruin/pkg/config"
"github.com/bruin-data/bruin/pkg/git"
Expand Down Expand Up @@ -256,26 +257,56 @@ func (r *ParseCommand) Run(assetPath string, lineage bool) error {
return cli.Exit("", 1)
}

pipelinePath, err := path.GetPipelineRootFromTask(assetPath, pipelineDefinitionFiles)
absoluteAssetPath, err := filepath.Abs(assetPath)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

repoRoot, err := git.FindRepoFromPath(assetPath)
pipelinePath, err := path.GetPipelineRootFromTask(absoluteAssetPath, pipelineDefinitionFiles)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

foundPipeline, err := DefaultPipelineBuilder.CreatePipelineFromPath(pipelinePath, true)
repoRoot, err := git.FindRepoFromPath(absoluteAssetPath)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

pipelineDefinitionPath, err := getPipelineDefinitionFullPath(pipelinePath)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

var foundPipeline *pipeline.Pipeline

// column-level lineage requires the whole pipeline to be parsed by nature, therefore we need to use the default pipeline builder.
// however, since the primary usecase of this command requires speed, we'll use a faster alternative if there's no lineage requested.
if lineage {
foundPipeline, err = DefaultPipelineBuilder.CreatePipelineFromPath(pipelineDefinitionPath, true)
} else {
foundPipeline, err = pipeline.PipelineFromPath(pipelineDefinitionPath, afero.NewOsFs())
}

if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

asset, err := DefaultPipelineBuilder.CreateAssetFromFile(absoluteAssetPath, nil)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

asset := foundPipeline.GetAssetByPath(assetPath)
asset, err = DefaultPipelineBuilder.MutateAsset(asset, foundPipeline)
if err != nil {
printErrorJSON(err)
return cli.Exit("", 1)
}

if lineage {
panics := lineageWg.WaitAndRecover()
Expand Down

0 comments on commit 16a503f

Please sign in to comment.