diff --git a/integration-tests/happy-path/expectations/asset.py.json b/integration-tests/happy-path/expectations/asset.py.json index c6c58048a..e6b542672 100644 --- a/integration-tests/happy-path/expectations/asset.py.json +++ b/integration-tests/happy-path/expectations/asset.py.json @@ -11,7 +11,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -64,7 +65,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -181,11 +183,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", @@ -241,4 +245,4 @@ "repo": { "path": "/integration-tests" } -} \ No newline at end of file +} diff --git a/integration-tests/happy-path/expectations/chess_games.asset.yml.json b/integration-tests/happy-path/expectations/chess_games.asset.yml.json index c321a7e46..ada96e84d 100644 --- a/integration-tests/happy-path/expectations/chess_games.asset.yml.json +++ b/integration-tests/happy-path/expectations/chess_games.asset.yml.json @@ -58,7 +58,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -175,11 +176,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", @@ -188,7 +191,7 @@ "executable_file": { "name": "player_summary.sql", "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" }, "definition_file": { "name": "player_summary.sql", @@ -235,4 +238,4 @@ "repo": { "path": "/integration-tests" } -} \ No newline at end of file +} diff --git a/integration-tests/happy-path/expectations/chess_profiles.asset.yml.json b/integration-tests/happy-path/expectations/chess_profiles.asset.yml.json index 2771cf675..b73f40d1f 100644 --- a/integration-tests/happy-path/expectations/chess_profiles.asset.yml.json +++ b/integration-tests/happy-path/expectations/chess_profiles.asset.yml.json @@ -58,7 +58,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -175,11 +176,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", @@ -235,4 +238,4 @@ "repo": { "path": "/integration-tests" } -} \ No newline at end of file +} diff --git a/integration-tests/happy-path/expectations/pipeline.yml.json b/integration-tests/happy-path/expectations/pipeline.yml.json index ac816d42a..0cc5aef38 100644 --- a/integration-tests/happy-path/expectations/pipeline.yml.json +++ b/integration-tests/happy-path/expectations/pipeline.yml.json @@ -22,7 +22,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -139,11 +140,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", diff --git a/integration-tests/happy-path/expectations/player_summary.sql.json b/integration-tests/happy-path/expectations/player_summary.sql.json index b5ec2f21c..e20ae4ae0 100644 --- a/integration-tests/happy-path/expectations/player_summary.sql.json +++ b/integration-tests/happy-path/expectations/player_summary.sql.json @@ -17,11 +17,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", @@ -86,7 +88,8 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.player_summary" + "value": "chess_playground.player_summary", + "columns": [] } ], "image": "python:3.11", @@ -203,11 +206,13 @@ "upstreams": [ { "type": "asset", - "value": "chess_playground.games" + "value": "chess_playground.games", + "columns": [] }, { "type": "asset", - "value": "chess_playground.profiles" + "value": "chess_playground.profiles", + "columns": [] } ], "image": "", @@ -263,4 +268,4 @@ "repo": { "path": "/integration-tests" } -} \ No newline at end of file +} diff --git a/pkg/pipeline/lineage.go b/pkg/pipeline/lineage.go index c62814d5f..8d7627d6c 100644 --- a/pkg/pipeline/lineage.go +++ b/pkg/pipeline/lineage.go @@ -98,7 +98,7 @@ func (p *LineageExtractor) processLineageColumns(foundPipeline *Pipeline, asset return errors.New("asset cannot be nil") } - upstreams := []Upstream{} + upstreams := make([]Upstream, 0) for _, up := range asset.Upstreams { upstream := up lineage.NonSelectedColumns = append(lineage.NonSelectedColumns, lineage.Columns...) diff --git a/pkg/pipeline/pipeline.go b/pkg/pipeline/pipeline.go index 6bb889aea..d58dee067 100644 --- a/pkg/pipeline/pipeline.go +++ b/pkg/pipeline/pipeline.go @@ -508,7 +508,7 @@ type Upstream struct { Type string `json:"type" yaml:"type" mapstructure:"type"` Value string `json:"value" yaml:"value" mapstructure:"value"` Metadata EmptyStringMap `json:"metadata,omitempty" yaml:"metadata,omitempty" mapstructure:"metadata"` - Columns []DependsColumn `json:"columns,omitempty" yaml:"columns,omitempty" mapstructure:"columns"` + Columns []DependsColumn `json:"columns" yaml:"columns,omitempty" mapstructure:"columns"` } func (u Upstream) MarshalYAML() (interface{}, error) { diff --git a/pkg/pipeline/testdata/pipeline/first-pipeline_unix.json b/pkg/pipeline/testdata/pipeline/first-pipeline_unix.json index 6516be086..5f76175fb 100644 --- a/pkg/pipeline/testdata/pipeline/first-pipeline_unix.json +++ b/pkg/pipeline/testdata/pipeline/first-pipeline_unix.json @@ -54,7 +54,8 @@ "upstreams": [ { "type": "asset", - "value": "gcs-to-bq" + "value": "gcs-to-bq", + "columns": [] } ] }, @@ -129,27 +130,33 @@ "upstreams": [ { "type": "asset", - "value": "task1" + "value": "task1", + "columns": [] }, { "type": "asset", - "value": "task2" + "value": "task2", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] }, { "type": "asset", - "value": "task4" + "value": "task4", + "columns": [] }, { "type": "asset", - "value": "task5" + "value": "task5", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] } ] }, @@ -188,27 +195,33 @@ "upstreams": [ { "type": "asset", - "value": "task1" + "value": "task1", + "columns": [] }, { "type": "asset", - "value": "task2" + "value": "task2", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] }, { "type": "asset", - "value": "task4" + "value": "task4", + "columns": [] }, { "type": "asset", - "value": "task5" + "value": "task5", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] } ] } diff --git a/pkg/pipeline/testdata/pipeline/first-pipeline_windows.json b/pkg/pipeline/testdata/pipeline/first-pipeline_windows.json index b887a8015..70b22b766 100644 --- a/pkg/pipeline/testdata/pipeline/first-pipeline_windows.json +++ b/pkg/pipeline/testdata/pipeline/first-pipeline_windows.json @@ -46,7 +46,8 @@ "upstreams": [ { "type": "asset", - "value": "gcs-to-bq" + "value": "gcs-to-bq", + "columns": [] } ], "materialization": null, @@ -121,27 +122,33 @@ "upstreams": [ { "type": "asset", - "value": "task1" + "value": "task1", + "columns": [] }, { "type": "asset", - "value": "task2" + "value": "task2", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] }, { "type": "asset", - "value": "task4" + "value": "task4", + "columns": [] }, { "type": "asset", - "value": "task5" + "value": "task5", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] } ], "materialization": null, @@ -180,27 +187,33 @@ "upstreams": [ { "type": "asset", - "value": "task1" + "value": "task1", + "columns": [] }, { "type": "asset", - "value": "task2" + "value": "task2", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] }, { "type": "asset", - "value": "task4" + "value": "task4", + "columns": [] }, { "type": "asset", - "value": "task5" + "value": "task5", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] } ], "materialization": null, diff --git a/pkg/pipeline/testdata/pipeline/second-pipeline_unix.json b/pkg/pipeline/testdata/pipeline/second-pipeline_unix.json index f824d25f7..8699943a0 100644 --- a/pkg/pipeline/testdata/pipeline/second-pipeline_unix.json +++ b/pkg/pipeline/testdata/pipeline/second-pipeline_unix.json @@ -194,23 +194,28 @@ "upstreams": [ { "type": "asset", - "value": "task1" + "value": "task1", + "columns": [] }, { "type": "asset", - "value": "task2" + "value": "task2", + "columns": [] }, { "type": "asset", - "value": "task3" + "value": "task3", + "columns": [] }, { "type": "asset", - "value": "task4" + "value": "task4", + "columns": [] }, { "type": "asset", - "value": "task5" + "value": "task5", + "columns": [] } ] } diff --git a/pkg/pipeline/testdata/pipeline/second-pipeline_windows.json b/pkg/pipeline/testdata/pipeline/second-pipeline_windows.json index 23c91a6ed..0c9795e53 100644 --- a/pkg/pipeline/testdata/pipeline/second-pipeline_windows.json +++ b/pkg/pipeline/testdata/pipeline/second-pipeline_windows.json @@ -124,11 +124,16 @@ ], "athena": null, "upstreams": [ - {"type" : "asset", "value" : "task1"}, - {"type" : "asset", "value" : "task2"}, - {"type" : "asset", "value" : "task3"}, - {"type" : "asset", "value" : "task4"}, - {"type" : "asset", "value" : "task5"} + {"type" : "asset", "value" : "task1", + "columns": []}, + {"type" : "asset", "value" : "task2", + "columns": []}, + {"type" : "asset", "value" : "task3", + "columns": []}, + {"type" : "asset", "value" : "task4", + "columns": []}, + {"type" : "asset", "value" : "task5", + "columns": []} ], "materialization": null, "columns": [