diff --git a/cmd/internal.go b/cmd/internal.go index 14456ec37..de1ae59bc 100644 --- a/cmd/internal.go +++ b/cmd/internal.go @@ -73,6 +73,12 @@ func ParsePipeline() *cli.Command { Required: false, DefaultText: "false", }, + &cli.BoolFlag{ + Name: "exp-slim-response", + Usage: "experimental flag to return a slim response", + Required: false, + DefaultText: "false", + }, }, Action: func(c *cli.Context) error { r := ParseCommand{ @@ -80,7 +86,7 @@ func ParsePipeline() *cli.Command { errorPrinter: errorPrinter, } - return r.ParsePipeline(c.Args().Get(0), c.Bool("column-lineage")) + return r.ParsePipeline(c.Args().Get(0), c.Bool("column-lineage"), c.Bool("exp-slim-response")) }, Before: telemetry.BeforeCommand, After: telemetry.AfterCommand, @@ -111,7 +117,7 @@ type ParseCommand struct { errorPrinter *color2.Color } -func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool) error { +func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool, slimResponse bool) error { // defer RecoverFromPanic() var lineageWg conc.WaitGroup var sqlParser *sqlparser.SQLParser @@ -170,7 +176,48 @@ func (r *ParseCommand) ParsePipeline(assetPath string, lineage bool) error { foundPipeline.WipeContentOfAssets() - js, err := json.Marshal(foundPipeline) + if !slimResponse { + js, err := json.Marshal(foundPipeline) + if err != nil { + printErrorJSON(err) + return cli.Exit("", 1) + } + + fmt.Println(string(js)) + return nil + } + + type assetSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Type pipeline.AssetType `json:"type"` + ExecutableFile *pipeline.ExecutableFile `json:"executable_file"` + DefinitionFile *pipeline.TaskDefinitionFile `json:"definition_file"` + Upstreams []pipeline.Upstream `json:"upstreams"` + } + + type pipelineSummary struct { + *pipeline.Pipeline + Assets []*assetSummary `json:"assets"` + } + + ps := pipelineSummary{ + Pipeline: foundPipeline, + Assets: make([]*assetSummary, len(foundPipeline.Assets)), + } + + for i, asset := range foundPipeline.Assets { + ps.Assets[i] = &assetSummary{ + ID: asset.ID, + Name: asset.Name, + Type: asset.Type, + ExecutableFile: &asset.ExecutableFile, + DefinitionFile: &asset.DefinitionFile, + Upstreams: asset.Upstreams, + } + } + + js, err := json.Marshal(ps) if err != nil { printErrorJSON(err) return cli.Exit("", 1) @@ -244,14 +291,22 @@ func (r *ParseCommand) Run(assetPath string, lineage bool) error { } } + type pipelineSummary struct { + Name string `json:"name"` + Schedule pipeline.Schedule `json:"schedule"` + } + js, err := json.Marshal(struct { - Asset *pipeline.Asset `json:"asset"` - Pipeline *pipeline.Pipeline `json:"pipeline"` - Repo *git.Repo `json:"repo"` + Asset *pipeline.Asset `json:"asset"` + Pipeline pipelineSummary `json:"pipeline"` + Repo *git.Repo `json:"repo"` }{ - Asset: asset, - Pipeline: foundPipeline, - Repo: repoRoot, + Asset: asset, + Pipeline: pipelineSummary{ + Name: foundPipeline.Name, + Schedule: foundPipeline.Schedule, + }, + Repo: repoRoot, }) if err != nil { printErrorJSON(err) diff --git a/cmd/internal_test.go b/cmd/internal_test.go index 19d3fdd7b..69089e6d1 100644 --- a/cmd/internal_test.go +++ b/cmd/internal_test.go @@ -14,7 +14,7 @@ func BenchmarkInternalParsePipeline(b *testing.B) { for range [10]int{} { b.ResetTimer() start := time.Now() - if err := r.ParsePipeline("./testdata/lineage", true); err != nil { + if err := r.ParsePipeline("./testdata/lineage", true, false); err != nil { b.Fatalf("Failed to run Internal Parse Pipeline command: %v", err) } b.StopTimer() @@ -52,7 +52,7 @@ func BenchmarkInternalParsePipelineWithoutColumnLineage(b *testing.B) { for range [10]int{} { b.ResetTimer() start := time.Now() - if err := r.ParsePipeline("./testdata/lineage", false); err != nil { + if err := r.ParsePipeline("./testdata/lineage", false, false); err != nil { b.Fatalf("Failed to run Internal Parse Pipeline command: %v", err) } b.StopTimer() diff --git a/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json index b4ef6fb64..abf330aac 100644 --- a/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json +++ b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json @@ -135,458 +135,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/pipeline.yml" - }, - "default_connections": {}, - "assets": [ - { - "id": "aff64e4fd520bd185cb01adab98d2d20060f621c62d5cad5204712cfa2294ef7", - "uri": "", - "name": "country", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "users", - "columns": [ - { - "name": "id", - "usage": "" - }, - { - "name": "country", - "usage": "" - } - ] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "country.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/country.sql", - "content": "select id, country from users where id > 10000;" - }, - "definition_file": { - "name": "country.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/country.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "country", - "type": "varchar", - "description": "Just a country", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "country", - "table": "users" - } - ] - }, - { - "entity_attribute": null, - "name": "id", - "type": "integer", - "description": "Just a number", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "id", - "table": "users" - } - ] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", - "uri": "", - "name": "example", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "country", - "columns": [ - { - "name": "id", - "usage": "" - }, - { - "name": "country", - "usage": "" - } - ] - }, - { - "type": "asset", - "value": "people", - "columns": [ - { - "name": "id", - "usage": "" - }, - { - "name": "last_name", - "usage": "" - }, - { - "name": "name", - "usage": "" - }, - { - "name": "created_at", - "usage": "" - } - ] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "example.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/example.sql", - "content": "select \n a.name, \n a.last_name,\n a.created_at as updated_at,\n b.country \nfrom people a \njoin country b on a.id = b.id;" - }, - "definition_file": { - "name": "example.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/example.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "country", - "type": "varchar", - "description": "Just a country", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "country", - "table": "country" - } - ] - }, - { - "entity_attribute": null, - "name": "last_name", - "type": "varchar", - "description": "Just a last name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "last_name", - "table": "people" - } - ] - }, - { - "entity_attribute": null, - "name": "name", - "type": "varchar", - "description": "Just a name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "name", - "table": "people" - } - ] - }, - { - "entity_attribute": null, - "name": "updated_at", - "type": "timestamp", - "description": "Just a timestamp", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "created_at", - "table": "people" - } - ] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c9022680f888674e2b2274758755bfa07dea729b68d71cde5c521ed70ef261bf", - "uri": "", - "name": "people", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "users", - "columns": [ - { - "name": "country", - "usage": "" - }, - { - "name": "created_at", - "usage": "" - }, - { - "name": "id", - "usage": "" - }, - { - "name": "last_name", - "usage": "" - }, - { - "name": "name", - "usage": "" - } - ] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "people.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/people.sql", - "content": "select \n id, \n name, \n last_name,\n created_at\nfrom users where country = 'UK';" - }, - "definition_file": { - "name": "people.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/people.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "created_at", - "type": "timestamp", - "description": "Just a timestamp", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "created_at", - "table": "users" - } - ] - }, - { - "entity_attribute": null, - "name": "id", - "type": "integer", - "description": "Just a number", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "id", - "table": "users" - } - ] - }, - { - "entity_attribute": null, - "name": "last_name", - "type": "varchar", - "description": "Just a last name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "last_name", - "table": "users" - } - ] - }, - { - "entity_attribute": null, - "name": "name", - "type": "varchar", - "description": "Just a name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [ - { - "column": "name", - "table": "users" - } - ] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "7dfb4cf67742cb0660305e56ef816c53fcec892cae7f6ee39b75f34e659d672c", - "uri": "", - "name": "users", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "users.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/users.sql", - "content": "select * from user_data;" - }, - "definition_file": { - "name": "users.sql", - "path": "integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/users.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "id", - "type": "integer", - "description": "Just a number", - "primary_key": true, - "update_on_merge": false, - "checks": [], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "name", - "type": "varchar", - "description": "Just a name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "last_name", - "type": "varchar", - "description": "Just a last name", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "country", - "type": "varchar", - "description": "Just a country", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "created_at", - "type": "timestamp", - "description": "Just a timestamp", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "" }, "repo": { "path": "integration-tests" diff --git a/integration-tests/test-pipelines/parse-default-option/expectations/asset.py.json b/integration-tests/test-pipelines/parse-default-option/expectations/asset.py.json index 23dc873cd..a6c81593c 100644 --- a/integration-tests/test-pipelines/parse-default-option/expectations/asset.py.json +++ b/integration-tests/test-pipelines/parse-default-option/expectations/asset.py.json @@ -49,246 +49,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "integration-tests/test-pipelines/parse-default-option/pipeline.yml" - }, - "default_connections": { - "chess": "chess-run-default-option", - "duckdb": "duckdb-env-run-default-option" - }, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "integration-tests/test-pipelines/parse-default-option/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb-files/env-run-default-option.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "integration-tests/test-pipelines/parse-default-option/assets/asset.py", - "type": "comment" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - }, - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\nparameters:\n source_table: games" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option", - "source_table": "games" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_table: profiles" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option", - "source_table": "profiles" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "integration-tests/test-pipelines/parse-default-option/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "integration-tests/test-pipelines/parse-default-option/assets/player_summary.sql", - "type": "comment" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0, - "default": { - "type": "ingestr", - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "SecretKey": "KEY2", - "InjectedKey": "INJECTED2" - } - ] - } + "schedule": "" }, "repo": { "path": "integration-tests" diff --git a/integration-tests/test-pipelines/parse-default-option/expectations/chess_games.asset.yml.json b/integration-tests/test-pipelines/parse-default-option/expectations/chess_games.asset.yml.json index 47779d84f..1a558fe83 100644 --- a/integration-tests/test-pipelines/parse-default-option/expectations/chess_games.asset.yml.json +++ b/integration-tests/test-pipelines/parse-default-option/expectations/chess_games.asset.yml.json @@ -40,246 +40,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "integration-tests/test-pipelines/parse-default-option/pipeline.yml" - }, - "default_connections": { - "chess": "chess-run-default-option", - "duckdb": "duckdb-env-run-default-option" - }, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "integration-tests/test-pipelines/parse-default-option/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb-files/env-run-default-option.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "integration-tests/test-pipelines/parse-default-option/assets/asset.py", - "type": "comment" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - }, - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\nparameters:\n source_table: games" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option", - "source_table": "games" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_table: profiles" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "integration-tests/test-pipelines/parse-default-option/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option", - "source_table": "profiles" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "integration-tests/test-pipelines/parse-default-option/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "integration-tests/test-pipelines/parse-default-option/assets/player_summary.sql", - "type": "comment" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "secret_key": "KEY2", - "injected_key": "INJECTED2" - } - ], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0, - "default": { - "type": "ingestr", - "parameters": { - "destination": "duckdb", - "source_connection": "chess-run-default-option" - }, - "secrets": [ - { - "SecretKey": "KEY2", - "InjectedKey": "INJECTED2" - } - ] - } + "schedule": "" }, "repo": { "path": "integration-tests" diff --git a/integration-tests/test-pipelines/parse-happy-path/expectations/asset.py.json b/integration-tests/test-pipelines/parse-happy-path/expectations/asset.py.json index 2d7626736..bae670803 100644 --- a/integration-tests/test-pipelines/parse-happy-path/expectations/asset.py.json +++ b/integration-tests/test-pipelines/parse-happy-path/expectations/asset.py.json @@ -42,205 +42,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "" }, "repo": { "path": "/integration-tests" diff --git a/integration-tests/test-pipelines/parse-happy-path/expectations/chess_games.asset.yml.json b/integration-tests/test-pipelines/parse-happy-path/expectations/chess_games.asset.yml.json index e0cebbc08..dd8f15a72 100644 --- a/integration-tests/test-pipelines/parse-happy-path/expectations/chess_games.asset.yml.json +++ b/integration-tests/test-pipelines/parse-happy-path/expectations/chess_games.asset.yml.json @@ -35,205 +35,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "" }, "repo": { "path": "/integration-tests" diff --git a/integration-tests/test-pipelines/parse-happy-path/expectations/chess_profiles.asset.yml.json b/integration-tests/test-pipelines/parse-happy-path/expectations/chess_profiles.asset.yml.json index b936d46fa..d5c500dfc 100644 --- a/integration-tests/test-pipelines/parse-happy-path/expectations/chess_profiles.asset.yml.json +++ b/integration-tests/test-pipelines/parse-happy-path/expectations/chess_profiles.asset.yml.json @@ -35,205 +35,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "" }, "repo": { "path": "/integration-tests" diff --git a/integration-tests/test-pipelines/parse-happy-path/expectations/player_summary.sql.json b/integration-tests/test-pipelines/parse-happy-path/expectations/player_summary.sql.json index 35c5775ec..8f35c5c9b 100644 --- a/integration-tests/test-pipelines/parse-happy-path/expectations/player_summary.sql.json +++ b/integration-tests/test-pipelines/parse-happy-path/expectations/player_summary.sql.json @@ -66,205 +66,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "" }, "repo": { "path": "/integration-tests" diff --git a/integration-tests/test-pipelines/run-seed-data/expectations/seed.asset.yml.json b/integration-tests/test-pipelines/run-seed-data/expectations/seed.asset.yml.json index 6b0764d23..45939e68d 100644 --- a/integration-tests/test-pipelines/run-seed-data/expectations/seed.asset.yml.json +++ b/integration-tests/test-pipelines/run-seed-data/expectations/seed.asset.yml.json @@ -106,135 +106,8 @@ "athena": null }, "pipeline": { - "legacy_id": "", "name": "bruin-init", - "schedule": "daily", - "start_date": "2023-03-20", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/test-pipelines/run-seed-data/pipeline.yml" - }, - "default_connections": { - "duckdb": "duckdb-run-seed-data" - }, - "assets": [ - { - "id": "3c80839e56779484af2542f23fd99f49d1a6905a99122ecb3a29c0b57dc83fe9", - "uri": "", - "name": "seed.raw", - "type": "duckdb.seed", - "description": "This asset loads a CSV file into a DuckDB database.", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "seed.asset.yml", - "path": "/integration-tests/test-pipelines/run-seed-data/assets/seed.asset.yml", - "content": "name: seed.raw\ntype: duckdb.seed\n\ndescription: This asset loads a CSV file into a DuckDB database.\ncolumns:\n - name: name\n type: varchar\n description: \"Contact person's full name\"\n checks:\n - name: not_null\n - name: networking_through\n type: varchar\n description: \"Source or connection through which contact was made\"\n checks:\n - name: not_null\n - name: accepted_values\n value:\n - LinkedIn\n - Twitter\n - Facebook\n - Instagram\n - name: position\n type: varchar\n description: \"Contact's job position or title\"\n checks:\n - name: not_null\n - name: contact_date\n type: varchar\n description: \"Date when contact was established\"\n\n\nparameters:\n path: ./seed.csv\n" - }, - "definition_file": { - "name": "seed.asset.yml", - "path": "/integration-tests/test-pipelines/run-seed-data/assets/seed.asset.yml", - "type": "yaml" - }, - "parameters": { - "path": "./seed.csv" - }, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "name", - "type": "varchar", - "description": "Contact person's full name", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "6d0edf3e6836006758d1213e7927cf9530d9dcdf9dcccc6638bee257dd73e857", - "name": "not_null", - "value": null, - "blocking": true - } - ], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "networking_through", - "type": "varchar", - "description": "Source or connection through which contact was made", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "3a6115e7ac5dffc8f0c0c6cb8ea427296f7656167d574e5cefde62a37535222f", - "name": "not_null", - "value": null, - "blocking": true - }, - { - "id": "5d4569e858cbc6f56cf8ca9d934b743f4fa3b9b870d24f99dab7a390b1bb5168", - "name": "accepted_values", - "value": [ - "LinkedIn", - "Twitter", - "Facebook", - "Instagram" - ], - "blocking": true - } - ], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "position", - "type": "varchar", - "description": "Contact's job position or title", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "d625425165fba6cc0d5c0ecf233d6d719b3bcadba8845f31396ce94acf051d60", - "name": "not_null", - "value": null, - "blocking": true - } - ], - "upstreams": [] - }, - { - "entity_attribute": null, - "name": "contact_date", - "type": "varchar", - "description": "Date when contact was established", - "primary_key": false, - "update_on_merge": false, - "checks": [], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 + "schedule": "daily" }, "repo": { "path": "/integration-tests" diff --git a/pkg/ingestr/types.go b/pkg/ingestr/types.go index fbacea7b3..435b9356b 100644 --- a/pkg/ingestr/types.go +++ b/pkg/ingestr/types.go @@ -42,7 +42,7 @@ var typeHintMapping = map[string]string{ // columnHints returns an ingestr compatible type hint string // that can be passed via the --column flag to the CLI. func columnHints(cols []pipeline.Column) string { - var hints = make([]string, 0) + hints := make([]string, 0) for _, col := range cols { typ := normaliseColumnType(col.Type) hint, exists := typeHintMapping[typ] diff --git a/pkg/ingestr/types_test.go b/pkg/ingestr/types_test.go index 87f22bef3..d68a1e3e4 100644 --- a/pkg/ingestr/types_test.go +++ b/pkg/ingestr/types_test.go @@ -4,7 +4,7 @@ import "testing" func TestNormaliseColumnName(t *testing.T) { t.Parallel() - var testCases = map[string]string{ + testCases := map[string]string{ "CamelCase": "camel_case", "With Space": "with_space", "With Two Space": "with_two_space", diff --git a/pkg/pipeline/lineage_test.go b/pkg/pipeline/lineage_test.go index 4a4b50c37..50a4fd039 100644 --- a/pkg/pipeline/lineage_test.go +++ b/pkg/pipeline/lineage_test.go @@ -8,9 +8,7 @@ import ( "github.com/bruin-data/bruin/pkg/sqlparser" ) -var ( - SQLParser *sqlparser.SQLParser -) +var SQLParser *sqlparser.SQLParser func TestMain(m *testing.M) { err := SetupSQLParser()