diff --git a/.gitignore b/.gitignore index 7c0e75b..b59ef20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,29 @@ +#--------------------------------------------------# +# The following was generated with gitignore.nvim: # +#--------------------------------------------------# +# Gitignore for the following technologies: Go + +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + test_build tbd diff --git a/forms.go b/forms.go index 622a755..4373664 100644 --- a/forms.go +++ b/forms.go @@ -13,6 +13,8 @@ type FormResponse struct { Account string Database string Schema string + Project string + Dataset string BuildDir string GenerateDescriptions bool GroqKeyEnvVar string @@ -30,14 +32,13 @@ func Forms() (formResponse FormResponse) { Description(`A sweet and speedy code generator for dbt. tbd will generate source YAML config and SQL staging models for all the tables in the schema you specify. To prepare, make sure you have the following: -* An existing dbt profile.yml file to reference -OR -✴︎ *_Username_* (e.g. aragorn@dunedain.king) -✴︎ *_Account ID_* (e.g. elfstone-consulting.us-west-1) -✴︎ *_Schema_* you want to generate (e.g. minas-tirith) -✴︎ *_Database_* that schema is in (e.g. gondor) -Authentication will be handled via SSO in the web browser. -For security, we don't currently support password-based authentication.`), + +✴︎ An existing dbt profile.yml file to reference +**OR** +✴︎ The necessary connection details for your warehouse + +_Authentication will be handled via SSO._ +_For security, we don't support password auth._`), ), huh.NewGroup( huh.NewNote(). @@ -45,7 +46,7 @@ For security, we don't currently support password-based authentication.`), Description(`I'm currently exploring *_optional_* LLM-powered alpha features. At present this is limited to generating column descriptions and inferring tests via Groq. You'll need: -✴︎ A Groq API key stored in an environment variable.`), +✴︎ A Groq API key stored in an env var`), huh.NewConfirm().Affirmative("Sure!").Negative("Nope"). Title("Do you want to generate column descriptions and tests via LLM?"). Value(&formResponse.GenerateDescriptions), @@ -68,15 +69,19 @@ You'll need: Placeholder("dev"), ), ) - manual_form := huh.NewForm( + warehouse_form := huh.NewForm( huh.NewGroup( huh.NewSelect[string](). Title("Choose your warehouse."). Options( huh.NewOption("Snowflake", "snowflake"), + huh.NewOption("BigQuery", "bigquery"), ). Value(&formResponse.Warehouse), - + ), + ) + snowflake_form := huh.NewForm( + huh.NewGroup( huh.NewInput(). Title("What is your username?"). Value(&formResponse.Username).Placeholder("aragorn@dunedain.king"), @@ -94,6 +99,14 @@ You'll need: Value(&formResponse.Database).Placeholder("gondor"), ), ) + bigquery_form := huh.NewForm( + huh.NewGroup( + huh.NewInput().Title("What is your GCP project's id?"). + Value(&formResponse.Project).Placeholder("legolas_inc"), + huh.NewInput().Title("What is the dataset you want to generate?"). + Value(&formResponse.Dataset).Placeholder("mirkwood"), + ), + ) llm_form := huh.NewForm( huh.NewGroup( huh.NewInput(). @@ -125,7 +138,9 @@ If you use an existing directory, tbd will overwrite any existing files with the ) intro_form.WithTheme(huh.ThemeCatppuccin()) dbt_form.WithTheme(huh.ThemeCatppuccin()) - manual_form.WithTheme(huh.ThemeCatppuccin()) + warehouse_form.WithTheme(huh.ThemeCatppuccin()) + snowflake_form.WithTheme(huh.ThemeCatppuccin()) + bigquery_form.WithTheme(huh.ThemeCatppuccin()) llm_form.WithTheme(huh.ThemeCatppuccin()) dir_form.WithTheme(huh.ThemeCatppuccin()) confirm_form.WithTheme(huh.ThemeCatppuccin()) @@ -136,10 +151,22 @@ If you use an existing directory, tbd will overwrite any existing files with the if formResponse.UseDbtProfile { err = dbt_form.Run() } else { - err = manual_form.Run() + err = warehouse_form.Run() + switch formResponse.Warehouse { + case "snowflake": + err = snowflake_form.Run() + if err != nil { + log.Fatalf("Error running snowflake form %v\n", err) + } + case "bigquery": + err = bigquery_form.Run() + if err != nil { + log.Fatalf("Error running bigquery form %v\n", err) + } + } } if err != nil { - log.Fatalf("Error running connectiond details form %v\n", err) + log.Fatalf("Error running connection details form %v\n", err) } if formResponse.GenerateDescriptions { err = llm_form.Run() diff --git a/get_dbt_profile_test.go b/get_dbt_profile_test.go index 91d1ad7..209985a 100644 --- a/get_dbt_profile_test.go +++ b/get_dbt_profile_test.go @@ -2,40 +2,15 @@ package main import ( "os" - "path/filepath" "testing" ) func TestGetDbtProfile(t *testing.T) { - // Create a temporary profiles.yml file for testing - tmpDir := t.TempDir() - err := os.Mkdir(filepath.Join(tmpDir, ".dbt"), 0755) - if err != nil { - t.Fatalf("Failed to create temporary .dbt directory: %v", err) - } - profilesFile := filepath.Join(tmpDir, ".dbt", "profiles.yml") - content := []byte(` -test_profile: - target: dev - outputs: - dev: - type: snowflake - account: testaccount - user: testuser - password: testpassword - database: testdb - warehouse: testwh - schema: testschema -`) - err = os.WriteFile(profilesFile, content, 0644) - if err != nil { - t.Fatalf("Failed to create temporary profiles.yml file: %v", err) - } - - os.Setenv("HOME", tmpDir) - + CreateTempDbtProfile(t) + defer os.RemoveAll(os.Getenv("HOME")) + defer os.Unsetenv("HOME") // Profile exists - profile, err := GetDbtProfile("test_profile") + profile, err := GetDbtProfile("elf") if err != nil { t.Errorf("GetDbtProfile returned an error for an existing profile: %v", err) } @@ -44,7 +19,7 @@ test_profile: } // Profile does not exist - profile, err = GetDbtProfile("aragorn") + profile, err = GetDbtProfile("dunedain") if err == nil { t.Error("GetDbtProfile did not return an error for a non-existing profile") } diff --git a/go.mod b/go.mod index 9dc5dc4..af64b6e 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,25 @@ go 1.22.0 require ( github.com/charmbracelet/huh v0.3.1-0.20240306161957-71f31c155b08 github.com/charmbracelet/huh/spinner v0.0.0-20240306161957-71f31c155b08 - github.com/snowflakedb/gosnowflake v1.8.0 + github.com/snowflakedb/gosnowflake v1.9.0 gopkg.in/yaml.v2 v2.4.0 ) require ( + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/bigquery v1.60.0 // indirect + cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.7 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 // indirect + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/apache/arrow/go/v14 v14.0.2 // indirect + github.com/apache/arrow/go/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.25.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect @@ -41,12 +48,21 @@ require ( github.com/danieljoos/wincred v1.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -66,14 +82,29 @@ require ( github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/crypto v0.19.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.18.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/api v0.170.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index c3f8801..b937239 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,14 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go/bigquery v1.60.0 h1:kA96WfgvCbkqfLnr7xI5uEfJ4h4FrnkdEb0yty0KSZo= +cloud.google.com/go/bigquery v1.60.0/go.mod h1:Clwk2OeC0ZU5G5LDg7mo+h8U7KlAa5v06z5rptKdM3g= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= @@ -14,10 +25,15 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPK github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/arrow/go/v15 v15.0.0 h1:1zZACWf85oEZY5/kd9dsQS7i+2G5zVQcbKTHgslqHNA= +github.com/apache/arrow/go/v15 v15.0.0/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= @@ -62,6 +78,7 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/catppuccin/go v0.2.0 h1:ktBeIrIP42b/8FGiScP9sgrWOss3lw0Z5SktRoithGA= github.com/catppuccin/go v0.2.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= @@ -74,6 +91,8 @@ github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1 github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/charmbracelet/x/exp/strings v0.0.0-20240304160204-3835fda67169 h1:1QdJhraY3DLWdZqVys2qYB2MT5/uo/WrpeEjRpPWcvg= github.com/charmbracelet/x/exp/strings v0.0.0-20240304160204-3835fda67169/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= @@ -87,28 +106,72 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= @@ -147,6 +210,7 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -157,42 +221,151 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/snowflakedb/gosnowflake v1.8.0 h1:4bQj8eAYGMkou/nICiIEb9jSbBLDDp5cB6JaKx9WwiA= github.com/snowflakedb/gosnowflake v1.8.0/go.mod h1:7yyY2MxtDti2eXgtvlZ8QxzCN6KV2B4qb1HuygMI+0U= +github.com/snowflakedb/gosnowflake v1.9.0 h1:s2ZdwFxFfpqwa5CqlhnzRESnLmwU3fED6zyNOJHFBQA= +github.com/snowflakedb/gosnowflake v1.9.0/go.mod h1:4ZgHxVf2OKwecx07WjfyAMr0gn8Qj4yvwAo68Og8wsU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48= +google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -203,3 +376,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index d50b0ea..de3674b 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "time" "github.com/charmbracelet/huh/spinner" - _ "github.com/snowflakedb/gosnowflake" ) type DbtProfile struct { @@ -22,6 +21,8 @@ type DbtProfile struct { Authenticator string `yaml:"authenticator"` Database string `yaml:"database"` Schema string `yaml:"schema"` + Project string `yaml:"project"` + Dataset string `yaml:"dataset"` Threads int `yaml:"threads"` } `yaml:"outputs"` } @@ -34,7 +35,7 @@ func main() { if !formResponse.Confirm { log.Fatal("⛔ User cancelled.") } - connectionDetails := SetConnectionDetails(formResponse) + cd := SetConnectionDetails(formResponse) var ( dbElapsed float64 @@ -45,7 +46,11 @@ func main() { connectionStart := time.Now() buildDir := formResponse.BuildDir - tables, err := sourcerer.GetSources(ctx, connectionDetails) + dbc, err := sourcerer.GetConn(cd) + if err != nil { + log.Fatalf("Error getting connection: %v\n", err) + } + tables, err := dbc.GetSources(ctx) if err != nil { log.Fatalf("Error getting sources: %v\n", err) } diff --git a/set_connection_details.go b/set_connection_details.go index 075659f..c215537 100644 --- a/set_connection_details.go +++ b/set_connection_details.go @@ -18,6 +18,8 @@ func SetConnectionDetails(formResponse FormResponse) shared.ConnectionDetails { Account: profile.Outputs[formResponse.DbtProfileOutput].Account, Schema: profile.Outputs[formResponse.DbtProfileOutput].Schema, Database: profile.Outputs[formResponse.DbtProfileOutput].Database, + Project: profile.Outputs[formResponse.DbtProfileOutput].Project, + Dataset: profile.Outputs[formResponse.DbtProfileOutput].Dataset, } } else { connectionDetails = shared.ConnectionDetails{ @@ -26,6 +28,8 @@ func SetConnectionDetails(formResponse FormResponse) shared.ConnectionDetails { Account: formResponse.Account, Schema: formResponse.Schema, Database: formResponse.Database, + Project: formResponse.Project, + Dataset: formResponse.Dataset, } } return connectionDetails diff --git a/set_connection_details_test.go b/set_connection_details_test.go new file mode 100644 index 0000000..62451cb --- /dev/null +++ b/set_connection_details_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "os" + "tbd/shared" + "testing" +) + +func TestSetConnectionDetailsWithoutDbtProfile(t *testing.T) { + formResponse := FormResponse{ + UseDbtProfile: false, + Warehouse: "snowflake", + Username: "aragorn", + Account: "123456.us-east-1", + Database: "gondor", + Schema: "minas_tirith", + GenerateDescriptions: false, + BuildDir: "test_build", + Confirm: true, + } + connectionDetails := SetConnectionDetails(formResponse) + want := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "aragorn", + Account: "123456.us-east-1", + Database: "gondor", + Schema: "minas_tirith", + } + if connectionDetails != want { + t.Errorf("got %v, want %v", connectionDetails, want) + } +} + +func TestSetConnectionDetailsWithDbtProfile(t *testing.T) { + CreateTempDbtProfile(t) + defer os.RemoveAll(os.Getenv("HOME")) + defer os.Unsetenv("HOME") + formResponse := FormResponse{ + UseDbtProfile: true, + DbtProfile: "elf", + DbtProfileOutput: "dev", + GenerateDescriptions: false, + BuildDir: "test_build", + Confirm: true, + } + connectionDetails := SetConnectionDetails(formResponse) + want := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "legolas", + Account: "123456.us-east-1", + Database: "mirkwood", + Schema: "hall_of_thranduil", + } + if connectionDetails != want { + t.Errorf("got %v, want %v", connectionDetails, want) + } +} diff --git a/shared/types.go b/shared/types.go index 44f7737..53906a2 100644 --- a/shared/types.go +++ b/shared/types.go @@ -21,6 +21,8 @@ type ConnectionDetails struct { ConnType string Username string Account string - Schema string Database string + Schema string + Project string + Dataset string } diff --git a/sourcerer/connect_to_db.go b/sourcerer/connect_to_db.go index 6ce312f..d484d53 100644 --- a/sourcerer/connect_to_db.go +++ b/sourcerer/connect_to_db.go @@ -5,41 +5,35 @@ import ( "database/sql" "fmt" "log" - "strings" - "tbd/shared" "time" -) - -type DBConnection interface { - ConnectToDB() (*sql.DB, error) -} -type SnowflakeConnection struct { - Username string - Account string - Schema string - Database string -} - -type BigQueryConnection struct { - Project string - Dataset string -} + "cloud.google.com/go/bigquery" +) -func (sfc *SnowflakeConnection) ConnectToDB(ctx context.Context, connectionDetails shared.ConnectionDetails) (db *sql.DB, cancel context.CancelFunc, err error) { +func (sfc *SfConn) ConnectToDB(ctx context.Context) (err error) { connStr := fmt.Sprintf( "%s@%s/%s/%s?authenticator=externalbrowser", - strings.ToUpper(connectionDetails.Username), - strings.ToUpper(connectionDetails.Account), - strings.ToUpper(connectionDetails.Database), - strings.ToUpper(connectionDetails.Schema), + sfc.Username, + sfc.Account, + sfc.Database, + sfc.Schema, ) - _, cancel = context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - db, err = sql.Open("snowflake", connStr) + _, sfc.Cancel = context.WithTimeout(ctx, 1*time.Minute) + defer sfc.Cancel() + sfc.Db, err = sql.Open("snowflake", connStr) if err != nil { log.Fatalf("Could not connect to Snowflake %v\n", err) } - return db, cancel, err + return err +} + +func (bqc *BqConn) ConnectToDB(ctx context.Context) (err error) { + _, bqc.Cancel = context.WithTimeout(ctx, 1*time.Minute) + defer bqc.Cancel() + bqc.Bq, err = bigquery.NewClient(ctx, bqc.Project) + if err != nil { + log.Fatalf("Could not connect to BigQuery %v\n", err) + } + return err } diff --git a/sourcerer/get_columns.go b/sourcerer/get_columns.go index 38cd536..5cd6899 100644 --- a/sourcerer/get_columns.go +++ b/sourcerer/get_columns.go @@ -2,29 +2,59 @@ package sourcerer import ( "context" - "database/sql" "fmt" "log" "tbd/shared" + + "cloud.google.com/go/bigquery" + "google.golang.org/api/iterator" ) -func GetColumns(db *sql.DB, ctx context.Context, table shared.SourceTable, connectionDetails shared.ConnectionDetails) ([]shared.Column, error) { - var columns []shared.Column +func (sfc *SfConn) GetColumns(ctx context.Context, t shared.SourceTable) ([]shared.Column, error) { + var cs []shared.Column - query := fmt.Sprintf("SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = '%s' AND table_name = '%s'", connectionDetails.Schema, table.Name) - rows, err := db.QueryContext(ctx, query) + q := fmt.Sprintf("SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = '%s' AND table_name = '%s'", sfc.Schema, t.Name) + rows, err := sfc.Db.QueryContext(ctx, q) if err != nil { - log.Fatalf("Error fetching columns for table %s: %v\n", table.Name, err) + log.Fatalf("Error fetching columns for table %s: %v\n", t.Name, err) } defer rows.Close() for rows.Next() { - column := shared.Column{} - if err := rows.Scan(&column.Name, &column.DataType); err != nil { - log.Fatalf("Error scanning columns for table %s: %v\n", table.Name, err) + c := shared.Column{} + if err := rows.Scan(&c.Name, &c.DataType); err != nil { + log.Fatalf("Error scanning columns for table %s: %v\n", t.Name, err) } - columns = append(columns, column) + cs = append(cs, c) } + return cs, nil +} - return columns, nil +func (bqc *BqConn) GetColumns(ctx context.Context, t shared.SourceTable) ([]shared.Column, error) { + var cs []shared.Column + qs := fmt.Sprintf("SELECT column_name, data_type FROM %s.%s.INFORMATION_SCHEMA.COLUMNS WHERE table_name = @table", bqc.Project, bqc.Dataset) + q := bqc.Bq.Query(qs) + q.Parameters = []bigquery.QueryParameter{ + {Name: "table", Value: t.Name}, + } + it, err := q.Read(ctx) + if err != nil { + log.Fatalf("Error fetching columns for table %s: %v\n", t.Name, err) + } + for { + var values []bigquery.Value + err := it.Next(&values) + if err == iterator.Done { + break + } + if err != nil { + log.Fatalf("Error scanning columns for table %s: %v\n", t.Name, err) + } + c := shared.Column{ + Name: values[0].(string), + DataType: values[1].(string), + } + cs = append(cs, c) + } + return cs, nil } diff --git a/sourcerer/get_conn.go b/sourcerer/get_conn.go new file mode 100644 index 0000000..cc192d7 --- /dev/null +++ b/sourcerer/get_conn.go @@ -0,0 +1,27 @@ +package sourcerer + +import ( + "errors" + "strings" + "tbd/shared" +) + +func GetConn(cd shared.ConnectionDetails) (DbConn, error) { + switch cd.ConnType { + case "snowflake": + // TODO: Why do I need to use a pointer here? + return &SfConn{ + Account: strings.ToUpper(cd.Account), + Username: strings.ToUpper(cd.Username), + Database: strings.ToUpper(cd.Database), + Schema: strings.ToUpper(cd.Schema), + }, nil + case "bigquery": + return &BqConn{ + Project: cd.Project, + Dataset: cd.Dataset, + }, nil + default: + return nil, errors.New("unsupported connection type") + } +} diff --git a/sourcerer/get_sources.go b/sourcerer/get_sources.go index 46ca160..ec7a932 100644 --- a/sourcerer/get_sources.go +++ b/sourcerer/get_sources.go @@ -2,48 +2,82 @@ package sourcerer import ( "context" + "database/sql" "fmt" "log" - "strings" "tbd/shared" + + "cloud.google.com/go/bigquery" + "google.golang.org/api/iterator" ) -func GetSources(ctx context.Context, connectionDetails shared.ConnectionDetails) (shared.SourceTables, error) { - tables := shared.SourceTables{} - - switch connectionDetails.ConnType { - case "snowflake": - { - dbConn := SnowflakeConnection{} - // Snowflake requires uppercase for all identifiers - connectionDetails.Username = strings.ToUpper(connectionDetails.Username) - connectionDetails.Account = strings.ToUpper(connectionDetails.Account) - connectionDetails.Database = strings.ToUpper(connectionDetails.Database) - connectionDetails.Schema = strings.ToUpper(connectionDetails.Schema) - - db, cancel, err := dbConn.ConnectToDB(ctx, connectionDetails) - defer cancel() - if err != nil { - log.Fatalf("Couldn't connect to database: %v\n", err) - } - rows, err := db.QueryContext(ctx, fmt.Sprintf("SELECT table_name FROM information_schema.tables where table_schema = '%s'", connectionDetails.Schema)) - if err != nil { - log.Fatalf("Error fetching tables: %v\n", err) - } - defer rows.Close() - for rows.Next() { - var table shared.SourceTable - if err := rows.Scan(&table.Name); err != nil { - log.Fatalf("Error scanning tables: %v\n", err) - } - tables.SourceTables = append(tables.SourceTables, table) - } - PutColumnsOnTables(ctx, db, tables, connectionDetails) +type DbConn interface { + ConnectToDB(ctx context.Context) error + GetSources(ctx context.Context) (shared.SourceTables, error) + GetColumns(ctx context.Context, t shared.SourceTable) ([]shared.Column, error) + PutColumnsOnTables(ctx context.Context, tables shared.SourceTables) +} + +type SfConn struct { + Account string + Username string + Database string + Schema string + Db *sql.DB + Cancel context.CancelFunc +} + +type BqConn struct { + Project string + Dataset string + Bq *bigquery.Client + Cancel context.CancelFunc +} + +func (sfc *SfConn) GetSources(ctx context.Context) (shared.SourceTables, error) { + ts := shared.SourceTables{} + + err := sfc.ConnectToDB(ctx) + defer sfc.Cancel() + if err != nil { + log.Fatalf("Couldn't connect to database: %v\n", err) + } + rows, err := sfc.Db.QueryContext(ctx, fmt.Sprintf("SELECT table_name FROM information_schema.tables where table_schema = '%s'", sfc.Schema)) + if err != nil { + log.Fatalf("Error fetching tables: %v\n", err) + } + defer rows.Close() + for rows.Next() { + var table shared.SourceTable + if err := rows.Scan(&table.Name); err != nil { + log.Fatalf("Error scanning tables: %v\n", err) + } + ts.SourceTables = append(ts.SourceTables, table) + } + sfc.PutColumnsOnTables(ctx, ts) + + return ts, nil +} + +func (bqc *BqConn) GetSources(ctx context.Context) (shared.SourceTables, error) { + ts := shared.SourceTables{} + err := bqc.ConnectToDB(ctx) + defer bqc.Cancel() + if err != nil { + log.Fatalf("Couldn't connect to database: %v\n", err) + } + bqDataset := bqc.Bq.Dataset(bqc.Dataset) + tableIter := bqDataset.Tables(ctx) + for { + table, err := tableIter.Next() + if err == iterator.Done { + break } - default: - { - return tables, fmt.Errorf("unsupported warehouse: %s", connectionDetails.ConnType) + if err != nil { + log.Fatalf("Error fetching tables: %v\n", err) } + ts.SourceTables = append(ts.SourceTables, shared.SourceTable{Name: table.TableID}) } - return tables, nil + bqc.PutColumnsOnTables(ctx, ts) + return ts, nil } diff --git a/sourcerer/put_columns_on_tables.go b/sourcerer/put_columns_on_tables.go index a628503..7073e34 100644 --- a/sourcerer/put_columns_on_tables.go +++ b/sourcerer/put_columns_on_tables.go @@ -2,7 +2,6 @@ package sourcerer import ( "context" - "database/sql" "fmt" "log" "regexp" @@ -10,7 +9,7 @@ import ( "tbd/shared" ) -func PutColumnsOnTables(ctx context.Context, db *sql.DB, tables shared.SourceTables, connectionDetails shared.ConnectionDetails) { +func (sfc *SfConn) PutColumnsOnTables(ctx context.Context, tables shared.SourceTables) { mutex := sync.Mutex{} var wg sync.WaitGroup @@ -29,7 +28,7 @@ func PutColumnsOnTables(ctx context.Context, db *sql.DB, tables shared.SourceTab go func(i int) { defer wg.Done() - columns, err := GetColumns(db, ctx, tables.SourceTables[i], connectionDetails) + columns, err := sfc.GetColumns(ctx, tables.SourceTables[i]) if err != nil { log.Fatalf("Error fetching columns for table %s: %v\n", tables.SourceTables[i].Name, err) return @@ -53,3 +52,42 @@ func PutColumnsOnTables(ctx context.Context, db *sql.DB, tables shared.SourceTab } wg.Wait() } + +func (bqc *BqConn) PutColumnsOnTables(ctx context.Context, tables shared.SourceTables) { + mutex := sync.Mutex{} + var wg sync.WaitGroup + wg.Add(len(tables.SourceTables)) + dataTypeGroupMap := map[string]string{ + "(string)": "text", + "(float|int)": "numbers", + "(bool)": "booleans", + "(json)": "json", + "(date)": "datetimes", + "(timestamp)": "timestamps", + } + for i := range tables.SourceTables { + go func(i int) { + defer wg.Done() + columns, err := bqc.GetColumns(ctx, tables.SourceTables[i]) + if err != nil { + log.Fatalf("Error fetching columns for table %s: %v\n", tables.SourceTables[i].Name, err) + return + } + mutex.Lock() + tables.SourceTables[i].Columns = columns + tables.SourceTables[i].DataTypeGroups = make(map[string][]shared.Column) + // Create a map of data types groups to hold column slices by data type + // This lets us group columns by their data type e.g. in templates + for j := range tables.SourceTables[i].Columns { + for k, v := range dataTypeGroupMap { + r, _ := regexp.Compile(fmt.Sprintf(`(?i).*%s.*`, k)) + if r.MatchString(tables.SourceTables[i].Columns[j].DataType) { + tables.SourceTables[i].DataTypeGroups[v] = append(tables.SourceTables[i].DataTypeGroups[v], tables.SourceTables[i].Columns[j]) + } + } + } + mutex.Unlock() + }(i) + } + wg.Wait() +} diff --git a/test_helpers.go b/test_helpers.go new file mode 100644 index 0000000..ab521bb --- /dev/null +++ b/test_helpers.go @@ -0,0 +1,33 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func CreateTempDbtProfile(t *testing.T) string { + content := []byte(` +elf: + target: dev + outputs: + dev: + type: snowflake + account: 123456.us-east-1 + user: legolas + database: mirkwood + schema: hall_of_thranduil +`) + tmpDir := t.TempDir() + err := os.Mkdir(filepath.Join(tmpDir, ".dbt"), 0755) + if err != nil { + t.Fatalf("Failed to create temporary .dbt directory: %v", err) + } + profilesFile := filepath.Join(tmpDir, ".dbt", "profiles.yml") + err = os.WriteFile(profilesFile, []byte(content), 0644) + if err != nil { + t.Fatalf("Failed to create temporary profiles.yml file: %v", err) + } + os.Setenv("HOME", tmpDir) + return tmpDir +}