Skip to content

Commit

Permalink
Collect information about git repository of SQLMesh project
Browse files Browse the repository at this point in the history
  • Loading branch information
lustefaniak committed Jun 18, 2024
1 parent dceaa51 commit 996b1cb
Show file tree
Hide file tree
Showing 6 changed files with 115 additions and 10 deletions.
9 changes: 9 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/getsynq/synq-sqlmesh/build"
"github.com/getsynq/synq-sqlmesh/git"
"github.com/getsynq/synq-sqlmesh/process"
"github.com/getsynq/synq-sqlmesh/sqlmesh"
"github.com/getsynq/synq-sqlmesh/synq"
Expand Down Expand Up @@ -32,13 +33,17 @@ var collectCmd = &cobra.Command{
Short: "Collect metadata information from SQLMesh and store to the file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {

gitContext := git.CollectGitContext(cmd.Context(), SQLMeshProjectDir)

err := WithSQLMesh(func(baseUrl url.URL) error {
logrus.Info("SQLMesh base URL:", baseUrl.String())

output, err := sqlmesh.CollectMetadata(baseUrl, createFileContentGlobFilter())
if err != nil {
return err
}
output.GitContext = gitContext

if err := synq.DumpMetadata(output, args[0]); err != nil {
return err
Expand All @@ -58,13 +63,17 @@ var uploadCmd = &cobra.Command{
Short: "Collect metadata information from SQLMesh and send to Synq API",
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {

gitContext := git.CollectGitContext(cmd.Context(), SQLMeshProjectDir)

err := WithSQLMesh(func(baseUrl url.URL) error {
logrus.Info("SQLMesh base URL:", baseUrl.String())

output, err := sqlmesh.CollectMetadata(baseUrl, createFileContentGlobFilter())
if err != nil {
return err
}
output.GitContext = gitContext

if SynqApiToken == "" {
return fmt.Errorf("SYNQ_TOKEN environment variable is not set")
Expand Down
62 changes: 62 additions & 0 deletions git/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package git

import (
ingestsqlmeshv1 "buf.build/gen/go/getsynq/api/protocolbuffers/go/synq/ingest/sqlmesh/v1"
"context"
"os/exec"
"strings"
"time"
)

func CollectGitContext(ctx context.Context, dir string) *ingestsqlmeshv1.GitContext {
if !commandExists("git") {
return nil
}

return &ingestsqlmeshv1.GitContext{
CloneUrl: getCloneUrl(ctx, dir),
Branch: getCurrentBranch(ctx, dir),
CommitSha: getCommitSHA(ctx, dir),
}
}

func getCurrentBranch(ctx context.Context, dir string) string {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "git", "rev-parse", "--abbrev-ref", "HEAD")
cmd.Dir = dir
out, err := cmd.Output()
if err != nil {
return ""
}
return strings.TrimSpace(string(out))
}

func getCommitSHA(ctx context.Context, dir string) string {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "git", "rev-parse", "HEAD")
cmd.Dir = dir
out, err := cmd.Output()
if err != nil {
return ""
}
return strings.TrimSpace(string(out))
}

func getCloneUrl(ctx context.Context, dir string) string {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "git", "remote", "get-url", "origin")
cmd.Dir = dir
out, err := cmd.Output()
if err != nil {
return ""
}
return strings.TrimSpace(string(out))
}

func commandExists(cmd string) bool {
_, err := exec.LookPath(cmd)
return err == nil
}
18 changes: 9 additions & 9 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,29 @@ module github.com/getsynq/synq-sqlmesh
go 1.21

require (
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240610094605-d52d1beff75d.1
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.1-20240610094605-d52d1beff75d.1
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240618221816-2eaa79a347e1.1
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.2-20240618221816-2eaa79a347e1.2
github.com/samber/lo v1.39.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/cobra v1.8.1
github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816
github.com/valyala/fasthttp v1.54.0
github.com/valyala/fasthttp v1.55.0
golang.org/x/oauth2 v0.21.0
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.1
google.golang.org/protobuf v1.34.2
)

require (
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 // indirect
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
)
20 changes: 20 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
buf.build/gen/go/bufbuild/protovalidate/grpc/go v1.4.0-20240508200655-46a4cf4ba109.1/go.mod h1:nNP0PbuwYRItihjXpU/brBwpv+wis+rGy2bLk8Qf4JI=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 h1:LEXWFH/xZ5oOWrC3oOtHbUyBdzRWMCPpAQmKC9v05mA=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1/go.mod h1:XF+P8+RmfdufmIYpGUC+6bF7S+IlmHDEnCrO3OXaUAQ=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 h1:cFrEG/pJch6t62+jqndcPXeTNkYcztS4tBRgNkR+drw=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw=
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240610094605-d52d1beff75d.1 h1:N1LRWdLl3lxp4WHzUbiCiVdWzjD5sOg1Hq9cHEdRZVM=
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240610094605-d52d1beff75d.1/go.mod h1:ydo7olain2hcXY8FkUpFMkzIE5ZXuvaH7B5FVmeWYQ8=
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240618221816-2eaa79a347e1.1 h1:5lTz66K3UuGpjK9UrkXX0VP9y63QNL7iVRPNCgMW8Is=
buf.build/gen/go/getsynq/api/grpc/go v1.4.0-20240618221816-2eaa79a347e1.1/go.mod h1:RmmWxe9FxjD2vTbsqkNBxLSIs3OQ6nQsxsLx+ALCkCM=
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.1-20240610094605-d52d1beff75d.1 h1:wZSOrIFz0C2aH6ol8u+9A+m4Ufc55aCtQbs3IGG3cDI=
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.1-20240610094605-d52d1beff75d.1/go.mod h1:6WdB1fXqAMYRc4JGGJgh44WrTNSYqp9jSdUDL0e9z2E=
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.1-20240618221816-2eaa79a347e1.1/go.mod h1:6WdB1fXqAMYRc4JGGJgh44WrTNSYqp9jSdUDL0e9z2E=
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.2-20240618221816-2eaa79a347e1.2 h1:EswkREZFLEQ9OzKaL4MLPxeCSQP3ygW8AYq1bMAV2bk=
buf.build/gen/go/getsynq/api/protocolbuffers/go v1.34.2-20240618221816-2eaa79a347e1.2/go.mod h1:1T8LGMjkTnanxxkyv9NgNSnKd6VqyQOpk7QPvsURBjU=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
Expand Down Expand Up @@ -1189,6 +1196,7 @@ github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand Down Expand Up @@ -1366,6 +1374,8 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
Expand Down Expand Up @@ -1421,6 +1431,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z
github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand All @@ -1444,6 +1456,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -1500,6 +1514,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM=
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
Expand Down Expand Up @@ -2145,6 +2161,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.
google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down Expand Up @@ -2217,6 +2235,8 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
1 change: 0 additions & 1 deletion process/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ func ExecuteCommand(ctx context.Context, cmdName string, args []string, opts ...
for _, opt := range opts {
opt(cmd)
}

stdOutReader, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "error creating StdoutPipe for Cmd", err)
Expand Down
15 changes: 15 additions & 0 deletions synq/synq.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ import (
"time"
)

type GitContextDump struct {
CloneUrl string `json:"clone_url"`
Branch string `json:"branch"`
CommitSha string `json:"commit_sha"`
}

type IngestMetadataRequestDump struct {
ApiMeta json.RawMessage `json:"api_meta"`
Models json.RawMessage `json:"models"`
Expand All @@ -27,6 +33,7 @@ type IngestMetadataRequestDump struct {
UploaderVersion string `json:"uploader_version"`
UploaderBuildTime string `json:"uploader_build_time"`
StateAt time.Time `json:"state_at"`
GitContext *GitContextDump `json:"git_context"`
}

func DumpMetadata(output *ingestsqlmeshv1.IngestMetadataRequest, filename string) error {
Expand All @@ -43,6 +50,14 @@ func DumpMetadata(output *ingestsqlmeshv1.IngestMetadataRequest, filename string
StateAt: output.StateAt.AsTime(),
}

if output.GitContext != nil {
outputRaw.GitContext = &GitContextDump{
CloneUrl: output.GitContext.CloneUrl,
Branch: output.GitContext.Branch,
CommitSha: output.GitContext.CommitSha,
}
}

asJson, err := json.MarshalIndent(outputRaw, "", " ")
if err != nil {
return err
Expand Down

0 comments on commit 996b1cb

Please sign in to comment.