Initial implementation of collecting SqlMesh metadata

.github/workflows/go.yml

@@ -0,0 +1,27 @@
+name: Go
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+          go-version-file: 'go.mod'
+
+      - name: Build
+        run: |
+          go generate
+          go build -v ./...
+
+      - name: Test
+        run: go test -v ./...

.github/workflows/release.yaml

@@ -0,0 +1,40 @@
+defaults:
+  run:
+    shell: bash
+env:
+  GONOPROXY: github.com/getsynq/*
+  GONOSUMDB: github.com/getsynq/*
+  GOPRIVATE: github.com/getsynq/*
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.21
+        go-version-file: 'go.mod'
+    - name: Configure git
+      run: git config --global --add url."[email protected]:".insteadOf "https://github.com/"
+    - name: Build
+      run: |
+        go generate
+        GOOS=darwin CGO_ENABLED=0 GOARCH=arm64 go build -o synq-sqlmesh-arm64-darwin main.go
+        GOOS=darwin CGO_ENABLED=0 GOARCH=amd64 go build -o synq-sqlmesh-amd64-darwin main.go
+        GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -o synq-sqlmesh-amd64-linux main.go
+        GOOS=windows CGO_ENABLED=0 GOARCH=amd64 go build -o synq-sqlmesh-amd64-windows.exe main.go
+    - name: Release
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: softprops/action-gh-release@v2
+      with:
+        files: |
+         synq-sqlmesh-arm64-darwin
+         synq-sqlmesh-amd64-darwin
+         synq-sqlmesh-amd64-linux
+         synq-sqlmesh-amd64-windows.exe
+name: release synq-dbt
+"on":
+  push:
+    tags:
+      - v*.*.*

.gitignore

@@ -0,0 +1,17 @@
+# Ignore everything
+*
+
+# But not these files...
+!/.gitignore
+
+!*.go
+!go.sum
+!go.mod
+
+!README.md
+!LICENSE
+
+# !Makefile
+
+# ...even if they are in subdirectories
+!*/

bin/version.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+git describe --tags --abbrev=0 > build/version.txt
+date +%FT%T%z > build/time.txt

build/build.go

@@ -0,0 +1,9 @@
+package build
+
+import _ "embed"
+
+//go:embed version.txt
+var Version string
+
+//go:embed time.txt
+var Time string

cmd/root.go

@@ -0,0 +1,105 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/getsynq/synq-sqlmesh/build"
+	"github.com/getsynq/synq-sqlmesh/process"
+	"github.com/getsynq/synq-sqlmesh/sqlmesh"
+	"github.com/getsynq/synq-sqlmesh/synq"
+	"github.com/spf13/cobra"
+	"net/url"
+	"os"
+	"strings"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "synq-sqlmesh",
+	Short: "Small utility to collect SqlMesh metadata information and upload it to Synq",
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version number of synq-sqlmesh",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("synq-sqlmesh %s (%s)", strings.TrimSpace(build.Version), strings.TrimSpace(build.Time))
+	},
+}
+
+var collectCmd = &cobra.Command{
+	Use:   "collect",
+	Short: "Collect metadata information from SqlMesh and store to the file",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		err := WithSqlMesh(func(baseUrl url.URL) error {
+			fmt.Println("SqlMesh base URL:", baseUrl.String())
+
+			output, err := sqlmesh.CollectMetadata(baseUrl)
+			if err != nil {
+				return err
+			}
+
+			if err := synq.DumpMetadata(output, args[0]); err != nil {
+				return err
+			}
+
+			return nil
+		})
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+	},
+}
+
+func WithSqlMesh(f func(baseUrl url.URL) error) error {
+	baseUrl := url.URL{
+		Host:   fmt.Sprintf("%s:%d", SqlMeshUiHost, SqlMeshUiPort),
+		Scheme: "http",
+	}
+	if SqlMeshUiStart {
+		ctx, cancelFn := context.WithCancel(context.Background())
+		defer cancelFn()
+		sqlMeshProcess, err := process.ExecuteCommand(ctx, SqlMesh, []string{"ui", "--host", SqlMeshUiHost, "--port", fmt.Sprintf("%d", SqlMeshUiPort)}, process.WithDir(SqlMeshProjectDir))
+		if err != nil {
+			return err
+		}
+
+		sqlmesh.WaitForSqlMeshToStart(baseUrl)
+
+		err = f(baseUrl)
+		_ = sqlMeshProcess.Kill()
+		if err != nil {
+			return err
+		}
+		return nil
+	} else {
+		return f(baseUrl)
+	}
+}
+
+var SynqApiEndpoint string = "https://developer.synq.io/"
+var SynqApiToken string = os.Getenv("SYNQ_TOKEN")
+var SqlMesh string = "sqlmesh"
+var SqlMeshProjectDir string = "."
+var SqlMeshUiStart bool = true
+var SqlMeshUiHost string = "localhost"
+var SqlMeshUiPort int = 8080
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&SynqApiToken, "synq-token", SynqApiToken, "Synq API token")
+	rootCmd.PersistentFlags().StringVar(&SynqApiEndpoint, "synq-endpoint", SynqApiEndpoint, "Synq API endpoint URL")
+	rootCmd.PersistentFlags().StringVar(&SqlMesh, "sqlmesh-cmd", SqlMesh, "SqlMesh launcher location")
+	rootCmd.PersistentFlags().StringVar(&SqlMeshProjectDir, "sqlmesh-project-dir", SqlMeshProjectDir, "Location of SqlMesh project directory")
+	rootCmd.PersistentFlags().BoolVar(&SqlMeshUiStart, "sqlmesh-ui-start", SqlMeshUiStart, "Launch and control SqlMesh UI process automatically")
+	rootCmd.PersistentFlags().StringVar(&SqlMeshUiHost, "sqlmesh-ui-host", SqlMeshUiHost, "SqlMesh UI host")
+	rootCmd.PersistentFlags().IntVar(&SqlMeshUiPort, "sqlmesh-ui-port", SqlMeshUiPort, "SqlMesh UI port")
+
+	rootCmd.AddCommand(collectCmd)
+	rootCmd.AddCommand(versionCmd)
+
+}
+
+func Execute() error {
+	return rootCmd.Execute()
+}

go.mod

@@ -0,0 +1,20 @@
+module github.com/getsynq/synq-sqlmesh
+
+go 1.21
+
+require (
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
+	github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816
+	github.com/valyala/fasthttp v1.54.0
+)
+
+require (
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)

go.sum

@@ -0,0 +1,54 @@
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
+github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 h1:J6v8awz+me+xeb/cUTotKgceAYouhIB3pjzgRd6IlGk=
+github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816/go.mod h1:tzym/CEb5jnFI+Q0k4Qq3+LvRF4gO3E2pxS8fHP8jcA=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
+github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"github.com/getsynq/synq-sqlmesh/cmd"
+	"github.com/sirupsen/logrus"
+	easy "github.com/t-tomalak/logrus-easy-formatter"
+)
+
+//go:generate bash bin/version.sh
+
+func main() {
+	logrus.SetFormatter(&easy.Formatter{
+		TimestampFormat: "15:04:05",
+		LogFormat:       "%time%  %msg%\n",
+	})
+
+	cmd.Execute()
+}