Continue integration support for build (#1)

* fix build script
* now create space
* changed make source to warn on testing
* ci automation now functions

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: CI Workflow
+name: Build Status
 
 on: [push]
 
@@ -17,13 +17,15 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Build, Setup
-      run: sudo apt -y install golang
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
 
     - name: Verify build
-      run: ./script/build
+      run: ./scripts/build
 
     - name: Run tests
-      run: ./script/test
+      run: sleep 70 && ./scripts/test && ./scripts/cover
 
     - name: Run linting
-      run: ./script/lint
+      run: ./scripts/lint

README.md

@@ -1,5 +1,7 @@
 # dremio-batch-execute
 
+[![build](https://github.com/rsvihladremio/dremio-batch-execute/actions/workflows/ci.yaml/badge.svg)](https://github.com/rsvihladremio/dremio-batch-execute/actions/workflows/ci.yaml)
+
 Batch run Dremio a list of queries with limits on throughput, concurrency and support for resuming
 
 ## How to run

cmd/dremio-batch-execute/main_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/rsvihladremio/dremio-batch-execute/pkg/conf"
+	"github.com/rsvihladremio/dremio-batch-execute/pkg/protocol"
 )
 
 func cleanup(t *testing.T) {
@@ -51,14 +52,30 @@ func cleanup(t *testing.T) {
 	if err != nil {
 		t.Fatalf("cleanup failure: %v", err)
 	}
+
 }
 
 func setup(t *testing.T) {
 	t.Helper()
+	eng, err := protocol.NewHTTPEngine(conf.ProtocolArgs{
+		User:     "dremio",
+		Password: "dremio123",
+		URL:      "http://localhost:9047",
+		Timeout:  time.Second * 5,
+		SkipSSL:  true,
+	})
+	if err != nil {
+		t.Fatalf("cleanup failure on new http engine: %v", err)
+	}
+	if err := eng.MakeSource("a"); err != nil {
+		t.Logf("WARN: unable to make source: %v", err)
+	}
+	time.Sleep(100 * time.Millisecond)
+
 	srcFile := "testdata/create.sql"
 
 	progressFile := filepath.Join(t.TempDir(), "progress-create.txt")
-	err := Execute(conf.Args{
+	err = Execute(conf.Args{
 		DremioUsername:   "dremio",
 		DremioPassword:   "dremio123",
 		DremioURL:        "http://localhost:9047",
@@ -72,6 +89,7 @@ func setup(t *testing.T) {
 		t.Fatal(err)
 	}
 	time.Sleep(500 * time.Millisecond)
+
 }
 
 func TestExecute(t *testing.T) {

pkg/output/output.go

@@ -10,7 +10,7 @@ import (
 )
 
 var Version = "dev"
-var Sha256 = "unknown"
+var GitSha = "unknown"
 
 type QueryResults struct {
 	Completed int
@@ -25,7 +25,7 @@ func LogQueriesCompleted(q QueryResults) {
 }
 
 func LogStartMessage(args conf.Args) error {
-	log.Printf("dbe version: %v-%v", Version, Sha256)
+	log.Printf("dbe version: %v-%v", Version, GitSha)
 	log.Printf("parameters")
 	log.Printf("----------")
 	fullSourcePath, err := filepath.Abs(args.SourceQueryFile)

pkg/protocol/http.go

@@ -38,6 +38,7 @@ type HTTPProtocolEngine struct {
 	client              http.Client
 	queryTimeoutMinutes int
 	queryURL            string
+	sourceURL           string
 	queryStatusURL      string
 }
 
@@ -46,6 +47,63 @@ func (h *HTTPProtocolEngine) Name() string {
 	return "HTTP"
 }
 
+func (h *HTTPProtocolEngine) MakeSource(sourceName string) error {
+	jsonBody := fmt.Sprintf(`{
+		"metadataPolicy": {       
+			"authTTLMs":86400000,
+			"namesRefreshMs":3600000,
+			"datasetRefreshAfterMs": 3600000,
+			"datasetExpireAfterMs": 10800000,
+			"datasetUpdateMode":"PREFETCH_QUERIED",
+			"deleteUnavailableDatasets": true,
+			"autoPromoteDatasets": true
+			},
+		"entityType": "source",
+		"type": "NAS",
+		"config": {"path": "/tmp/"},
+		"name": "%v"
+	}
+`, sourceName)
+	req, err := http.NewRequest(http.MethodPost, h.sourceURL, bytes.NewBuffer([]byte(jsonBody)))
+	if err != nil {
+		return fmt.Errorf("unable to create request %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", h.token)
+
+	res, err := h.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed sending login request: %w", err)
+	}
+	resBody, err := io.ReadAll(res.Body)
+	if err != nil {
+		return fmt.Errorf("could not read response body: %w", err)
+	}
+	var resultMap map[string]interface{}
+	err = json.Unmarshal(resBody, &resultMap)
+	if err != nil {
+		return fmt.Errorf("could not read %v json %w", string(resBody), err)
+	}
+
+	v, ok := resultMap["id"]
+	if ok {
+		token := fmt.Sprintf("%v", v)
+		if token == "" {
+			return errors.New("blank id cannot proceed")
+		}
+		// TODO: add stats on job status at some point
+		lastState, err := h.checkQueryStatus(fmt.Sprintf("%v", v))
+		if err != nil {
+			return err
+		}
+		if lastState != "COMPLETED" {
+			return fmt.Errorf("failed with state of %v", lastState)
+		}
+		return nil
+	}
+	return fmt.Errorf("no job id in response %#v so failing the query", resultMap)
+}
+
 func (h *HTTPProtocolEngine) Execute(query string) error {
 	data := map[string]string{
 		"sql": query,
@@ -151,6 +209,7 @@ func NewHTTPEngine(a conf.ProtocolArgs) (*HTTPProtocolEngine, error) {
 	return &HTTPProtocolEngine{
 		token:               fmt.Sprintf("_dremio%v", token),
 		queryURL:            fmt.Sprintf("%v/api/v3/sql", a.URL),
+		sourceURL:           fmt.Sprintf("%v/api/v3/catalog", a.URL),
 		queryStatusURL:      fmt.Sprintf("%v/api/v3/job", a.URL),
 		client:              client,
 		queryTimeoutMinutes: 60,

scripts/build

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-go build -o ./bin/dremio-batch-execute.
+go build -o ./bin/dremio-batch-execute ./cmd/dremio-batch-execute/

scripts/cover

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+go tool cover -func=coverage.out

scripts/release

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# scripts/release: build binaries in all supported platforms and upload them with the gh client
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+BIN_NAME="dremio-batch-execute"
+echo "Checking if gh is installed…"
+date "+%H:%M:%S"
+
+if ! type "gh" > /dev/null; then
+    echo "gh not found installing…"
+    date "+%H:%M:%S"
+    OS="`uname`"
+    case $OS in
+      'Linux')
+        if type "apt" > /dev/null; then
+            echo "detected debian based system installing via apt"
+            curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+            echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+            sudo apt update -y
+            sudo apt install gh -y 
+        fi
+        if type "yum" > /dev/null; then
+            echo "detected rpm based system installing via yum"
+            sudo dnf install 'dnf-command(config-manager)' -y
+            sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+            sudo dnf install gh -y
+        fi
+        ;;
+      'WindowsNT')
+        echo "install gh and try again https://github.com/cli/cli/releases"
+        exit 1
+        ;;
+      'Darwin') 
+        if type "brew" > /dev/null; then
+            brew install gh
+        else
+            echo "install homebrew to have zip https://brew.sh/"
+            exit 1
+        fi
+        ;;
+      *) ;;
+      esac
+fi
+
+# this is also set in scripts/build-release and is a copy paste
+GIT_SHA=`git rev-parse --short HEAD`
+VERSION=$1
+
+./scripts/release-build $VERSION
+
+gh release create $VERSION --title $VERSION --generate-notes ./bin/$BIN_NAME-windows-arm64.zip  ./bin/$BIN_NAME-windows-amd64.zip ./bin/$BIN_NAME-darwin-arm64.zip ./bin/$BIN_NAME-darwin-amd64.zip ./bin/$BIN_NAME-linux-arm64.zip ./bin/$BIN_NAME-linux-amd64.zip

scripts/release-build

@@ -0,0 +1,71 @@
+#!/bin/sh
+
+# scripts/release-build: build binaries in all supported platforms
+
+echo "Checking if zip is installed…"
+date "+%H:%M:%S"
+BIN_NAME="dremio-batch-execute"
+
+if ! type "zip" > /dev/null; then
+    echo "zip not found installing…"
+    date "+%H:%M:%S"
+    OS="`uname`"
+    case $OS in
+      'Linux')
+        if type "apt" > /dev/null; then
+            echo "detected debian based system installing via apt"
+            sudo apt install zip
+        fi
+        if type "yum" > /dev/null; then
+            echo "detected rpm based system installing via yum"
+            sudo yum install zip
+        fi
+        ;;
+      'WindowsNT')
+        echo "install gnu zip and try again http://gnuwin32.sourceforge.net/packages/zip.htm"
+        exit 1
+        ;;
+      'Darwin') 
+        if type "brew" > /dev/null; then
+            brew install zip
+        else
+            echo "install homebrew to have zip https://brew.sh/"
+            exit 1
+        fi
+        ;;
+      *) ;;
+      esac
+fi
+# this is also set in script/build and is a copy paste
+GIT_SHA=`git rev-parse --short HEAD`
+VERSION=$1
+LDFLAGS="-X github.com/rsvihladremio/$BIN_NAME/pkg/output.GitSha=$GIT_SHA -X github.com/rsvihladremio/$BIN_NAME/pkg/output.Version=$VERSION"
+
+echo "Cleaning bin folder…"
+date "+%H:%M:%S"
+./scripts/clean
+
+echo "Building linux-amd64…"
+date "+%H:%M:%S"
+GOOS=linux GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-linux-amd64.zip ./bin/$BIN_NAME
+echo "Building linux-arm64…"
+date "+%H:%M:%S"
+GOOS=linux GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-linux-arm64.zip ./bin/$BIN_NAME
+echo "Building darwin-os-x-amd64…"
+date "+%H:%M:%S"
+GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-darwin-amd64.zip ./bin/$BIN_NAME
+echo "Building darwin-os-x-arm64…"
+date "+%H:%M:%S"
+GOOS=darwin GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-darwin-arm64.zip ./bin/$BIN_NAME 
+echo "Building windows-amd64…"
+date "+%H:%M:%S"
+GOOS=windows GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME.exe ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-windows-amd64.zip ./bin/$BIN_NAME.exe
+echo "Building windows-arm64…"
+date "+%H:%M:%S"
+GOOS=windows GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/$BIN_NAME.exe ./cmd/$BIN_NAME/
+zip ./bin/$BIN_NAME-windows-arm64.zip ./bin/$BIN_NAME.exe 

scripts/test

@@ -1,4 +1,3 @@
 #!/bin/sh
 
-go test -race -cover -coverprofile=coverage.out ./...
-go tool cover -func=coverage.out
+go test -race -cover -coverprofile=coverage.out ./...