Webhook Controller

.github/workflows/t1-failure-no-template.yml

@@ -0,0 +1,18 @@
+name: 't1-failure-no-template'
+on:
+  workflow_dispatch:
+
+jobs:
+  testJob:
+    runs-on: [ 
+      "self-hosted", 
+      "anka", 
+      "anka-template:c092c6f6-198c-470f-9526-9c998efe7ab5", 
+      "anka-template-tag:vanilla+port-forward-22+brew-git",
+      "run-id:${{ github.run_id }}", 
+      "unique-id:1"
+    ]
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          echo "hello"

.github/workflows/t1-with-tag-1-failure.yml → .github/workflows/t1-failure-tag-1-in-vm.yml

@@ -1,4 +1,4 @@
-name: 't1-with-tag-1-failure'
+name: 't1-failure-tag-1-in-vm'
 on:
   workflow_dispatch:
 

.github/workflows/t2-dual-without-tag.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: [ 
       "self-hosted", 
       "anka", 
-      "anka-template:ae1d3b9b-314e-44bc-9e6b-0b8e2bbc823b",
+      "anka-template:d792c6f6-198c-470f-9526-9c998efe7ab4",
       "run-id:${{ github.run_id }}", 
       "unique-id:1"
     ]
@@ -22,7 +22,7 @@ jobs:
     runs-on: [ 
       "self-hosted", 
       "anka", 
-      "anka-template:ae1d3b9b-314e-44bc-9e6b-0b8e2bbc823b",
+      "anka-template:c0847bc9-5d2d-4dbc-ba6a-240f7ff08032",
       "run-id:${{ github.run_id }}", 
       "unique-id:2"
     ]

README.md

@@ -477,3 +477,7 @@ Any of the services you run are done from within worker context. Each service al
 For example, the `github` plugin will update the metrics for the service it is running in to be `running`, `pulling`, and `idle` when it is done or has yet to pick up a new job. To do this, it uses `metrics.UpdateService` with the worker and service context. See `github` plugin for an example.
 
 But metrics.UpdateService can also update things like `LastSuccess`, and `LastFailure`. See `metrics.UpdateService` for more information.
+
+
+## Copyright
+All rights reserved, Veertu Inc.

anklet.code-workspace

@@ -5,6 +5,9 @@
 		},
 		{
 			"path": "../ci-tools"
+		},
+		{
+			"path": "../webhooks"
 		}
 	],
 	"settings": {}

go.mod

@@ -3,11 +3,10 @@ module github.com/veertuinc/anklet
 go 1.22.2
 
 require (
-	github.com/bradleyfalzon/ghinstallation/v2 v2.10.0
+	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
 	github.com/gofri/go-github-ratelimit v1.1.0
-	github.com/google/go-github/v61 v61.0.0
+	github.com/google/go-github/v63 v63.0.0
 	github.com/google/uuid v1.6.0
-	github.com/norsegaud/go-daemon v0.1.10
 	github.com/redis/go-redis/v9 v9.5.1
 	github.com/shirou/gopsutil/v3 v3.24.4
 	gopkg.in/yaml.v2 v2.4.0
@@ -18,9 +17,8 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-github/v60 v60.0.0 // indirect
+	github.com/google/go-github/v62 v62.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
@@ -31,3 +29,5 @@ require (
 )
 
 replace github.com/veertuinc/anklet/plugins/github => ./plugins/github
+
+replace github.com/veertuinc/anklet/plugins/controllers => ./plugins/controllers

go.sum

@@ -1,5 +1,5 @@
-github.com/bradleyfalzon/ghinstallation/v2 v2.10.0 h1:XWuWBRFEpqVrHepQob9yPS3Xg4K3Wr9QCx4fu8HbUNg=
-github.com/bradleyfalzon/ghinstallation/v2 v2.10.0/go.mod h1:qoGA4DxWPaYTgVCrmEspVSjlTu4WYAiSxMIhorMRXXc=
+github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53AL4ED5MzgEMwGTZag=
+github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -22,20 +22,16 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8=
-github.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk=
-github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go=
-github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY=
+github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4=
+github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4=
+github.com/google/go-github/v63 v63.0.0 h1:13xwK/wk9alSokujB9lJkuzdmQuVn2QCPeck76wR3nE=
+github.com/google/go-github/v63 v63.0.0/go.mod h1:IqbcrgUmIcEaioWrGYei/09o+ge5vhffGOcxrO0AfmA=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
-github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/norsegaud/go-daemon v0.1.10 h1:JnYLPJ1uHU1SsZ29iDkXQbzvy0Fxecy2o2buDgnI528=
-github.com/norsegaud/go-daemon v0.1.10/go.mod h1:Xfzc5NRusltO7M151oKbmC4bBXBi44uFviKbP2BUxQ8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=

internal/config/config.go

@@ -63,6 +63,9 @@ type Service struct {
 	AppID          int      `yaml:"app_id"`
 	InstallationID int64    `yaml:"installation_id"`
 	Workflows      Workflow `yaml:"workflows"`
+	Port           string   `yaml:"port"`
+	Secret         string   `yaml:"secret"`
+	HookID         int64    `yaml:"hook_id"`
 }
 
 func LoadConfig(configPath string) (Config, error) {
@@ -145,14 +148,14 @@ func LoadInEnvs(config Config) (Config, error) {
 	if workDir != "" {
 		config.WorkDir = workDir
 	}
-	pidFileDir := os.Getenv("ANKLET_PID_FILE_DIR")
-	if pidFileDir != "" {
-		config.PidFileDir = pidFileDir
-	}
-	logFileDir := os.Getenv("ANKLET_LOG_FILE_DIR")
-	if logFileDir != "" {
-		config.Log.FileDir = logFileDir
-	}
+	// pidFileDir := os.Getenv("ANKLET_PID_FILE_DIR")
+	// if pidFileDir != "" {
+	// 	config.PidFileDir = pidFileDir
+	// }
+	// logFileDir := os.Getenv("ANKLET_LOG_FILE_DIR")
+	// if logFileDir != "" {
+	// 	config.Log.FileDir = logFileDir
+	// }
 	return config, nil
 }
 

internal/database/database.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -112,3 +113,27 @@ func AddUniqueRunKey(ctx context.Context) (bool, error) {
 	}
 	return true, errors.New("unique run key already exists")
 }
+
+func UnwrapPayload[T any](payload string) (T, error, error) {
+	var wrappedPayload map[string]interface{}
+	var t T
+	err := json.Unmarshal([]byte(payload), &wrappedPayload)
+	if err != nil {
+		return t, err, nil
+	}
+	payloadBytes, err := json.Marshal(wrappedPayload["payload"])
+	if err != nil {
+		return t, err, nil
+	}
+	if err := json.Unmarshal(payloadBytes, &t); err != nil {
+		return t, err, nil
+	}
+	jobType, ok := wrappedPayload["type"].(string)
+	if !ok {
+		return t, nil, errors.New("job type not found or not a string")
+	}
+	if jobType == "anka.VM" {
+		return t, nil, errors.New("job type " + jobType + " is not what we expect")
+	}
+	return t, nil, nil
+}

internal/github/client.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/google/go-github/v61/github"
+	"github.com/google/go-github/v63/github"
 	"github.com/veertuinc/anklet/internal/config"
 )
 

internal/metrics/aggregator.go

@@ -15,7 +15,7 @@ import (
 
 // Start runs the HTTP server
 func (s *Server) StartAggregatorServer(workerCtx context.Context, logger *slog.Logger) {
-	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/metrics/v1", func(w http.ResponseWriter, r *http.Request) {
 		databaseContainer, err := database.GetDatabaseFromContext(workerCtx)
 		if err != nil {
 			logger.ErrorContext(workerCtx, "error getting database client from context", "error", err)
@@ -30,6 +30,10 @@ func (s *Server) StartAggregatorServer(workerCtx context.Context, logger *slog.L
 			http.Error(w, "unsupported format, please use '?format=json' or '?format=prometheus'", http.StatusBadRequest)
 		}
 	})
+	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotImplemented)
+		w.Write([]byte("please use /metrics/v1"))
+	})
 	http.ListenAndServe(":"+s.Port, nil)
 }
 

internal/metrics/metrics.go

@@ -204,7 +204,7 @@ func NewServer(port string) *Server {
 
 // Start runs the HTTP server
 func (s *Server) Start(parentCtx context.Context, logger *slog.Logger) {
-	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/metrics/v1", func(w http.ResponseWriter, r *http.Request) {
 		// update system metrics each call
 		metricsData := GetMetricsDataFromContext(parentCtx)
 		UpdateSystemMetrics(parentCtx, logger, metricsData)
@@ -217,6 +217,10 @@ func (s *Server) Start(parentCtx context.Context, logger *slog.Logger) {
 			http.Error(w, "unsupported format, please use '?format=json' or '?format=prometheus'", http.StatusBadRequest)
 		}
 	})
+	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotImplemented)
+		w.Write([]byte("please use /metrics/v1"))
+	})
 	http.ListenAndServe(":"+s.Port, nil)
 }
 

internal/run/run.go

@@ -6,18 +6,39 @@ import (
 
 	"github.com/veertuinc/anklet/internal/config"
 	"github.com/veertuinc/anklet/internal/logging"
+	"github.com/veertuinc/anklet/internal/metrics"
+	github_controller "github.com/veertuinc/anklet/plugins/controllers"
 	"github.com/veertuinc/anklet/plugins/github"
 )
 
-func Plugin(workerCtx context.Context, serviceCtx context.Context, logger *slog.Logger) {
+func Plugin(workerCtx context.Context, serviceCtx context.Context, serviceCancel context.CancelFunc, logger *slog.Logger, firstServiceStarted chan bool) {
 	service := config.GetServiceFromContext(serviceCtx)
 	// fmt.Printf("%+v\n", service)
 	serviceCtx = logging.AppendCtx(serviceCtx, slog.String("plugin", service.Plugin))
 	if service.Plugin == "" {
 		panic("plugin is not set in yaml:services:" + service.Name + ":plugin")
 	}
 	if service.Plugin == "github" {
-		github.Run(workerCtx, serviceCtx, logger)
+		for {
+			select {
+			case <-serviceCtx.Done():
+				serviceCancel()
+				return
+			default:
+				// notify the main thread that the service has started
+				select {
+				case <-firstServiceStarted:
+				default:
+					close(firstServiceStarted)
+				}
+				github.Run(workerCtx, serviceCtx, serviceCancel, logger)
+				metrics.UpdateService(workerCtx, serviceCtx, logger, metrics.Service{
+					Status: "idle",
+				})
+			}
+		}
+	} else if service.Plugin == "github_controller" {
+		github_controller.Run(workerCtx, serviceCtx, serviceCancel, logger, firstServiceStarted)
 	} else {
 		panic("plugin not found: " + service.Plugin)
 	}