From 608c39028091f298d8f806d1a809c0a659d2b92f Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 01:53:27 +0200 Subject: [PATCH 1/7] chore:prometeus based metrics (#236) * migrated the metrics from influx to prometeus * rebase * rebase --- README.md | 20 ++---- server/backend/influx/influxClient.go | 67 ------------------- server/backend/influx/iosLogging.go | 60 ----------------- server/backend/influxdb.go | 43 ------------ .../ios_apns/iosAPNsService.go | 9 +-- .../ios_device/iosDeviceService.go | 24 ++++--- .../iosRequestResponseService.go | 14 ++-- .../ios_scheduling/iosSchedulingService.go | 16 +++-- server/go.mod | 11 +-- server/go.sum | 49 ++++---------- server/main.go | 16 ++--- 11 files changed, 63 insertions(+), 266 deletions(-) delete mode 100644 server/backend/influx/influxClient.go delete mode 100644 server/backend/influx/iosLogging.go delete mode 100644 server/backend/influxdb.go diff --git a/README.md b/README.md index 825ce41f..350490e2 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,7 @@ docker compose up -d ``` The docker compose will start the server and a mariadb instance. The server will be available at `localhost:50051` and the mariadb instance at `localhost:3306`. -Additionally, docker creates the volume `campus-db-data` and `campus-influxdb-data` -to persist the data of the mariadb and influxdb instances. +Additionally, docker creates the volume `campus-db-data` to persist the data of the mariadb instances. ### Setting up the Database The mariadb schema can be installed by executing the following command inside the mariadb container: @@ -93,26 +92,15 @@ The following environment variables need to be set for the server to work proper * [REQUIRED] `DB_ROOT_PASSWORD`: The password of the root user. * [OPTIONAL] `DB_PORT`: The port of the database server. Defaults to `3306`. * [OPTIONAL] `SENTRY_DSN`: The Sentry [Data Source Name](https://sentry-docs-git-patch-1.sentry.dev/product/sentry-basics/dsn-explainer/) for reporting issues and crashes. -* **[InfluxDB [OPTIONAL]](#influxdb)**: - * [OPTIONAL] `INFLUXDB_USER`: The InfluxDB username to set for the systems initial superuser. - * [OPTIONAL] `INFLUXDB_PASSWORD`: The InfluxDB password to set for the systems initial superuser. - * [OPTIONAL] `INFLUXDB_ORG`: The InfluxDB organization to set for the systems initial organization. - * [OPTIONAL] `INFLUXDB_BUCKET`: The InfluxDB bucket to set for the systems initial bucket. - * [REQUIRED] `INFLUXDB_URL`: The InfluxDB URL to use for writing metrics. - * [REQUIRED] `INFLUXDB_ADMIN_TOKEN`: The InfluxDB admin token to use for authenticating with the InfluxDB server. If set initially the system will associate the token with the initial superuser. * **[iOS Push Notification Service [OPTIONAL]](#ios-push-notifications-service)**: * [REQUIRED] `APNS_KEY_ID`: The key ID of the APNs key => APNs Key needs to be downloaded from the Apple Developer Portal the name of the file also contains the key ID. * [REQUIRED] `APNS_TEAM_ID`: The team ID of the iOS app can be found in AppStoreConnect. * [REQUIRED] `APNS_P8_FILE_PATH`: The path to the APNs key file (e.g. `/secrets/AuthKey_XXXX.p8`) in the docker container. The file itself needs to exist in the same directory as the `docker-compose.yml` file and called `apns_auth_key.p8`. * [REQUIRED] `CAMPUS_API_TOKEN`: A token used to authenticate with TUMonline (used for example for the grades) -## InfluxDB -InfluxDB can be used to store metrics. - -If an InfluxDB instance is already set up, just the `INFLUXDB_URL` and the `INFLUXDB_ADMIN_TOKEN` environment variable needs to be set -to enable the metrics endpoint. -All the other environment variables are optional and only needed if the InfluxDB instance needs to be set up. -If `INFLUXDB_URL` or `INFLUXDB_ADMIN_TOKEN` are not set, the metrics endpoint will be disabled. +## Metrics +Our service uses prometheus to collect metrics to display in grafana. +To see the metrics we aggregate, head over to `http://localhost:50051/metrics` ## iOS Push Notifications Service The iOS Push Notifications Service can be used to send push notifications to iOS devices. diff --git a/server/backend/influx/influxClient.go b/server/backend/influx/influxClient.go deleted file mode 100644 index afb985d6..00000000 --- a/server/backend/influx/influxClient.go +++ /dev/null @@ -1,67 +0,0 @@ -// Package influx provides a handy wrapper around the influxdb client -package influx - -import ( - "crypto/sha256" - "encoding/hex" - "errors" - "os" - - "github.com/TUM-Dev/Campus-Backend/server/env" - influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "github.com/influxdata/influxdb-client-go/v2/api" - "github.com/influxdata/influxdb-client-go/v2/api/write" - log "github.com/sirupsen/logrus" -) - -var ( - influxOrg = os.Getenv("INFLUXDB_ORG") - influxBucket = os.Getenv("INFLUXDB_BUCKET") - - ErrInfluxClientNotConfigured = errors.New("influx client not configured") -) - -var Client *influxdb2.Client - -func GetClient() influxdb2.Client { - return *Client -} - -func SetClient(client *influxdb2.Client) { - Client = client -} - -func hashSha256(s string) string { - h := sha256.New() - - h.Write([]byte(s)) - - return hex.EncodeToString(h.Sum(nil)) -} - -func LogPoint(p *write.Point) { - w, err := writeAPI() - - if err != nil { - log.Warn("could not log because influx client is not configured") - return - } - - w.WritePoint(p) - - flushIfDevelop(w) -} - -func flushIfDevelop(write api.WriteAPI) { - if env.IsDev() { - write.Flush() - } -} - -func writeAPI() (api.WriteAPI, error) { - if Client != nil { - return GetClient().WriteAPI(influxOrg, influxBucket), nil - } - - return nil, ErrInfluxClientNotConfigured -} diff --git a/server/backend/influx/iosLogging.go b/server/backend/influx/iosLogging.go deleted file mode 100644 index 5183d957..00000000 --- a/server/backend/influx/iosLogging.go +++ /dev/null @@ -1,60 +0,0 @@ -package influx - -import influxdb2 "github.com/influxdata/influxdb-client-go/v2" - -func LogIOSRegisterDevice(deviceId string) { - hashedDeviceId := hashSha256(deviceId) - - p := influxdb2.NewPointWithMeasurement("ios_register_device"). - AddTag("device_id", hashedDeviceId) - - LogPoint(p) -} - -func LogIOSRemoveDevice(deviceId string) { - hashedDeviceId := hashSha256(deviceId) - - p := influxdb2.NewPointWithMeasurement("ios_remove_device"). - AddTag("device_id", hashedDeviceId) - - LogPoint(p) -} - -func LogIOSNewGrades(deviceId string, gradesCount int) { - hashedDeviceId := hashSha256(deviceId) - - p := influxdb2.NewPointWithMeasurement("ios_new_grades"). - AddTag("device_id", hashedDeviceId). - AddField("new_grades_count", gradesCount) - - LogPoint(p) -} - -func LogIOSSchedulingDevicesToUpdate(devicesToUpdateCount int, priority int) { - p := influxdb2.NewPointWithMeasurement("ios_scheduling_devices_to_update"). - AddTag("priority", string(rune(priority))). - AddField("devices_to_update", devicesToUpdateCount) - - LogPoint(p) -} - -func LogIOSBackgroundRequest(deviceId, requestType, reason string) { - hashedDeviceId := hashSha256(deviceId) - - p := influxdb2.NewPointWithMeasurement("ios_campus_token_request"). - AddTag("device_id", hashedDeviceId). - AddField("request_type", requestType). - AddField("notification_reason_response", reason) - - LogPoint(p) -} - -func LogIOSBackgroundRequestResponse(deviceId, requestType string) { - hashedDeviceId := hashSha256(deviceId) - - p := influxdb2.NewPointWithMeasurement("ios_campus_token_response"). - AddTag("device_id", hashedDeviceId). - AddField("request_type", requestType) - - LogPoint(p) -} diff --git a/server/backend/influxdb.go b/server/backend/influxdb.go deleted file mode 100644 index 28ae23d9..00000000 --- a/server/backend/influxdb.go +++ /dev/null @@ -1,43 +0,0 @@ -package backend - -import ( - "context" - "errors" - "os" - - "github.com/TUM-Dev/Campus-Backend/server/backend/influx" - influxdb2 "github.com/influxdata/influxdb-client-go/v2" -) - -const ( - // InfluxBatchSize is the number of points that are sent to influxdb at once - // This is to reduce the number of requests to the database - InfluxBatchSize = 500 -) - -var ( - influxToken = os.Getenv("INFLUXDB_TOKEN") - influxURL = os.Getenv("INFLUXDB_URL") - - ErrInfluxTokenNotConfigured = errors.New("influxdb token not configured") - ErrInfluxURLNotConfigured = errors.New("influxdb url not configured") -) - -func ConnectToInfluxDB() error { - if influxToken == "" { - return ErrInfluxTokenNotConfigured - } - - if influxURL == "" { - return ErrInfluxURLNotConfigured - } - - client := influxdb2.NewClientWithOptions(influxURL, influxToken, - influxdb2.DefaultOptions().SetBatchSize(InfluxBatchSize)) - - influx.SetClient(&client) - - _, err := influx.GetClient().Health(context.Background()) - - return err -} diff --git a/server/backend/ios_notifications/ios_apns/iosAPNsService.go b/server/backend/ios_notifications/ios_apns/iosAPNsService.go index ef79c2e1..42b4d02c 100644 --- a/server/backend/ios_notifications/ios_apns/iosAPNsService.go +++ b/server/backend/ios_notifications/ios_apns/iosAPNsService.go @@ -5,7 +5,6 @@ package ios_apns import ( "errors" - "github.com/TUM-Dev/Campus-Backend/server/backend/influx" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns/ios_apns_jwt" "github.com/TUM-Dev/Campus-Backend/server/model" log "github.com/sirupsen/logrus" @@ -27,7 +26,6 @@ type Service struct { // and the request ID. func (s *Service) RequestGradeUpdateForDevice(deviceID string) error { campusRequestToken, err := s.Repository.CreateCampusTokenRequest(deviceID) - if err != nil { log.WithError(err).Error("Could not create campus token request") return ErrCouldNotCreateTokenRequest @@ -35,15 +33,10 @@ func (s *Service) RequestGradeUpdateForDevice(deviceID string) error { notification := model.NewIOSNotificationPayload(deviceID).Background(campusRequestToken.RequestID, model.IOSBackgroundCampusTokenRequest) - res, err := s.Repository.SendBackgroundNotification(notification) - - if err != nil { + if _, err := s.Repository.SendBackgroundNotification(notification); err != nil { log.WithError(err).Error("Could not send background notification") return ErrCouldNotSendNotification } - - influx.LogIOSBackgroundRequest(deviceID, campusRequestToken.RequestType, res.Reason) - return nil } diff --git a/server/backend/ios_notifications/ios_device/iosDeviceService.go b/server/backend/ios_notifications/ios_device/iosDeviceService.go index 8f5209d7..8a92db49 100644 --- a/server/backend/ios_notifications/ios_device/iosDeviceService.go +++ b/server/backend/ios_notifications/ios_device/iosDeviceService.go @@ -3,10 +3,12 @@ package ios_device import ( pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev" - "github.com/TUM-Dev/Campus-Backend/server/backend/influx" "github.com/TUM-Dev/Campus-Backend/server/model" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) type Service struct { @@ -16,6 +18,12 @@ type Service struct { var ( ErrCouldNotRegisterDevice = status.Error(codes.Internal, "Could not register device") ErrCouldNotRemoveDevice = status.Error(codes.Internal, "Could not remove device") + + iosRegisteredDevices = promauto.NewGauge(prometheus.GaugeOpts{ + Subsystem: "ios", + Name: "ios_registered_devices", + Help: "The number of currently registered ios devices", + }) ) func (service *Service) RegisterDevice(request *pb.RegisterDeviceRequest) (*pb.RegisterDeviceReply, error) { @@ -24,13 +32,10 @@ func (service *Service) RegisterDevice(request *pb.RegisterDeviceRequest) (*pb.R PublicKey: request.GetPublicKey(), } - err := service.Repository.RegisterDevice(&device) - - if err != nil { + if err := service.Repository.RegisterDevice(&device); err != nil { return nil, ErrCouldNotRegisterDevice } - - influx.LogIOSRegisterDevice(request.GetDeviceId()) + iosRegisteredDevices.Inc() return &pb.RegisterDeviceReply{ DeviceId: device.DeviceID, @@ -38,14 +43,11 @@ func (service *Service) RegisterDevice(request *pb.RegisterDeviceRequest) (*pb.R } func (service *Service) RemoveDevice(request *pb.RemoveDeviceRequest) (*pb.RemoveDeviceReply, error) { - err := service.Repository.RemoveDevice(request.GetDeviceId()) - - if err != nil { + if err := service.Repository.RemoveDevice(request.GetDeviceId()); err != nil { return nil, ErrCouldNotRemoveDevice } - influx.LogIOSRemoveDevice(request.GetDeviceId()) - + iosRegisteredDevices.Dec() return &pb.RemoveDeviceReply{ DeviceId: request.GetDeviceId(), }, nil diff --git a/server/backend/ios_notifications/ios_request_response/iosRequestResponseService.go b/server/backend/ios_notifications/ios_request_response/iosRequestResponseService.go index 4bc9bfec..798ab588 100644 --- a/server/backend/ios_notifications/ios_request_response/iosRequestResponseService.go +++ b/server/backend/ios_notifications/ios_request_response/iosRequestResponseService.go @@ -6,9 +6,11 @@ package ios_request_response import ( "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev" "github.com/TUM-Dev/Campus-Backend/server/backend/campus_api" - "github.com/TUM-Dev/Campus-Backend/server/backend/influx" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" "github.com/TUM-Dev/Campus-Backend/server/model" @@ -28,6 +30,12 @@ var ( ErrInternalHandleGrades = status.Error(codes.Internal, "Could not handle grades request") ErrCouldNotGetDevice = status.Error(codes.Internal, "Could not get device") ErrAPNSNotActive = status.Error(codes.Internal, "APNS is not active") + + collectedNewGrades = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "ios_new_grades", + Help: "The total number of processed events", + Buckets: prometheus.LinearBuckets(0, 5, 5), + }) ) func (service *Service) HandleDeviceRequestResponse(request *pb.IOSDeviceRequestResponseRequest, apnsIsActive bool) (*pb.IOSDeviceRequestResponseReply, error) { @@ -43,8 +51,6 @@ func (service *Service) HandleDeviceRequestResponse(request *pb.IOSDeviceRequest return nil, ErrOutdatedRequest } - influx.LogIOSBackgroundRequestResponse(requestLog.DeviceID, requestLog.RequestType) - switch requestLog.RequestType { case model.IOSBackgroundCampusTokenRequest.String(): campusToken := request.GetPayload() @@ -94,6 +100,7 @@ func (service *Service) handleDeviceCampusTokenRequest(requestLog *model.IOSDevi } newGrades := compareAndFindNewGrades(apiGrades.Grades, oldGrades) + collectedNewGrades.Observe(float64(len(newGrades))) if len(newGrades) == 0 { log.Info("No new grades found") service.deleteRequestLog(requestLog) @@ -116,7 +123,6 @@ func (service *Service) handleDeviceCampusTokenRequest(requestLog *model.IOSDevi if len(newGrades) > 0 && len(oldGrades) > 0 { apnsRepository := ios_apns.NewRepository(service.Repository.DB, service.Repository.Token) sendGradesToDevice(device, newGrades, apnsRepository) - influx.LogIOSNewGrades(requestLog.DeviceID, len(newGrades)) } service.deleteRequestLog(requestLog) diff --git a/server/backend/ios_notifications/ios_scheduling/iosSchedulingService.go b/server/backend/ios_notifications/ios_scheduling/iosSchedulingService.go index 715684db..517c8351 100644 --- a/server/backend/ios_notifications/ios_scheduling/iosSchedulingService.go +++ b/server/backend/ios_notifications/ios_scheduling/iosSchedulingService.go @@ -5,7 +5,9 @@ package ios_scheduling import ( "sync" - "github.com/TUM-Dev/Campus-Backend/server/backend/influx" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_scheduled_update_log" @@ -18,6 +20,11 @@ const ( MaxRoutineCount = 10 ) +var devicesToUpdate = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "ios_scheduling_devices_to_update", + Help: "The numer of devices that should be updated for a given priority", +}) + type Service struct { Repository *Repository DevicesRepository *ios_device.Repository @@ -41,12 +48,9 @@ func (service *Service) HandleScheduledCron() error { log.WithError(err).Error("can't get devices") return err } + devicesToUpdate.Set(float64(len(devices))) - devicesLen := len(devices) - - influx.LogIOSSchedulingDevicesToUpdate(devicesLen, currentPriority.Priority) - - if devicesLen == 0 { + if len(devices) == 0 { log.Info("No devices to update") return nil } diff --git a/server/go.mod b/server/go.mod index e25b8ccb..4ed9ae94 100644 --- a/server/go.mod +++ b/server/go.mod @@ -12,11 +12,11 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 github.com/guregu/null v4.0.0+incompatible - github.com/influxdata/influxdb-client-go/v2 v2.12.3 github.com/makasim/sentryhook v0.4.2 github.com/microcosm-cc/bluemonday v1.0.25 github.com/mmcdole/gofeed v1.2.1 github.com/onrik/gorm-logrus v0.5.0 + github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.9.3 github.com/soheilhy/cmux v0.1.5 github.com/stretchr/testify v1.8.4 @@ -33,20 +33,23 @@ require ( github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mmcdole/goxpp v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect golang.org/x/image v0.5.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/server/go.sum b/server/go.sum index 86e36c83..4c8afde7 100644 --- a/server/go.sum +++ b/server/go.sum @@ -60,12 +60,16 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -83,15 +87,11 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= -github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -114,14 +114,12 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.17.0/go.mod h1:B82dxtBvxG0KaPD8/hfSV+VcHD+Lg/xUS4JuQn1P4cM= github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk= github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -131,8 +129,6 @@ github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnq github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= @@ -188,7 +184,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -232,7 +227,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -265,10 +259,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0IpXeMSkY/uJa/O/vC4= -github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0= -github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= -github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/httpexpect/v2 v2.3.1/go.mod h1:ICTf89VBKSD3KB0fsyyHviKF8G8hyepP0dOXJPWz3T0= github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5UybogqrXvJTBE= @@ -311,35 +301,26 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/echo/v4 v4.9.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailgun/raymond/v2 v2.0.46/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/makasim/sentryhook v0.4.2 h1:rqx637SCMMV5mAd9PpSg0OUIpZZaQOQZ6JHONOpUlRI= github.com/makasim/sentryhook v0.4.2/go.mod h1:AlAU2qjpS3R9bmpfVNcCVQw1tKkH8/lOPa2RQTk8Bzw= -github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v3 v3.8.0/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= @@ -406,7 +387,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -476,7 +465,6 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= @@ -519,8 +507,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -655,7 +641,6 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -664,7 +649,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -674,7 +658,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -687,7 +670,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -698,7 +680,6 @@ golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -727,7 +708,6 @@ golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -747,7 +727,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/server/main.go b/server/main.go index 8481f15d..c9f65f4b 100644 --- a/server/main.go +++ b/server/main.go @@ -4,7 +4,6 @@ import ( "context" "embed" "encoding/json" - "errors" "io/fs" "net" "net/http" @@ -13,6 +12,8 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/TUM-Dev/Campus-Backend/server/env" "github.com/makasim/sentryhook" @@ -45,17 +46,7 @@ var swagfs embed.FS func main() { setupTelemetry() - defer sentry.Flush(2 * time.Second) // make sure that sentry handles shutdowns gracefully - - // initializing connection to InfluxDB - err := backend.ConnectToInfluxDB() - if errors.Is(err, backend.ErrInfluxTokenNotConfigured) { - log.Warn("InfluxDB token not configured - continuing without InfluxDB") - } else if errors.Is(err, backend.ErrInfluxURLNotConfigured) { - log.Warn("InfluxDB url not configured - continuing without InfluxDB") - } else if err != nil { - log.WithError(err).Error("InfluxDB connection failed - health check failed") - } + defer sentry.Flush(10 * time.Second) // make sure that sentry handles shutdowns gracefully db := setupDB() @@ -82,6 +73,7 @@ func main() { httpMux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("healthy")) }) + httpMux.Handle("/metrics", promhttp.Handler()) static, _ := fs.Sub(swagfs, "swagger") httpMux.Handle("/", http.FileServer(http.FS(static))) From bed9f7954c3387fba53ca7e6d72749e2a460c782 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 02:06:48 +0200 Subject: [PATCH 2/7] adding a bunch of debug statements to the file handling code --- server/backend/cron/fileDownload.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/backend/cron/fileDownload.go b/server/backend/cron/fileDownload.go index 3e16f5a0..2d2ae866 100644 --- a/server/backend/cron/fileDownload.go +++ b/server/backend/cron/fileDownload.go @@ -35,18 +35,22 @@ func (c *CronService) fileDownloadCron() error { continue } + log.WithFields(fields).Info("ensuring file does not exist") if err := ensureFileDoesNotExist(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not ensure file does not exist") continue } + log.WithFields(fields).Info("downloading file") if err := downloadFile(file.URL.String, dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not download file") continue } + log.WithFields(fields).Info("maybeResizeImage") if err := maybeResizeImage(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not resize image") continue } + log.WithFields(fields).Info("updating downloaded") // everything went well => we can mark the file as downloaded if err = tx.Model(&model.Files{URL: file.URL}).Update("downloaded", true).Error; err != nil { log.WithError(err).WithFields(fields).Error("Could not set image to downloaded.") From 18e664fa4ea3fa08d1b22fd209d599f07dc2aa28 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 02:37:01 +0200 Subject: [PATCH 3/7] added different log output to hone in on issue --- server/backend/cron/fileDownload.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/backend/cron/fileDownload.go b/server/backend/cron/fileDownload.go index 2d2ae866..690efb10 100644 --- a/server/backend/cron/fileDownload.go +++ b/server/backend/cron/fileDownload.go @@ -35,22 +35,18 @@ func (c *CronService) fileDownloadCron() error { continue } - log.WithFields(fields).Info("ensuring file does not exist") if err := ensureFileDoesNotExist(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not ensure file does not exist") continue } - log.WithFields(fields).Info("downloading file") if err := downloadFile(file.URL.String, dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not download file") continue } - log.WithFields(fields).Info("maybeResizeImage") if err := maybeResizeImage(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not resize image") continue } - log.WithFields(fields).Info("updating downloaded") // everything went well => we can mark the file as downloaded if err = tx.Model(&model.Files{URL: file.URL}).Update("downloaded", true).Error; err != nil { log.WithError(err).WithFields(fields).Error("Could not set image to downloaded.") @@ -76,6 +72,7 @@ func maybeResizeImage(dstPath string) error { if err != nil { return err } + log.WithFields(log.Fields{"mime": mime.String(), "dstPath": dstPath}).Info("Detected mime type") if !strings.HasPrefix(mime.String(), "image/") { return nil } @@ -84,7 +81,9 @@ func maybeResizeImage(dstPath string) error { if err != nil { return err } + log.WithFields(log.Fields{"Bounds": img.Bounds()}).Info("image opened") resizedImage := imaging.Resize(img, 1280, 0, imaging.Lanczos) + log.WithFields(log.Fields{"Bounds": resizedImage.Bounds()}).Info("saving image") return imaging.Save(resizedImage, dstPath, imaging.JPEGQuality(75)) } From 83b4497b9d18996cef2932897c62b4219865cd0b Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 02:51:56 +0200 Subject: [PATCH 4/7] addded more debug statements --- server/backend/cron/fileDownload.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/backend/cron/fileDownload.go b/server/backend/cron/fileDownload.go index 690efb10..16e05b3f 100644 --- a/server/backend/cron/fileDownload.go +++ b/server/backend/cron/fileDownload.go @@ -43,10 +43,13 @@ func (c *CronService) fileDownloadCron() error { log.WithError(err).WithFields(fields).Warn("Could not download file") continue } + log.WithFields(fields).Info("pre maybeResizeImage") if err := maybeResizeImage(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not resize image") continue } + log.WithFields(fields).Info("post maybeResizeImage") + // everything went well => we can mark the file as downloaded if err = tx.Model(&model.Files{URL: file.URL}).Update("downloaded", true).Error; err != nil { log.WithError(err).WithFields(fields).Error("Could not set image to downloaded.") From 2cde057a31f536b77719853f8165a96d0b207d9d Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 03:13:56 +0200 Subject: [PATCH 5/7] added even more logging and tested a different resizer --- server/backend/cron/fileDownload.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/backend/cron/fileDownload.go b/server/backend/cron/fileDownload.go index 16e05b3f..67b5726c 100644 --- a/server/backend/cron/fileDownload.go +++ b/server/backend/cron/fileDownload.go @@ -35,21 +35,22 @@ func (c *CronService) fileDownloadCron() error { continue } + log.WithFields(fields).Info("ensuring file does not exist") if err := ensureFileDoesNotExist(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not ensure file does not exist") continue } + log.WithFields(fields).Info("downloading file") if err := downloadFile(file.URL.String, dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not download file") continue } - log.WithFields(fields).Info("pre maybeResizeImage") + log.WithFields(fields).Info("maybeResizeImage") if err := maybeResizeImage(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not resize image") continue } - log.WithFields(fields).Info("post maybeResizeImage") - + log.WithFields(fields).Info("updating downloaded") // everything went well => we can mark the file as downloaded if err = tx.Model(&model.Files{URL: file.URL}).Update("downloaded", true).Error; err != nil { log.WithError(err).WithFields(fields).Error("Could not set image to downloaded.") @@ -85,7 +86,7 @@ func maybeResizeImage(dstPath string) error { return err } log.WithFields(log.Fields{"Bounds": img.Bounds()}).Info("image opened") - resizedImage := imaging.Resize(img, 1280, 0, imaging.Lanczos) + resizedImage := imaging.Resize(img, 1280, 0, imaging.CatmullRom) log.WithFields(log.Fields{"Bounds": resizedImage.Bounds()}).Info("saving image") return imaging.Save(resizedImage, dstPath, imaging.JPEGQuality(75)) } From 86f5ffbbb2bd5026f6195a4ce3f8975db0f46460 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 03:24:37 +0200 Subject: [PATCH 6/7] Revert "adding a bunch of debug statements to the file handling code" This reverts commit bed9f7954c3387fba53ca7e6d72749e2a460c782. Revert "added different log output to hone in on issue" This reverts commit 18e664fa4ea3fa08d1b22fd209d599f07dc2aa28. Revert "addded more debug statements" This reverts commit 83b4497b9d18996cef2932897c62b4219865cd0b. Revert "added even more logging and tested a different resizer" This reverts commit 2cde057a31f536b77719853f8165a96d0b207d9d. --- server/backend/cron/fileDownload.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/server/backend/cron/fileDownload.go b/server/backend/cron/fileDownload.go index 67b5726c..3e16f5a0 100644 --- a/server/backend/cron/fileDownload.go +++ b/server/backend/cron/fileDownload.go @@ -35,22 +35,18 @@ func (c *CronService) fileDownloadCron() error { continue } - log.WithFields(fields).Info("ensuring file does not exist") if err := ensureFileDoesNotExist(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not ensure file does not exist") continue } - log.WithFields(fields).Info("downloading file") if err := downloadFile(file.URL.String, dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not download file") continue } - log.WithFields(fields).Info("maybeResizeImage") if err := maybeResizeImage(dstPath); err != nil { log.WithError(err).WithFields(fields).Warn("Could not resize image") continue } - log.WithFields(fields).Info("updating downloaded") // everything went well => we can mark the file as downloaded if err = tx.Model(&model.Files{URL: file.URL}).Update("downloaded", true).Error; err != nil { log.WithError(err).WithFields(fields).Error("Could not set image to downloaded.") @@ -76,7 +72,6 @@ func maybeResizeImage(dstPath string) error { if err != nil { return err } - log.WithFields(log.Fields{"mime": mime.String(), "dstPath": dstPath}).Info("Detected mime type") if !strings.HasPrefix(mime.String(), "image/") { return nil } @@ -85,9 +80,7 @@ func maybeResizeImage(dstPath string) error { if err != nil { return err } - log.WithFields(log.Fields{"Bounds": img.Bounds()}).Info("image opened") - resizedImage := imaging.Resize(img, 1280, 0, imaging.CatmullRom) - log.WithFields(log.Fields{"Bounds": resizedImage.Bounds()}).Info("saving image") + resizedImage := imaging.Resize(img, 1280, 0, imaging.Lanczos) return imaging.Save(resizedImage, dstPath, imaging.JPEGQuality(75)) } From 888c45775a9e78d8feaa729d8430628c5fa1b777 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 22 Sep 2023 03:26:11 +0200 Subject: [PATCH 7/7] increased the backends memory footprint --- .../charts/backend/templates/deployments/backend-v2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/charts/backend/templates/deployments/backend-v2.yaml b/deployment/charts/backend/templates/deployments/backend-v2.yaml index 3cb884b0..3434ba86 100644 --- a/deployment/charts/backend/templates/deployments/backend-v2.yaml +++ b/deployment/charts/backend/templates/deployments/backend-v2.yaml @@ -84,9 +84,9 @@ spec: resources: requests: cpu: 1000m - memory: 50Mi + memory: 100Mi limits: - memory: 200Mi + memory: 500Mi livenessProbe: httpGet: path: /health