From 263ccda494cb958368d9257ccedbc43545e06874 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 10 Oct 2023 23:09:35 +0200 Subject: [PATCH 01/62] added a rough testcase that assures that migrations don't fail without us noticing --- .github/workflows/main.yml | 25 ++++++++++++++++++++++++- server/migrate.go | 11 +++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 server/migrate.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56123e25..0ce9f8e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,9 +17,32 @@ jobs: - name: run tests run: go test -v ./... working-directory: ./server + test_migrations: + runs-on: ubuntu-latest + services: + mariadb: + image: bitnami/mariadb:latest + ports: + - 3306:3306 + env: + MARIADB_ROOT_PASSWORD: super_secret_pa$$w0rd + MARIADB_DATABASE: campus_db + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache-dependency-path: | + server/go.sum + - name: run migration + run: go run migrate.go + working-directory: ./server + env: + DB_DSN: root:super_secret_pa$$w0rd@tcp(db:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + ENVIRONMENT: dev build: runs-on: ubuntu-latest - needs: [test] + needs: [test, test_migrations] steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/server/migrate.go b/server/migrate.go new file mode 100644 index 00000000..35f94ef1 --- /dev/null +++ b/server/migrate.go @@ -0,0 +1,11 @@ +package main + +import ( + log "github.com/sirupsen/logrus" +) + +func main() { + if db := setupDB(); db != nil { + log.Fatal("db is nil. How did this happen?") + } +} From 7fe2da230e7db014b045efc7cd77858d5ff3ff7a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:37:37 +0200 Subject: [PATCH 02/62] fixed a build error --- server/main.go | 63 +++---------------------------------------- server/migrate.go | 3 ++- server/utils/utils.go | 63 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 60 deletions(-) create mode 100644 server/utils/utils.go diff --git a/server/main.go b/server/main.go index 5e014393..a174efd3 100644 --- a/server/main.go +++ b/server/main.go @@ -8,22 +8,18 @@ import ( "net" "net/http" "net/textproto" - "os" "strings" "time" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/TUM-Dev/Campus-Backend/server/utils" - "github.com/TUM-Dev/Campus-Backend/server/env" - "github.com/makasim/sentryhook" + "github.com/prometheus/client_golang/prometheus/promhttp" pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev" "github.com/TUM-Dev/Campus-Backend/server/backend" "github.com/TUM-Dev/Campus-Backend/server/backend/cron" - "github.com/TUM-Dev/Campus-Backend/server/backend/migration" "github.com/getsentry/sentry-go" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "github.com/onrik/gorm-logrus" log "github.com/sirupsen/logrus" "github.com/soheilhy/cmux" "golang.org/x/sync/errgroup" @@ -32,8 +28,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "gorm.io/driver/mysql" - "gorm.io/gorm" ) const httpPort = ":50051" @@ -45,10 +39,10 @@ var Version = "dev" var swagfs embed.FS func main() { - setupTelemetry() + utils.SetupTelemetry(Version) defer sentry.Flush(10 * time.Second) // make sure that sentry handles shutdowns gracefully - db := setupDB() + db := utils.SetupDB() // Create any other background services (these shouldn't do any long-running work here) cronService := cron.New(db) @@ -118,55 +112,6 @@ func main() { } } -// setupDB connects to the database and migrates it if necessary -func setupDB() *gorm.DB { - dbHost := os.Getenv("DB_DSN") - if dbHost == "" { - log.Fatal("Failed to start! The 'DB_DSN' environment variable is not defined. Take a look at the README.md for more details.") - } - - log.Info("Connecting to dsn") - db, err := gorm.Open(mysql.Open(dbHost), &gorm.Config{Logger: gorm_logrus.New()}) - if err != nil { - log.WithError(err).Fatal("failed to connect database") - } - - // Migrate the schema - // currently not activated as - if err := migration.New(db, false).Migrate(); err != nil { - log.WithError(err).Fatal("Failed to migrate database") - } - return db -} - -// setupTelemetry initializes our telemetry stack -// - sentry to be connected with log -// - logrus to -func setupTelemetry() { - environment := "development" - log.SetLevel(log.TraceLevel) - if env.IsProd() { - log.SetLevel(log.InfoLevel) - environment = "production" - log.SetFormatter(&log.JSONFormatter{}) // simpler to query but harder to parse in the console - } - - if sentryDSN := os.Getenv("SENTRY_DSN"); sentryDSN != "" { - if err := sentry.Init(sentry.ClientOptions{ - Dsn: sentryDSN, - AttachStacktrace: true, - Release: Version, - Dist: Version, // see https://github.com/getsentry/sentry-react-native/issues/516 why this is equal - Environment: environment, - }); err != nil { - log.WithError(err).Error("Sentry initialization failed") - } - log.AddHook(sentryhook.New([]log.Level{log.PanicLevel, log.FatalLevel, log.ErrorLevel, log.WarnLevel})) - } else { - log.Info("continuing without sentry") - } -} - // addMethodNameInterceptor adds the method name (e.g. "ListNewsSources") to the metadata as x-campus-method for later use (currently logging the devices api usage) func addMethodNameInterceptor(ctx context.Context, method string, req interface{}, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx = metadata.AppendToOutgoingContext(ctx, "x-campus-method", method) diff --git a/server/migrate.go b/server/migrate.go index 35f94ef1..75ff09e2 100644 --- a/server/migrate.go +++ b/server/migrate.go @@ -1,11 +1,12 @@ package main import ( + "github.com/TUM-Dev/Campus-Backend/server/utils" log "github.com/sirupsen/logrus" ) func main() { - if db := setupDB(); db != nil { + if db := utils.SetupDB(); db != nil { log.Fatal("db is nil. How did this happen?") } } diff --git a/server/utils/utils.go b/server/utils/utils.go new file mode 100644 index 00000000..623aeccb --- /dev/null +++ b/server/utils/utils.go @@ -0,0 +1,63 @@ +package utils + +import ( + "os" + + "github.com/TUM-Dev/Campus-Backend/server/backend/migration" + "github.com/TUM-Dev/Campus-Backend/server/env" + "github.com/getsentry/sentry-go" + "github.com/makasim/sentryhook" + gormlogrus "github.com/onrik/gorm-logrus" + log "github.com/sirupsen/logrus" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +// SetupDB connects to the database and migrates it if necessary +func SetupDB() *gorm.DB { + dbHost := os.Getenv("DB_DSN") + if dbHost == "" { + log.Fatal("Failed to start! The 'DB_DSN' environment variable is not defined. Take a look at the README.md for more details.") + } + + log.Info("Connecting to dsn") + db, err := gorm.Open(mysql.Open(dbHost), &gorm.Config{Logger: gormlogrus.New()}) + if err != nil { + log.WithError(err).Fatal("failed to connect database") + } + + // Migrate the schema + // currently not activated as + if err := migration.New(db, false).Migrate(); err != nil { + log.WithError(err).Fatal("Failed to migrate database") + } + return db +} + +// SetupTelemetry initializes our telemetry stack +// - sentry to be connected with log +// - logrus to +func SetupTelemetry(Version string) { + environment := "development" + log.SetLevel(log.TraceLevel) + if env.IsProd() { + log.SetLevel(log.InfoLevel) + environment = "production" + log.SetFormatter(&log.JSONFormatter{}) // simpler to query but harder to parse in the console + } + + if sentryDSN := os.Getenv("SENTRY_DSN"); sentryDSN != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: sentryDSN, + AttachStacktrace: true, + Release: Version, + Dist: Version, // see https://github.com/getsentry/sentry-react-native/issues/516 why this is equal + Environment: environment, + }); err != nil { + log.WithError(err).Error("Sentry initialization failed") + } + log.AddHook(sentryhook.New([]log.Level{log.PanicLevel, log.FatalLevel, log.ErrorLevel, log.WarnLevel})) + } else { + log.Info("continuing without sentry") + } +} From d48cad0eca42193fdf9a35fed62f965f458b51c5 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:52:34 +0200 Subject: [PATCH 03/62] refactored files around --- server/main.go | 43 +--------------------- server/migrate.go | 12 ------ server/utils/db.go | 32 ++++++++++++++++ server/utils/grpc_to_web.go | 49 +++++++++++++++++++++++++ server/utils/{utils.go => telemetry.go} | 25 ------------- 5 files changed, 82 insertions(+), 79 deletions(-) delete mode 100644 server/migrate.go create mode 100644 server/utils/db.go create mode 100644 server/utils/grpc_to_web.go rename server/utils/{utils.go => telemetry.go} (59%) diff --git a/server/main.go b/server/main.go index 08b15d4f..516727fd 100644 --- a/server/main.go +++ b/server/main.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/textproto" - "strings" "time" "github.com/TUM-Dev/Campus-Backend/server/utils" @@ -24,10 +23,8 @@ import ( "github.com/soheilhy/cmux" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" ) const httpPort = ":50051" @@ -136,49 +133,11 @@ func addMethodNameInterceptor(ctx context.Context, method string, req interface{ // errorHandler translates gRPC raised by the backend into HTTP errors. func errorHandler(_ context.Context, _ *runtime.ServeMux, _ runtime.Marshaler, w http.ResponseWriter, _ *http.Request, err error) { - errorResp := errorResponse{Error: "Internal Server Error", StatusCode: http.StatusInternalServerError} - - if strings.HasPrefix(err.Error(), "no device id") { - errorResp.Error = "Not Authorized" - errorResp.StatusCode = http.StatusForbidden - } - - if s, ok := status.FromError(err); ok { - switch s.Code() { - case codes.NotFound: - errorResp.Error = "Not Found" - errorResp.StatusCode = http.StatusNotFound - case codes.Unimplemented: - errorResp.Error = "Not Implemented" - errorResp.StatusCode = http.StatusNotImplemented - case codes.InvalidArgument: - errorResp.Error = "Invalid Argument" - errorResp.StatusCode = http.StatusBadRequest - case codes.Internal: - errorResp.Error = "Internal Server Error" - errorResp.StatusCode = http.StatusInternalServerError - } - - if s.Message() != "" { - errorResp.Error = s.Message() - } - - if s.Details() != nil && len(s.Details()) > 0 { - errorResp.Details = s.Details() - } - } - + errorResp := utils.GrpcErrorToWebError(err) w.Header().Set("Content-Type", "application/json") w.WriteHeader(errorResp.StatusCode) if err = json.NewEncoder(w).Encode(errorResp); err != nil { log.WithError(err).Error("Marshal error response failed") - return } } - -type errorResponse struct { - Error string `json:"error"` - Details []interface{} `json:"details,omitempty"` - StatusCode int `json:"-"` -} diff --git a/server/migrate.go b/server/migrate.go deleted file mode 100644 index 75ff09e2..00000000 --- a/server/migrate.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -import ( - "github.com/TUM-Dev/Campus-Backend/server/utils" - log "github.com/sirupsen/logrus" -) - -func main() { - if db := utils.SetupDB(); db != nil { - log.Fatal("db is nil. How did this happen?") - } -} diff --git a/server/utils/db.go b/server/utils/db.go new file mode 100644 index 00000000..180bc023 --- /dev/null +++ b/server/utils/db.go @@ -0,0 +1,32 @@ +package utils + +import ( + "os" + + "github.com/TUM-Dev/Campus-Backend/server/backend/migration" + gormlogrus "github.com/onrik/gorm-logrus" + log "github.com/sirupsen/logrus" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +// SetupDB connects to the database and migrates it if necessary +func SetupDB() *gorm.DB { + dbHost := os.Getenv("DB_DSN") + if dbHost == "" { + log.Fatal("Failed to start! The 'DB_DSN' environment variable is not defined. Take a look at the README.md for more details.") + } + + log.Info("Connecting to dsn") + db, err := gorm.Open(mysql.Open(dbHost), &gorm.Config{Logger: gormlogrus.New()}) + if err != nil { + log.WithError(err).Fatal("failed to connect database") + } + + // Migrate the schema + // currently not activated as + if err := migration.New(db, false).Migrate(); err != nil { + log.WithError(err).Fatal("Failed to migrate database") + } + return db +} diff --git a/server/utils/grpc_to_web.go b/server/utils/grpc_to_web.go new file mode 100644 index 00000000..eecc41cc --- /dev/null +++ b/server/utils/grpc_to_web.go @@ -0,0 +1,49 @@ +package utils + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "net/http" + "strings" +) + +func GrpcErrorToWebError(err error) ErrorResponse { + errorResp := ErrorResponse{Error: "Internal Server Error", StatusCode: http.StatusInternalServerError} + + if strings.HasPrefix(err.Error(), "no device id") { + errorResp.Error = "Not Authorized" + errorResp.StatusCode = http.StatusForbidden + } + + if s, ok := status.FromError(err); ok { + switch s.Code() { + case codes.NotFound: + errorResp.Error = "Not Found" + errorResp.StatusCode = http.StatusNotFound + case codes.Unimplemented: + errorResp.Error = "Not Implemented" + errorResp.StatusCode = http.StatusNotImplemented + case codes.InvalidArgument: + errorResp.Error = "Invalid Argument" + errorResp.StatusCode = http.StatusBadRequest + case codes.Internal: + errorResp.Error = "Internal Server Error" + errorResp.StatusCode = http.StatusInternalServerError + } + + if s.Message() != "" { + errorResp.Error = s.Message() + } + + if s.Details() != nil && len(s.Details()) > 0 { + errorResp.Details = s.Details() + } + } + return errorResp +} + +type ErrorResponse struct { + Error string `json:"error"` + Details []interface{} `json:"details,omitempty"` + StatusCode int `json:"-"` +} diff --git a/server/utils/utils.go b/server/utils/telemetry.go similarity index 59% rename from server/utils/utils.go rename to server/utils/telemetry.go index 623aeccb..fd0ecf66 100644 --- a/server/utils/utils.go +++ b/server/utils/telemetry.go @@ -3,37 +3,12 @@ package utils import ( "os" - "github.com/TUM-Dev/Campus-Backend/server/backend/migration" "github.com/TUM-Dev/Campus-Backend/server/env" "github.com/getsentry/sentry-go" "github.com/makasim/sentryhook" - gormlogrus "github.com/onrik/gorm-logrus" log "github.com/sirupsen/logrus" - "gorm.io/driver/mysql" - "gorm.io/gorm" ) -// SetupDB connects to the database and migrates it if necessary -func SetupDB() *gorm.DB { - dbHost := os.Getenv("DB_DSN") - if dbHost == "" { - log.Fatal("Failed to start! The 'DB_DSN' environment variable is not defined. Take a look at the README.md for more details.") - } - - log.Info("Connecting to dsn") - db, err := gorm.Open(mysql.Open(dbHost), &gorm.Config{Logger: gormlogrus.New()}) - if err != nil { - log.WithError(err).Fatal("failed to connect database") - } - - // Migrate the schema - // currently not activated as - if err := migration.New(db, false).Migrate(); err != nil { - log.WithError(err).Fatal("Failed to migrate database") - } - return db -} - // SetupTelemetry initializes our telemetry stack // - sentry to be connected with log // - logrus to From 844b8f5b4970080bcb22a69f516b14c79ebdddce Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:53:48 +0200 Subject: [PATCH 04/62] added the ability to exit after migration --- server/utils/db.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/utils/db.go b/server/utils/db.go index 180bc023..3c9cb471 100644 --- a/server/utils/db.go +++ b/server/utils/db.go @@ -28,5 +28,10 @@ func SetupDB() *gorm.DB { if err := migration.New(db, false).Migrate(); err != nil { log.WithError(err).Fatal("Failed to migrate database") } + + if os.Getenv("EXIT_AFTER_MIGRATION") == "true" { + log.Info("Exiting after migration") + os.Exit(0) + } return db } From cc6b08463ee7ed02886c79c38d63ff01fc7d375e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:54:32 +0200 Subject: [PATCH 05/62] fixed the testcase --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ce9f8e5..3ea31efd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,9 +35,10 @@ jobs: cache-dependency-path: | server/go.sum - name: run migration - run: go run migrate.go + run: go run main.go working-directory: ./server env: + EXIT_AFTER_MIGRATION: "true" DB_DSN: root:super_secret_pa$$w0rd@tcp(db:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev build: From 96755cbf405ebbdedc1b58d7a34822bf2ef359fa Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:57:36 +0200 Subject: [PATCH 06/62] made the action sleeps enough that the db gets its shit together --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3ea31efd..11dfff35 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,7 +25,7 @@ jobs: ports: - 3306:3306 env: - MARIADB_ROOT_PASSWORD: super_secret_pa$$w0rd + MARIADB_ROOT_PASSWORD: super_secret_passw0rd MARIADB_DATABASE: campus_db steps: - uses: actions/checkout@v4 @@ -34,12 +34,14 @@ jobs: go-version: '1.21' cache-dependency-path: | server/go.sum + - name: wait for db + run: sleep 30 - name: run migration run: go run main.go working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" - DB_DSN: root:super_secret_pa$$w0rd@tcp(db:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + DB_DSN: root:super_secret_passw0rd@tcp(db:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev build: runs-on: ubuntu-latest From 790badd2b91790ab9d497763e2c3aebae99bf305 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 16 Oct 2023 23:58:08 +0200 Subject: [PATCH 07/62] linting fix --- server/utils/grpc_to_web.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/utils/grpc_to_web.go b/server/utils/grpc_to_web.go index eecc41cc..6571369c 100644 --- a/server/utils/grpc_to_web.go +++ b/server/utils/grpc_to_web.go @@ -1,10 +1,11 @@ package utils import ( - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "net/http" "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func GrpcErrorToWebError(err error) ErrorResponse { From 9a2801754994c7cf6c4e2aa77492f0697320c1ac Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:20:15 +0200 Subject: [PATCH 08/62] tested a different way of communicating with the db --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11dfff35..2cbaeead 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,7 @@ jobs: working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" - DB_DSN: root:super_secret_passw0rd@tcp(db:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + DB_DSN: root:super_secret_passw0rd@tcp(mariadb:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev build: runs-on: ubuntu-latest From 41f4d1f8af12f510a8bd2bfa6c5c0d2e45536e01 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:37:20 +0200 Subject: [PATCH 09/62] added a request for the docker logs after waiting --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cbaeead..08b83cc8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: cache-dependency-path: | server/go.sum - name: wait for db - run: sleep 30 + run: sleep 30 && docker logs mariadb - name: run migration run: go run main.go working-directory: ./server From 1d7d37ea32d9f3cd149c22f1dcac18bf081d124a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:39:17 +0200 Subject: [PATCH 10/62] more logging --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08b83cc8..0a594758 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: cache-dependency-path: | server/go.sum - name: wait for db - run: sleep 30 && docker logs mariadb + run: sleep 15 && docker ps -a - name: run migration run: go run main.go working-directory: ./server From e60f285a6ac9dfc0bad7618a06f689639b610d14 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:45:39 +0200 Subject: [PATCH 11/62] tested a different approach --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a594758..9ceb573c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,8 +34,10 @@ jobs: go-version: '1.21' cache-dependency-path: | server/go.sum + - run: docker logs $(docker ps -qa) - name: wait for db - run: sleep 15 && docker ps -a + run: sleep 15 + - run: docker logs $(docker ps -qa) - name: run migration run: go run main.go working-directory: ./server From 61e53f99aaf931f0331117e1409b837ed7c0b552 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:50:28 +0200 Subject: [PATCH 12/62] tested a different approach --- .github/workflows/main.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ceb573c..d501b396 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,12 +34,8 @@ jobs: go-version: '1.21' cache-dependency-path: | server/go.sum - - run: docker logs $(docker ps -qa) - - name: wait for db - run: sleep 15 - - run: docker logs $(docker ps -qa) - name: run migration - run: go run main.go + run: sleep 30 && docker logs $(docker ps -qa) && go run main.go working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" From 4f1611b5e0166df39e568227a5119b0a5b4b97f8 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 02:56:15 +0200 Subject: [PATCH 13/62] more debugging info --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d501b396..e183017c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: cache-dependency-path: | server/go.sum - name: run migration - run: sleep 30 && docker logs $(docker ps -qa) && go run main.go + run: docker ps -a && sleep 30 && docker logs $(docker ps -qa) && go run main.go working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" From be0f7f7804b8cb1b13fdfe711eae8d5b9f999f92 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 13:27:34 +0200 Subject: [PATCH 14/62] tested different config --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e183017c..131594a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,11 +35,11 @@ jobs: cache-dependency-path: | server/go.sum - name: run migration - run: docker ps -a && sleep 30 && docker logs $(docker ps -qa) && go run main.go + run: sleep 20 && docker logs $(docker ps -qa) && go run main.go working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" - DB_DSN: root:super_secret_passw0rd@tcp(mariadb:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev build: runs-on: ubuntu-latest From ba3d9c71de1a4123220f5f4a776a93ae11e64368 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 13:35:30 +0200 Subject: [PATCH 15/62] refactored the wait for db part and source-schema check --- .github/workflows/main.yml | 4 +- .../charts/backend/files/source-schema.sql | 953 ++++++------------ 2 files changed, 323 insertions(+), 634 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 131594a0..c61da345 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,8 +34,10 @@ jobs: go-version: '1.21' cache-dependency-path: | server/go.sum + - name: wait for db + run: sleep 20 && docker logs $(docker ps -qa) && - name: run migration - run: sleep 20 && docker logs $(docker ps -qa) && go run main.go + run: go run main.go working-directory: ./server env: EXIT_AFTER_MIGRATION: "true" diff --git a/deployment/charts/backend/files/source-schema.sql b/deployment/charts/backend/files/source-schema.sql index dad3cfc2..b33181b3 100644 --- a/deployment/charts/backend/files/source-schema.sql +++ b/deployment/charts/backend/files/source-schema.sql @@ -2,54 +2,44 @@ CREATE USER 'tca-students' IDENTIFIED BY 'strongpassword'; create table if not exists actions ( - action int auto_increment - primary key, + action int auto_increment primary key, name varchar(50) not null, description mediumtext not null, color varchar(6) not null -) - collate = utf8mb4_unicode_ci - auto_increment = 19; +) collate = utf8mb4_unicode_ci + auto_increment = 19; create table if not exists alarm_ban ( - ban int auto_increment - primary key, + ban int auto_increment primary key, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, ip binary(16) not null, - constraint ip - unique (ip) -) - collate = utf8mb4_unicode_ci; + constraint ip unique (ip) +) collate = utf8mb4_unicode_ci; create table if not exists alarm_log ( - alarm int auto_increment - primary key, + alarm int auto_increment primary key, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, message text not null, send int not null, received int not null, test enum ('true', 'false') default 'false' not null, ip binary(16) not null -) - collate = utf8mb4_unicode_ci; +) collate = utf8mb4_unicode_ci; create table if not exists barrierFree_moreInfo ( - id int(11) unsigned auto_increment - primary key, + id int(11) unsigned auto_increment primary key, title varchar(32) null, category varchar(11) null, url varchar(128) null -) - charset = utf8 - auto_increment = 11; +) charset = utf8 + auto_increment = 11; create table if not exists barrierFree_persons ( - id int(11) unsigned auto_increment - primary key, + id int(11) unsigned auto_increment primary key, name varchar(40) null, telephone varchar(32) null, email varchar(32) null, @@ -57,107 +47,81 @@ create table if not exists barrierFree_persons office varchar(16) null, officeHour varchar(16) null, tumID varchar(24) null -) - charset = utf8 - auto_increment = 19; +) charset = utf8 + auto_increment = 19; create table if not exists card_type ( - card_type int auto_increment - primary key, + card_type int auto_increment primary key, title varchar(255) null -) - auto_increment = 2; +) auto_increment = 2; create table if not exists chat_room ( - room int auto_increment - primary key, + room int auto_increment primary key, name varchar(100) not null, semester varchar(3) null, - constraint `Index 2` - unique (semester, name) -) - collate = utf8mb4_unicode_ci - auto_increment = 1724450; + constraint `Index 2` unique (semester, name) +) collate = utf8mb4_unicode_ci + auto_increment = 1724450; create table if not exists crontab ( cron int auto_increment, - `interval` int default 7200 not null, - lastRun int default 0 not null, + `interval` int default 7200 not null, + lastRun int default 0 not null, type enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload', 'canteenHeadCount') null, - id int null, - constraint cron - unique (cron) -) - collate = utf8mb4_unicode_ci - auto_increment = 44; + id int null, + constraint cron unique (cron) +) collate = utf8mb4_unicode_ci + auto_increment = 44; create table if not exists curricula ( - curriculum int auto_increment - primary key, + curriculum int auto_increment primary key, category enum ('bachelor', 'master') default 'bachelor' not null, name mediumtext not null, url mediumtext not null -) - collate = utf8mb4_unicode_ci - auto_increment = 16; +) collate = utf8mb4_unicode_ci + auto_increment = 16; create table if not exists dish ( - dish int auto_increment - primary key, + dish int auto_increment primary key, name varchar(150) not null, type varchar(20) not null -) - collate = utf8mb4_unicode_ci; +) collate = utf8mb4_unicode_ci; create table if not exists dishflags ( - flag int auto_increment - primary key, + flag int auto_increment primary key, short varchar(10) not null, description varchar(50) not null -) - collate = utf8mb4_unicode_ci; +) collate = utf8mb4_unicode_ci; create table if not exists dish2dishflags ( - dish2dishflags int auto_increment - primary key, + dish2dishflags int auto_increment primary key, dish int not null, flag int not null, - constraint dish - unique (dish, flag), - constraint dish2dishflags_ibfk_1 - foreign key (dish) references dish (dish) - on update cascade on delete cascade, - constraint dish2dishflags_ibfk_2 - foreign key (flag) references dishflags (flag) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci; - -create or replace index flag - on dish2dishflags (flag); + constraint dish unique (dish, flag), + constraint dish2dishflags_ibfk_1 foreign key (dish) references dish (dish) on update cascade on delete cascade, + constraint dish2dishflags_ibfk_2 foreign key (flag) references dishflags (flag) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci; + +create or replace index flag on dish2dishflags (flag); create table if not exists faculty ( - faculty int auto_increment - primary key, + faculty int auto_increment primary key, name varchar(150) not null, - constraint name - unique (name) -) - charset = utf8mb4 - auto_increment = 18; + constraint name unique (name) +) charset = utf8mb4 + auto_increment = 18; create table if not exists feedback ( - id int auto_increment - primary key, + id int auto_increment primary key, email_id text charset utf8 null, receiver text charset utf8 null, reply_to text charset utf8 null, @@ -166,26 +130,22 @@ create table if not exists feedback latitude decimal(11, 8) null, longitude decimal(11, 8) null, timestamp datetime /* mariadb-5.3 */ default current_timestamp() null -) - auto_increment = 293; +) auto_increment = 293; create table if not exists files ( - file int auto_increment - primary key, + file int auto_increment primary key, name mediumtext not null, path mediumtext not null, downloads int default 0 not null, url varchar(191) null, downloaded tinyint(1) default 1 null -) - collate = utf8mb4_unicode_ci - auto_increment = 34761; +) collate = utf8mb4_unicode_ci + auto_increment = 34761; create table if not exists kino ( - kino int auto_increment - primary key, + kino int auto_increment primary key, date datetime /* mariadb-5.3 */ not null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, title text not null, @@ -199,41 +159,32 @@ create table if not exists kino cover int null, trailer text null, link varchar(190) not null, - constraint link - unique (link), - constraint kino_ibfk_1 - foreign key (cover) references files (file) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci - auto_increment = 219; - -create or replace index cover - on kino (cover); + constraint link unique (link), + constraint kino_ibfk_1 foreign key (cover) references files (file) on update cascade on delete set null +) collate = utf8mb4_unicode_ci + auto_increment = 219; + +create or replace index cover on kino (cover); create table if not exists lecture ( - lecture int auto_increment - primary key, + lecture int auto_increment primary key, title varchar(255) null ); create table if not exists location ( - location int auto_increment - primary key, + location int auto_increment primary key, name text not null, lon float(10, 6) not null, lat float(10, 6) not null, radius int default 1000 not null comment 'in meters' -) - charset = utf8 - auto_increment = 2; +) charset = utf8 + auto_increment = 2; create table if not exists member ( - member int auto_increment - primary key, + member int auto_increment primary key, lrz_id varchar(7) not null, name varchar(150) not null, active_day int default 0 null, @@ -241,16 +192,13 @@ create table if not exists member student_id text null, employee_id text null, external_id text null, - constraint lrz_id - unique (lrz_id) -) - collate = utf8mb4_unicode_ci - auto_increment = 104353; + constraint lrz_id unique (lrz_id) +) collate = utf8mb4_unicode_ci + auto_increment = 104353; create table if not exists card ( - card int auto_increment - primary key, + card int auto_increment primary key, member int null, lecture int null, card_type int null, @@ -264,140 +212,94 @@ create table if not exists card updated_at date not null, duplicate_card int null, aggr_rating float default 0 null, - constraint card_ibfk_1 - foreign key (member) references member (member) - on delete set null, - constraint card_ibfk_2 - foreign key (lecture) references lecture (lecture) - on delete set null, - constraint card_ibfk_3 - foreign key (card_type) references card_type (card_type) - on delete set null, - constraint card_ibfk_4 - foreign key (duplicate_card) references card (card) - on delete set null + constraint card_ibfk_1 foreign key (member) references member (member) on delete set null, + constraint card_ibfk_2 foreign key (lecture) references lecture (lecture) on delete set null, + constraint card_ibfk_3 foreign key (card_type) references card_type (card_type) on delete set null, + constraint card_ibfk_4 foreign key (duplicate_card) references card (card) on delete set null ); -create or replace index card_type - on card (card_type); +create or replace index card_type on card (card_type); -create or replace index duplicate_card - on card (duplicate_card); +create or replace index duplicate_card on card (duplicate_card); -create or replace index lecture - on card (lecture); +create or replace index lecture on card (lecture); -create or replace index member - on card (member); +create or replace index member on card (member); create table if not exists card_box ( - card_box int auto_increment - primary key, + card_box int auto_increment primary key, member int null, title varchar(255) null, duration int not null, - constraint card_box_ibfk_1 - foreign key (member) references member (member) - on delete cascade -) - auto_increment = 6; + constraint card_box_ibfk_1 foreign key (member) references member (member) on delete cascade +) auto_increment = 6; -create or replace index member - on card_box (member); +create or replace index member on card_box (member); create table if not exists card_comment ( - card_comment int auto_increment - primary key, + card_comment int auto_increment primary key, member int null, card int not null, rating int default 0 null, created_at date not null, - constraint card_comment_ibfk_1 - foreign key (member) references member (member) - on delete set null, - constraint card_comment_ibfk_2 - foreign key (card) references card (card) - on delete cascade + constraint card_comment_ibfk_1 foreign key (member) references member (member) on delete set null, + constraint card_comment_ibfk_2 foreign key (card) references card (card) on delete cascade ); -create or replace index card - on card_comment (card); +create or replace index card on card_comment (card); -create or replace index member - on card_comment (member); +create or replace index member on card_comment (member); create table if not exists card_option ( - card_option int auto_increment - primary key, + card_option int auto_increment primary key, card int not null, text varchar(2000) default '' null, is_correct_answer tinyint(1) default 0 null, sort_order int default 0 not null, image varchar(2000) null, - constraint card_option_ibfk_1 - foreign key (card) references card (card) - on delete cascade + constraint card_option_ibfk_1 foreign key (card) references card (card) on delete cascade ); -create or replace index card - on card_option (card); +create or replace index card on card_option (card); create table if not exists chat_message ( - message int auto_increment - primary key, + message int auto_increment primary key, member int not null, room int not null, text longtext not null, created datetime /* mariadb-5.3 */ not null, signature longtext not null, - constraint FK_chat_message_chat_room - foreign key (room) references chat_room (room) - on update cascade on delete cascade, - constraint chat_message_ibfk_1 - foreign key (member) references member (member) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci - auto_increment = 1977; - -create or replace index chat_message_b3c09425 - on chat_message (member); - -create or replace index chat_message_ca20ebca - on chat_message (room); + constraint FK_chat_message_chat_room foreign key (room) references chat_room (room) on update cascade on delete cascade, + constraint chat_message_ibfk_1 foreign key (member) references member (member) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci + auto_increment = 1977; + +create or replace index chat_message_b3c09425 on chat_message (member); + +create or replace index chat_message_ca20ebca on chat_message (room); create table if not exists chat_room2members ( - room2members int auto_increment - primary key, + room2members int auto_increment primary key, room int not null, member int not null, - constraint chatroom_id - unique (room, member), - constraint FK_chat_room2members_chat_room - foreign key (room) references chat_room (room) - on update cascade on delete cascade, - constraint chat_room2members_ibfk_2 - foreign key (member) references member (member) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci - auto_increment = 63377426; - -create or replace index chat_chatroom_members_29801a33 - on chat_room2members (room); - -create or replace index chat_chatroom_members_b3c09425 - on chat_room2members (member); + constraint chatroom_id unique (room, member), + constraint FK_chat_room2members_chat_room foreign key (room) references chat_room (room) on update cascade on delete cascade, + constraint chat_room2members_ibfk_2 foreign key (member) references member (member) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci + auto_increment = 63377426; + +create or replace index chat_chatroom_members_29801a33 on chat_room2members (room); + +create or replace index chat_chatroom_members_b3c09425 on chat_room2members (member); create table if not exists devices ( - device int auto_increment - primary key, + device int auto_increment primary key, member int null, uuid varchar(50) not null, created timestamp /* mariadb-5.3 */ null, @@ -414,19 +316,14 @@ create table if not exists devices confirmationKey varchar(35) null, keyCreated datetime null, keyConfirmed datetime null, - constraint uuid - unique (uuid), - constraint devices_ibfk_1 - foreign key (member) references member (member) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci - auto_increment = 144352; + constraint uuid unique (uuid), + constraint devices_ibfk_1 foreign key (member) references member (member) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci + auto_increment = 144352; create table if not exists device2stats ( - device int default 0 not null - primary key, + device int default 0 not null primary key, LecturesPersonalActivity int default 0 not null, CafeteriaActivity int default 0 not null, WizNavStartActivity int default 0 not null, @@ -468,14 +365,10 @@ create table if not exists device2stats NewsCard2 int default 0 not null, NewsCard3 int default 0 not null, NewsCard7 int default 0 not null, - constraint device2stats_ibfk_2 - foreign key (device) references devices (device) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci; + constraint device2stats_ibfk_2 foreign key (device) references devices (device) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci; -create or replace index member - on devices (member); +create or replace index member on devices (member); create table if not exists members_card ( @@ -484,144 +377,105 @@ create table if not exists members_card card_box int null, last_answered_active_day int null, primary key (member, card), - constraint members_card_ibfk_1 - foreign key (member) references member (member) - on delete cascade, - constraint members_card_ibfk_2 - foreign key (card) references card (card) - on delete cascade, - constraint members_card_ibfk_3 - foreign key (card_box) references card_box (card_box) - on delete set null + constraint members_card_ibfk_1 foreign key (member) references member (member) on delete cascade, + constraint members_card_ibfk_2 foreign key (card) references card (card) on delete cascade, + constraint members_card_ibfk_3 foreign key (card_box) references card_box (card_box) on delete set null ); -create or replace index card - on members_card (card); +create or replace index card on members_card (card); -create or replace index card_box - on members_card (card_box); +create or replace index card_box on members_card (card_box); create table if not exists members_card_answer_history ( - members_card_answer_history int auto_increment - primary key, + members_card_answer_history int auto_increment primary key, member int not null, card int null, card_box int null, answer varchar(2000) null, answer_score float(10, 2) default 0.00 null, created_at date not null, - constraint members_card_answer_history_ibfk_1 - foreign key (member) references member (member) - on delete cascade, - constraint members_card_answer_history_ibfk_2 - foreign key (card) references card (card) - on delete set null, - constraint members_card_answer_history_ibfk_3 - foreign key (card_box) references card_box (card_box) - on delete set null + constraint members_card_answer_history_ibfk_1 foreign key (member) references member (member) on delete cascade, + constraint members_card_answer_history_ibfk_2 foreign key (card) references card (card) on delete set null, + constraint members_card_answer_history_ibfk_3 foreign key (card_box) references card_box (card_box) on delete set null ); -create or replace index card - on members_card_answer_history (card); +create or replace index card on members_card_answer_history (card); -create or replace index card_box - on members_card_answer_history (card_box); +create or replace index card_box on members_card_answer_history (card_box); -create or replace index member - on members_card_answer_history (member); +create or replace index member on members_card_answer_history (member); create table if not exists mensa ( - mensa int auto_increment - primary key, + mensa int auto_increment primary key, id int null, name mediumtext not null, address mediumtext not null, latitude float(10, 6) default 0.000000 not null, longitude float(10, 6) default 0.000000 not null, - constraint id - unique (id) -) - collate = utf8mb4_unicode_ci - auto_increment = 17; + constraint id unique (id) +) collate = utf8mb4_unicode_ci + auto_increment = 17; create table if not exists dish2mensa ( - dish2mensa int auto_increment - primary key, + dish2mensa int auto_increment primary key, mensa int not null, dish int not null, date date not null, created datetime /* mariadb-5.3 */ not null, modifierd timestamp /* mariadb-5.3 */ default '0000-00-00 00:00:00' not null on update current_timestamp(), - constraint dish2mensa_ibfk_1 - foreign key (mensa) references mensa (mensa) - on update cascade on delete cascade, - constraint dish2mensa_ibfk_2 - foreign key (dish) references dish (dish) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci; + constraint dish2mensa_ibfk_1 foreign key (mensa) references mensa (mensa) on update cascade on delete cascade, + constraint dish2mensa_ibfk_2 foreign key (dish) references dish (dish) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci; -create or replace index dish - on dish2mensa (dish); +create or replace index dish on dish2mensa (dish); -create or replace index mensa - on dish2mensa (mensa); +create or replace index mensa on dish2mensa (mensa); create table if not exists mensaplan_mensa ( - id int auto_increment - primary key, + id int auto_increment primary key, name varchar(100) not null, latitude double null, longitude double null, webid int null, category varchar(50) not null -) - engine = MyISAM - collate = utf8mb4_unicode_ci - auto_increment = 30; +) engine = MyISAM + collate = utf8mb4_unicode_ci + auto_increment = 30; create table if not exists mensaprices ( - price int auto_increment - primary key, + price int auto_increment primary key, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, person mediumtext not null, type mediumtext not null, typeLong mediumtext not null, typeNumber int not null, value decimal not null -) - collate = utf8mb4_unicode_ci; +) collate = utf8mb4_unicode_ci; create table if not exists migrations ( - id varchar(255) not null - primary key + id varchar(255) not null primary key ); create table if not exists newsSource ( - source int auto_increment - primary key, + source int auto_increment primary key, title mediumtext not null, url mediumtext null, icon int null, hook enum ('newspread', 'impulsivHook') null, - constraint newsSource_ibfk_1 - foreign key (icon) references files (file) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci - auto_increment = 17; + constraint newsSource_ibfk_1 foreign key (icon) references files (file) on update cascade on delete set null +) collate = utf8mb4_unicode_ci + auto_increment = 17; create table if not exists news ( - news int auto_increment - primary key, + news int auto_increment primary key, date datetime /* mariadb-5.3 */ not null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, title tinytext not null, @@ -630,57 +484,42 @@ create table if not exists news link varchar(190) not null, image text null, file int null, - constraint link - unique (link), - constraint news_ibfk_1 - foreign key (src) references newsSource (source) - on update cascade on delete cascade, - constraint news_ibfk_2 - foreign key (file) references files (file) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci - auto_increment = 770113; - -create or replace index file - on news (file); - -create or replace index src - on news (src); - -create or replace index icon - on newsSource (icon); + constraint link unique (link), + constraint news_ibfk_1 foreign key (src) references newsSource (source) on update cascade on delete cascade, + constraint news_ibfk_2 foreign key (file) references files (file) on update cascade on delete set null +) collate = utf8mb4_unicode_ci + auto_increment = 770113; + +create or replace index file on news (file); + +create or replace index src on news (src); + +create or replace index icon on newsSource (icon); create table if not exists news_alert ( - news_alert int auto_increment - primary key, + news_alert int auto_increment primary key, file int null, name varchar(100) null, link text null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, `from` datetime /* mariadb-5.3 */ default current_timestamp() not null, `to` datetime /* mariadb-5.3 */ default current_timestamp() not null, - constraint FK_File - unique (file) -) - charset = utf8mb4 - auto_increment = 7; + constraint FK_File unique (file) +) charset = utf8mb4 + auto_increment = 7; create table if not exists notification_type ( - type int auto_increment - primary key, + type int auto_increment primary key, name text not null, confirmation enum ('true', 'false') default 'false' not null -) - charset = utf8 - auto_increment = 4; +) charset = utf8 + auto_increment = 4; create table if not exists notification ( - notification int auto_increment - primary key, + notification int auto_increment primary key, type int not null, location int null, title text not null, @@ -688,21 +527,14 @@ create table if not exists notification created timestamp /* mariadb-5.3 */ default current_timestamp() not null, signature text null, silent tinyint(1) default 0 not null, - constraint notification_ibfk_1 - foreign key (type) references notification_type (type) - on update cascade on delete cascade, - constraint notification_ibfk_2 - foreign key (location) references location (location) - on update cascade on delete set null -) - charset = utf8 - auto_increment = 107; - -create or replace index location - on notification (location); - -create or replace index type - on notification (type); + constraint notification_ibfk_1 foreign key (type) references notification_type (type) on update cascade on delete cascade, + constraint notification_ibfk_2 foreign key (location) references location (location) on update cascade on delete set null +) charset = utf8 + auto_increment = 107; + +create or replace index location on notification (location); + +create or replace index type on notification (type); create table if not exists notification_confirmation ( @@ -712,20 +544,15 @@ create table if not exists notification_confirmation created timestamp /* mariadb-5.3 */ default current_timestamp() not null, received timestamp /* mariadb-5.3 */ null, primary key (notification, device), - constraint notification_confirmation_ibfk_1 - foreign key (notification) references notification (notification), - constraint notification_confirmation_ibfk_2 - foreign key (device) references devices (device) -) - charset = utf8; + constraint notification_confirmation_ibfk_1 foreign key (notification) references notification (notification), + constraint notification_confirmation_ibfk_2 foreign key (device) references devices (device) +) charset = utf8; -create or replace index device - on notification_confirmation (device); +create or replace index device on notification_confirmation (device); create table if not exists openinghours ( - id int auto_increment - primary key, + id int auto_increment primary key, category varchar(20) not null, name varchar(60) not null, address varchar(140) not null, @@ -736,31 +563,26 @@ create table if not exists openinghours url varchar(300) not null, language varchar(2) default 'de' null, reference_id int default -1 null -) - collate = utf8mb4_unicode_ci - auto_increment = 113; +) collate = utf8mb4_unicode_ci + auto_increment = 113; create table if not exists question ( - question int auto_increment - primary key, + question int auto_increment primary key, member int not null, text text not null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, end timestamp /* mariadb-5.3 */ null -) - auto_increment = 282; +) auto_increment = 282; -create or replace index member - on question (member); +create or replace index member on question (member); create table if not exists question2answer ( question int not null, answer int not null, member int not null, - constraint question - unique (question, member) + constraint question unique (question, member) ); create table if not exists question2faculty @@ -768,29 +590,21 @@ create table if not exists question2faculty question int not null, faculty int not null, primary key (question, faculty), - constraint question2faculty_ibfk_1 - foreign key (question) references question (question) - on update cascade on delete cascade, - constraint question2faculty_ibfk_2 - foreign key (faculty) references faculty (faculty) - on update cascade on delete cascade + constraint question2faculty_ibfk_1 foreign key (question) references question (question) on update cascade on delete cascade, + constraint question2faculty_ibfk_2 foreign key (faculty) references faculty (faculty) on update cascade on delete cascade ); -create or replace index faculty - on question2faculty (faculty); +create or replace index faculty on question2faculty (faculty); create table if not exists questionAnswers ( - answer int auto_increment - primary key, + answer int auto_increment primary key, text text not null -) - auto_increment = 3; +) auto_increment = 3; create table if not exists reports ( - report int auto_increment - primary key, + report int auto_increment primary key, device int null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, fixed enum ('true', 'false') default 'false' not null, @@ -810,32 +624,24 @@ create table if not exists reports screenHeight varchar(100) not null, screenOrientation varchar(100) not null, screenDpi varchar(100) not null, - constraint reports_ibfk_3 - foreign key (device) references devices (device) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci; + constraint reports_ibfk_3 foreign key (device) references devices (device) on update cascade on delete set null +) collate = utf8mb4_unicode_ci; -create or replace index device - on reports (device); +create or replace index device on reports (device); create table if not exists rights ( - `right` int auto_increment - primary key, + `right` int auto_increment primary key, name varchar(100) null, description mediumtext not null, category int default 0 not null, - constraint Unquie - unique (name) -) - collate = utf8mb4_unicode_ci - auto_increment = 14; + constraint Unquie unique (name) +) collate = utf8mb4_unicode_ci + auto_increment = 14; create table if not exists menu ( - menu int auto_increment - primary key, + menu int auto_increment primary key, `right` int null, parent int null, name varchar(255) null, @@ -844,126 +650,96 @@ create table if not exists menu icon varchar(200) not null, class varchar(200) not null, position int default 0 not null, - constraint menu_ibfk_1 - foreign key (`right`) references rights (`right`) - on update cascade on delete set null, - constraint menu_ibfk_2 - foreign key (parent) references menu (menu) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci - auto_increment = 25; - -create or replace index parent - on menu (parent); - -create or replace index `right` - on menu (`right`); + constraint menu_ibfk_1 foreign key (`right`) references rights (`right`) on update cascade on delete set null, + constraint menu_ibfk_2 foreign key (parent) references menu (menu) on update cascade on delete set null +) collate = utf8mb4_unicode_ci + auto_increment = 25; + +create or replace index parent on menu (parent); + +create or replace index `right` on menu (`right`); create table if not exists modules ( - module int auto_increment - primary key, + module int auto_increment primary key, name varchar(255) null, `right` int null, - constraint fkMod2Rights - foreign key (`right`) references rights (`right`) - on update cascade on delete set null -) - collate = utf8mb4_unicode_ci - auto_increment = 31; + constraint fkMod2Rights foreign key (`right`) references rights (`right`) on update cascade on delete set null +) collate = utf8mb4_unicode_ci + auto_increment = 31; -create or replace index module_right - on modules (`right`); +create or replace index module_right on modules (`right`); create table if not exists roles ( - role int auto_increment - primary key, + role int auto_increment primary key, name varchar(50) not null, description mediumtext not null -) - collate = utf8mb4_unicode_ci - auto_increment = 6; +) collate = utf8mb4_unicode_ci + auto_increment = 6; create table if not exists roles2rights ( role int not null, `right` int not null, primary key (role, `right`), - constraint fkRight - foreign key (`right`) references rights (`right`) - on delete cascade, - constraint fkRole - foreign key (role) references roles (role) - on delete cascade -) - collate = utf8mb4_unicode_ci; - -create or replace index fkRight_idx - on roles2rights (`right`); + constraint fkRight foreign key (`right`) references rights (`right`) on delete cascade, + constraint fkRole foreign key (role) references roles (role) on delete cascade +) collate = utf8mb4_unicode_ci; + +create or replace index fkRight_idx on roles2rights (`right`); create table if not exists roomfinder_building2area ( area_id int not null, - building_nr varchar(8) not null - primary key, + building_nr varchar(8) not null primary key, campus char not null, name varchar(32) not null -) - charset = utf8mb4; +) charset = utf8mb4; grant select on table roomfinder_building2area to 'tca-students'; create table if not exists roomfinder_buildings ( - building_nr varchar(8) not null - primary key, + building_nr varchar(8) not null primary key, utm_zone varchar(4) null, utm_easting varchar(32) null, utm_northing varchar(32) null, default_map_id int null -) - charset = utf8mb4; +) charset = utf8mb4; grant select on table roomfinder_buildings to 'tca-students'; create table if not exists roomfinder_buildings2gps ( - id varchar(8) default '' not null - primary key, + id varchar(8) default '' not null primary key, latitude varchar(30) null, longitude varchar(30) null -) - charset = utf8mb4; +) charset = utf8mb4; create table if not exists roomfinder_buildings2maps ( building_nr varchar(8) not null, map_id int not null, primary key (building_nr, map_id) -) - charset = utf8mb4; +) charset = utf8mb4; grant select on table roomfinder_buildings2maps to 'tca-students'; create table if not exists roomfinder_maps ( - map_id int not null - primary key, + map_id int not null primary key, description varchar(64) not null, scale int not null, width int not null, height int not null -) - charset = utf8mb4; +) charset = utf8mb4; grant select on table roomfinder_maps to 'tca-students'; create table if not exists roomfinder_rooms ( - room_id int not null - primary key, + room_id int not null primary key, room_code varchar(32) null, building_nr varchar(8) null, arch_id varchar(16) null, @@ -977,8 +753,7 @@ create table if not exists roomfinder_rooms utm_northing varchar(32) null, unit_id int null, default_map_id int null -) - charset = utf8mb4; +) charset = utf8mb4; grant select on table roomfinder_rooms to 'tca-students'; @@ -987,8 +762,7 @@ create table if not exists roomfinder_rooms2maps room_id int not null, map_id int not null, primary key (room_id, map_id) -) - charset = utf8mb4; +) charset = utf8mb4; create table if not exists roomfinder_schedules ( @@ -998,28 +772,22 @@ create table if not exists roomfinder_schedules title varchar(64) not null, event_id int not null, course_code varchar(32) null, - constraint `unique` - unique (room_id, start, end) -) - charset = utf8mb4; + constraint `unique` unique (room_id, start, end) +) charset = utf8mb4; grant select on table roomfinder_schedules to 'tca-students'; create table if not exists sessions ( - session varchar(255) charset utf8 not null - primary key, + session varchar(255) charset utf8 not null primary key, access int unsigned null, data text null, - constraint session - unique (session) -) - collate = utf8mb4_unicode_ci; + constraint session unique (session) +) collate = utf8mb4_unicode_ci; create table if not exists tag ( - tag int auto_increment - primary key, + tag int auto_increment primary key, title varchar(255) null ); @@ -1028,42 +796,32 @@ create table if not exists card2tag tag int not null, card int not null, primary key (tag, card), - constraint card2tag_ibfk_1 - foreign key (tag) references tag (tag) - on delete cascade, - constraint card2tag_ibfk_2 - foreign key (card) references card (card) - on delete cascade + constraint card2tag_ibfk_1 foreign key (tag) references tag (tag) on delete cascade, + constraint card2tag_ibfk_2 foreign key (card) references card (card) on delete cascade ); -create or replace index card - on card2tag (card); +create or replace index card on card2tag (card); create table if not exists ticket_admin ( - ticket_admin int auto_increment - primary key, + ticket_admin int auto_increment primary key, `key` text not null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, active tinyint(1) default 0 not null, comment text null -) - charset = utf8 - auto_increment = 48; +) charset = utf8 + auto_increment = 48; create table if not exists ticket_group ( - ticket_group int auto_increment - primary key, + ticket_group int auto_increment primary key, description text not null -) - charset = utf8 - auto_increment = 2; +) charset = utf8 + auto_increment = 2; create table if not exists event ( - event int auto_increment - primary key, + event int auto_increment primary key, news int null, kino int null, file int null, @@ -1074,84 +832,57 @@ create table if not exists event start datetime /* mariadb-5.3 */ null, end datetime /* mariadb-5.3 */ null, ticket_group int default 1 null, - constraint fkEventFile - foreign key (file) references files (file) - on update cascade on delete set null, - constraint fkEventGroup - foreign key (ticket_group) references ticket_group (ticket_group), - constraint fkKino - foreign key (kino) references kino (kino) - on update cascade on delete set null, - constraint fkNews - foreign key (news) references news (news) - on update cascade on delete set null -) - charset = utf8 - auto_increment = 39; - -create or replace index file - on event (file); - -create or replace fulltext index searchTitle - on event (title); + constraint fkEventFile foreign key (file) references files (file) on update cascade on delete set null, + constraint fkEventGroup foreign key (ticket_group) references ticket_group (ticket_group), + constraint fkKino foreign key (kino) references kino (kino) on update cascade on delete set null, + constraint fkNews foreign key (news) references news (news) on update cascade on delete set null +) charset = utf8 + auto_increment = 39; + +create or replace index file on event (file); + +create or replace fulltext index searchTitle on event (title); create table if not exists ticket_admin2group ( - ticket_admin2group int auto_increment - primary key, + ticket_admin2group int auto_increment primary key, ticket_admin int not null, ticket_group int not null, - constraint fkTicketAdmin - foreign key (ticket_admin) references ticket_admin (ticket_admin) - on update cascade on delete cascade, - constraint fkTicketGroup - foreign key (ticket_group) references ticket_group (ticket_group) - on update cascade on delete cascade -) - charset = utf8 - auto_increment = 10; - -create or replace index ticket_admin - on ticket_admin2group (ticket_admin); - -create or replace index ticket_group - on ticket_admin2group (ticket_group); + constraint fkTicketAdmin foreign key (ticket_admin) references ticket_admin (ticket_admin) on update cascade on delete cascade, + constraint fkTicketGroup foreign key (ticket_group) references ticket_group (ticket_group) on update cascade on delete cascade +) charset = utf8 + auto_increment = 10; + +create or replace index ticket_admin on ticket_admin2group (ticket_admin); + +create or replace index ticket_group on ticket_admin2group (ticket_group); create table if not exists ticket_payment ( - ticket_payment int auto_increment - primary key, + ticket_payment int auto_increment primary key, name varchar(50) not null, min_amount int null, max_amount int null, config text not null -) - charset = utf8 - auto_increment = 3; +) charset = utf8 + auto_increment = 3; create table if not exists ticket_type ( - ticket_type int auto_increment - primary key, + ticket_type int auto_increment primary key, event int not null, ticket_payment int not null, price double not null, contingent int not null, description varchar(100) not null, - constraint fkEvent - foreign key (event) references event (event) - on update cascade on delete cascade, - constraint fkPayment - foreign key (ticket_payment) references ticket_payment (ticket_payment) - on update cascade -) - charset = utf8 - auto_increment = 57; + constraint fkEvent foreign key (event) references event (event) on update cascade on delete cascade, + constraint fkPayment foreign key (ticket_payment) references ticket_payment (ticket_payment) on update cascade +) charset = utf8 + auto_increment = 57; create table if not exists ticket_history ( - ticket_history int auto_increment - primary key, + ticket_history int auto_increment primary key, member int not null, ticket_payment int null, ticket_type int not null, @@ -1159,46 +890,32 @@ create table if not exists ticket_history redemption datetime /* mariadb-5.3 */ null, created timestamp /* mariadb-5.3 */ default current_timestamp() not null, code char(128) not null, - constraint fkMember - foreign key (member) references member (member) - on update cascade on delete cascade, - constraint fkTicketPayment - foreign key (ticket_payment) references ticket_payment (ticket_payment) - on update cascade, - constraint fkTicketType - foreign key (ticket_type) references ticket_type (ticket_type) - on update cascade -) - charset = utf8 - auto_increment = 776; - -create or replace index member - on ticket_history (member); - -create or replace index ticket_payment - on ticket_history (ticket_payment); - -create or replace index ticket_type - on ticket_history (ticket_type); - -create or replace index event - on ticket_type (event); - -create or replace index ticket_payment - on ticket_type (ticket_payment); + constraint fkMember foreign key (member) references member (member) on update cascade on delete cascade, + constraint fkTicketPayment foreign key (ticket_payment) references ticket_payment (ticket_payment) on update cascade, + constraint fkTicketType foreign key (ticket_type) references ticket_type (ticket_type) on update cascade +) charset = utf8 + auto_increment = 776; + +create or replace index member on ticket_history (member); + +create or replace index ticket_payment on ticket_history (ticket_payment); + +create or replace index ticket_type on ticket_history (ticket_type); + +create or replace index event on ticket_type (event); + +create or replace index ticket_payment on ticket_type (ticket_payment); create table if not exists update_note ( - version_code int not null - primary key, + version_code int not null primary key, version_name text null, message text null ); create table if not exists users ( - user int auto_increment - primary key, + user int auto_increment primary key, username varchar(7) null, firstname varchar(100) null, surname varchar(100) null, @@ -1211,93 +928,64 @@ create table if not exists users tum_id_employee varchar(50) null comment 'OBFUSCATED_ID_B', tum_id_alumni varchar(50) null comment 'OBFUSCATED_ID_EXT', tum_id_preferred varchar(50) null comment 'OBFUSCATED_ID_BEVORZUGT', - constraint username - unique (username) -) - collate = utf8mb4_unicode_ci - auto_increment = 434; + constraint username unique (username) +) collate = utf8mb4_unicode_ci + auto_increment = 434; create table if not exists log ( - log int auto_increment - primary key, + log int auto_increment primary key, time int not null, user_executed int null, user_affected int null, action int null, comment varchar(255) not null, - constraint fkLog2Actions - foreign key (action) references actions (action) - on update set null on delete set null, - constraint fkLog2UsersAf - foreign key (user_affected) references users (user) - on update set null on delete set null, - constraint fkLog2UsersEx - foreign key (user_executed) references users (user) - on update set null on delete set null -) - collate = utf8mb4_unicode_ci; - -create or replace index action - on log (action); - -create or replace index user - on log (user_executed); - -create or replace index user_affected - on log (user_affected); + constraint fkLog2Actions foreign key (action) references actions (action) on update set null on delete set null, + constraint fkLog2UsersAf foreign key (user_affected) references users (user) on update set null on delete set null, + constraint fkLog2UsersEx foreign key (user_executed) references users (user) on update set null on delete set null +) collate = utf8mb4_unicode_ci; + +create or replace index action on log (action); + +create or replace index user on log (user_executed); + +create or replace index user_affected on log (user_affected); create table if not exists recover ( - recover int auto_increment - primary key, + recover int auto_increment primary key, user int not null, created int not null, hash varchar(190) not null, ip varchar(255) not null, - constraint hash - unique (hash), - constraint fkRecover2User - foreign key (user) references users (user) - on delete cascade -) - collate = utf8mb4_unicode_ci; + constraint hash unique (hash), + constraint fkRecover2User foreign key (user) references users (user) on delete cascade +) collate = utf8mb4_unicode_ci; -create or replace index user - on recover (user); +create or replace index user on recover (user); create table if not exists users2info ( - user int not null - primary key, + user int not null primary key, firstname varchar(255) not null, surname varchar(255) not null, lastPwChange int not null, pager int default 15 null, - constraint fkUsers - foreign key (user) references users (user) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci; + constraint fkUsers foreign key (user) references users (user) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci; create table if not exists users2roles ( user int not null, role int not null, primary key (user, role), - constraint fkUser2RolesRole - foreign key (role) references roles (role) - on update cascade on delete cascade, - constraint fkUser2RolesUser - foreign key (user) references users (user) - on update cascade on delete cascade -) - collate = utf8mb4_unicode_ci; + constraint fkUser2RolesRole foreign key (role) references roles (role) on update cascade on delete cascade, + constraint fkUser2RolesUser foreign key (user) references users (user) on update cascade on delete cascade +) collate = utf8mb4_unicode_ci; create table if not exists wifi_measurement ( - id int(20) unsigned auto_increment - primary key, + id int(20) unsigned auto_increment primary key, date date not null, SSID varchar(32) not null, BSSID varchar(64) not null, @@ -1305,5 +993,4 @@ create table if not exists wifi_measurement accuracyInMeters float not null, latitude double not null, longitude double not null -) - collate = utf8mb4_unicode_ci; +) collate = utf8mb4_unicode_ci; From 5aa08e0143053513b802816ada025cb3799520a0 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 13:44:07 +0200 Subject: [PATCH 16/62] migrated the src schema to live in an migation --- .../backend/templates/sql-init-config.yaml | 8 ----- deployment/charts/backend/values.yaml | 1 - server/backend/migration/20200000000000.go | 30 +++++++++++++++++++ .../migration/static_data}/source-schema.sql | 0 4 files changed, 30 insertions(+), 9 deletions(-) delete mode 100644 deployment/charts/backend/templates/sql-init-config.yaml create mode 100644 server/backend/migration/20200000000000.go rename {deployment/charts/backend/files => server/backend/migration/static_data}/source-schema.sql (100%) diff --git a/deployment/charts/backend/templates/sql-init-config.yaml b/deployment/charts/backend/templates/sql-init-config.yaml deleted file mode 100644 index 5183fe11..00000000 --- a/deployment/charts/backend/templates/sql-init-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: sql-init - namespace: {{ $.Values.namespace }} -data: - schema.sql: |- -{{ .Files.Get "files/source-schema.sql" | indent 4 }} diff --git a/deployment/charts/backend/values.yaml b/deployment/charts/backend/values.yaml index 1d2fa341..9ac3673b 100644 --- a/deployment/charts/backend/values.yaml +++ b/deployment/charts/backend/values.yaml @@ -38,7 +38,6 @@ mariadb: storageClass: local-path volumePermissions: enabled: true - initdbScriptsConfigMap: sql-init backend: diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go new file mode 100644 index 00000000..ae064cf8 --- /dev/null +++ b/server/backend/migration/20200000000000.go @@ -0,0 +1,30 @@ +package migration + +import ( + _ "embed" + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +//go:embed static_data/source-schema.sql +var sourceSchema string + +// migrate20200000000000 +// adds the source shema +func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20200000000000", + Migrate: func(tx *gorm.DB) error { + return tx.Exec(sourceSchema).Error + }, + Rollback: func(tx *gorm.DB) error { + tables := []string{"actions", "alarm_ban", "alarm_log", "barrierFree_moreInfo", "barrierFree_persons", "card_type", "chat_room", "crontab", "curricula", "dish", "dishflags", "dish2dishflags", "faculty", "feedback", "files", "kino", "lecture", "location", "member", "card", "card_box", "card_comment", "card_option", "chat_message", "chat_room2members", "devices", "device2stats", "members_card", "members_card_answer_history", "mensa", "dish2mensa", "mensaplan_mensa", "mensaprices", "migrations", "newsSource", "news", "news_alert", "notification_type", "notification", "notification_confirmation", "openinghours", "question", "question2answer", "question2faculty", "questionAnswers", "reports", "rights", "menu", "modules", "roles", "roles2rights", "roomfinder_building2area", "roomfinder_buildings", "roomfinder_buildings2gps", "roomfinder_buildings2maps", "roomfinder_maps", "roomfinder_rooms", "roomfinder_rooms2maps", "roomfinder_schedules", "sessions", "tag", "card2tag", "ticket_admin", "ticket_group", "event", "ticket_admin2group", "ticket_payment", "ticket_type", "ticket_history", "update_note", "users", "log", "recover", "users2info", "users2roles", "wifi_measurement"} + for _, table := range tables { + if err := tx.Migrator().DropTable(table); err != nil { + return err + } + } + return nil + }, + } +} diff --git a/deployment/charts/backend/files/source-schema.sql b/server/backend/migration/static_data/source-schema.sql similarity index 100% rename from deployment/charts/backend/files/source-schema.sql rename to server/backend/migration/static_data/source-schema.sql From ac86c0b54a9b7b0f4238739753a675c61dba2536 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 13:45:09 +0200 Subject: [PATCH 17/62] fixed formatting issue --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c61da345..5147183c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: cache-dependency-path: | server/go.sum - name: wait for db - run: sleep 20 && docker logs $(docker ps -qa) && + run: sleep 20 && docker logs $(docker ps -qa) - name: run migration run: go run main.go working-directory: ./server From 1456efb52a98d97b8d914613d88825ccaad5878a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 13:46:32 +0200 Subject: [PATCH 18/62] fixed the migration not being used --- server/backend/migration/migration.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 710c333f..2e1c19aa 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -43,6 +43,7 @@ func (m TumDBMigrator) Migrate() error { ValidateUnknownMigrations: true, } mig := gormigrate.New(m.database, gormigrateOptions, []*gormigrate.Migration{ + m.migrate20200000000000(), m.migrate20210709193000(), m.migrate20220126230000(), m.migrate20220713000000(), From 8a559af0d75e4d72c0b110f88c23596293a13951 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:00:15 +0200 Subject: [PATCH 19/62] linting --- server/backend/migration/20200000000000.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index ae064cf8..b5d1affe 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -2,6 +2,7 @@ package migration import ( _ "embed" + "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" ) From a7258bb49a9752575b0b57d8c04e1c1522a0fc4f Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:01:55 +0200 Subject: [PATCH 20/62] tested different way of execing sql --- server/backend/migration/20200000000000.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index b5d1affe..93de6d2e 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -16,7 +16,7 @@ func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20200000000000", Migrate: func(tx *gorm.DB) error { - return tx.Exec(sourceSchema).Error + return tx.Raw(sourceSchema).Error }, Rollback: func(tx *gorm.DB) error { tables := []string{"actions", "alarm_ban", "alarm_log", "barrierFree_moreInfo", "barrierFree_persons", "card_type", "chat_room", "crontab", "curricula", "dish", "dishflags", "dish2dishflags", "faculty", "feedback", "files", "kino", "lecture", "location", "member", "card", "card_box", "card_comment", "card_option", "chat_message", "chat_room2members", "devices", "device2stats", "members_card", "members_card_answer_history", "mensa", "dish2mensa", "mensaplan_mensa", "mensaprices", "migrations", "newsSource", "news", "news_alert", "notification_type", "notification", "notification_confirmation", "openinghours", "question", "question2answer", "question2faculty", "questionAnswers", "reports", "rights", "menu", "modules", "roles", "roles2rights", "roomfinder_building2area", "roomfinder_buildings", "roomfinder_buildings2gps", "roomfinder_buildings2maps", "roomfinder_maps", "roomfinder_rooms", "roomfinder_rooms2maps", "roomfinder_schedules", "sessions", "tag", "card2tag", "ticket_admin", "ticket_group", "event", "ticket_admin2group", "ticket_payment", "ticket_type", "ticket_history", "update_note", "users", "log", "recover", "users2info", "users2roles", "wifi_measurement"} From 35e9224b627148dc8c306afd5c0a64fb2a9a3c53 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:18:29 +0200 Subject: [PATCH 21/62] tested even more different way of executing the source shema --- server/backend/migration/20200000000000.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index 93de6d2e..994c8830 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -2,6 +2,7 @@ package migration import ( _ "embed" + "strings" "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" @@ -16,7 +17,12 @@ func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20200000000000", Migrate: func(tx *gorm.DB) error { - return tx.Raw(sourceSchema).Error + for _, line := range strings.Split(sourceSchema, ";") { + if err := tx.Exec(line).Error; err != nil { + return err + } + } + return nil }, Rollback: func(tx *gorm.DB) error { tables := []string{"actions", "alarm_ban", "alarm_log", "barrierFree_moreInfo", "barrierFree_persons", "card_type", "chat_room", "crontab", "curricula", "dish", "dishflags", "dish2dishflags", "faculty", "feedback", "files", "kino", "lecture", "location", "member", "card", "card_box", "card_comment", "card_option", "chat_message", "chat_room2members", "devices", "device2stats", "members_card", "members_card_answer_history", "mensa", "dish2mensa", "mensaplan_mensa", "mensaprices", "migrations", "newsSource", "news", "news_alert", "notification_type", "notification", "notification_confirmation", "openinghours", "question", "question2answer", "question2faculty", "questionAnswers", "reports", "rights", "menu", "modules", "roles", "roles2rights", "roomfinder_building2area", "roomfinder_buildings", "roomfinder_buildings2gps", "roomfinder_buildings2maps", "roomfinder_maps", "roomfinder_rooms", "roomfinder_rooms2maps", "roomfinder_schedules", "sessions", "tag", "card2tag", "ticket_admin", "ticket_group", "event", "ticket_admin2group", "ticket_payment", "ticket_type", "ticket_history", "update_note", "users", "log", "recover", "users2info", "users2roles", "wifi_measurement"} From ff4d9494721d00dca094f24df460e01b8ff55e7f Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:21:08 +0200 Subject: [PATCH 22/62] fixed executing empty statements --- server/backend/migration/20200000000000.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index 994c8830..1c2f2a7e 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -18,6 +18,10 @@ func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { ID: "20200000000000", Migrate: func(tx *gorm.DB) error { for _, line := range strings.Split(sourceSchema, ";") { + line = strings.TrimSpace(line) + if line == "" { + continue + } if err := tx.Exec(line).Error; err != nil { return err } From b99e3ceeee0cb4dd882a518e64fd4cda121ea74e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:25:16 +0200 Subject: [PATCH 23/62] made sure only nessesary things are logged for `migrate20200000000000` --- server/backend/migration/20200000000000.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index 1c2f2a7e..80d2c19d 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -2,6 +2,8 @@ package migration import ( _ "embed" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/logger" "strings" "github.com/go-gormigrate/gormigrate/v2" @@ -17,12 +19,14 @@ func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20200000000000", Migrate: func(tx *gorm.DB) error { + tx = tx.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}) for _, line := range strings.Split(sourceSchema, ";") { line = strings.TrimSpace(line) if line == "" { continue } if err := tx.Exec(line).Error; err != nil { + log.WithError(err).WithField("line", line).Error("failed to execute line") return err } } From 592ee9106bd566801cac16833ac71f52aa2c028f Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:31:15 +0200 Subject: [PATCH 24/62] formatting --- server/backend/migration/20200000000000.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index 80d2c19d..0e22ba8c 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -2,9 +2,10 @@ package migration import ( _ "embed" + "strings" + log "github.com/sirupsen/logrus" "gorm.io/gorm/logger" - "strings" "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" From 30c348e966056116628b35c0cd08be2176b4b177 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:46:16 +0200 Subject: [PATCH 25/62] refactored the migrations to not depend on the TumDBMigrator struct --- server/backend/migration/20200000000000.go | 4 +- server/backend/migration/20210709193000.go | 2 +- server/backend/migration/20220126230000.go | 2 +- server/backend/migration/20220713000000.go | 2 +- server/backend/migration/20221119131300.go | 2 +- server/backend/migration/20221210000000.go | 2 +- server/backend/migration/20230530000000.go | 2 +- server/backend/migration/20230825000000.go | 2 +- server/backend/migration/20230826000000.go | 2 +- server/backend/migration/20230904000000.go | 2 +- server/backend/migration/20230904100000.go | 2 +- server/backend/migration/20231003000000.go | 2 +- server/backend/migration/migration.go | 75 ++++++++++------------ server/utils/db.go | 2 +- 14 files changed, 49 insertions(+), 54 deletions(-) diff --git a/server/backend/migration/20200000000000.go b/server/backend/migration/20200000000000.go index 0e22ba8c..c8b7e376 100644 --- a/server/backend/migration/20200000000000.go +++ b/server/backend/migration/20200000000000.go @@ -15,8 +15,8 @@ import ( var sourceSchema string // migrate20200000000000 -// adds the source shema -func (m TumDBMigrator) migrate20200000000000() *gormigrate.Migration { +// adds the source schema +func migrate20200000000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20200000000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20210709193000.go b/server/backend/migration/20210709193000.go index 4968b74e..d18a3cb5 100644 --- a/server/backend/migration/20210709193000.go +++ b/server/backend/migration/20210709193000.go @@ -20,7 +20,7 @@ type File struct { // adds a "url" column to the database containing the url the file was downloaded from. // adds a "finished" column to the database that indicates, that a files download is finished. // adds a "fileDownload" cron job that runs every 5 minutes. -func (m TumDBMigrator) migrate20210709193000() *gormigrate.Migration { +func migrate20210709193000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20210709193000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20220126230000.go b/server/backend/migration/20220126230000.go index d6bd0f6f..da802364 100644 --- a/server/backend/migration/20220126230000.go +++ b/server/backend/migration/20220126230000.go @@ -8,7 +8,7 @@ import ( // migrate20220126230000 // adds a fulltext index to the roomfinder_rooms table -func (m TumDBMigrator) migrate20220126230000() *gormigrate.Migration { +func migrate20220126230000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20220126230000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20220713000000.go b/server/backend/migration/20220713000000.go index 0f3eadf5..3281c8bd 100644 --- a/server/backend/migration/20220713000000.go +++ b/server/backend/migration/20220713000000.go @@ -8,7 +8,7 @@ import ( //migrate20210709193000 -func (m TumDBMigrator) migrate20220713000000() *gormigrate.Migration { +func migrate20220713000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20220713000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20221119131300.go b/server/backend/migration/20221119131300.go index 8ea73cea..fd96ba3b 100644 --- a/server/backend/migration/20221119131300.go +++ b/server/backend/migration/20221119131300.go @@ -16,7 +16,7 @@ import ( //go:embed static_data/iosInitialSchedulingPriorities.json var iosInitialPrioritiesFile []byte -func (m TumDBMigrator) migrate20221119131300() *gormigrate.Migration { +func migrate20221119131300() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20221119131300", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20221210000000.go b/server/backend/migration/20221210000000.go index 9ec49d3c..45ac4c47 100644 --- a/server/backend/migration/20221210000000.go +++ b/server/backend/migration/20221210000000.go @@ -9,7 +9,7 @@ import ( // migrate20221210000000 // adds a "canteenHeadCount" cron job that runs every 5 minutes. -func (m TumDBMigrator) migrate20221210000000() *gormigrate.Migration { +func migrate20221210000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20221210000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index add100b8..9e044a70 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -26,7 +26,7 @@ type NewExamResultsSubscriber struct { LastNotifiedAt null.Time } -func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { +func migrate20230530000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230530000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20230825000000.go b/server/backend/migration/20230825000000.go index db35ca9d..b35630c0 100644 --- a/server/backend/migration/20230825000000.go +++ b/server/backend/migration/20230825000000.go @@ -9,7 +9,7 @@ import ( // migrate20230825000000 // Removes the ability to run chat cronjobs -func (m TumDBMigrator) migrate20230825000000() *gormigrate.Migration { +func migrate20230825000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230825000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20230826000000.go b/server/backend/migration/20230826000000.go index 3b8fe0dc..92bb8e40 100644 --- a/server/backend/migration/20230826000000.go +++ b/server/backend/migration/20230826000000.go @@ -20,7 +20,7 @@ func (n *Feedback) TableName() string { // migrate20230826000000 // adds a "feedbackEmail" cron job that runs every 30 minutes. -func (m TumDBMigrator) migrate20230826000000() *gormigrate.Migration { +func migrate20230826000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230826000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20230904000000.go b/server/backend/migration/20230904000000.go index 6c96cba6..aa15f2e6 100644 --- a/server/backend/migration/20230904000000.go +++ b/server/backend/migration/20230904000000.go @@ -9,7 +9,7 @@ import ( // migrate20230904000000 // Removes ticketsales from the db-enums -func (m TumDBMigrator) migrate20230904000000() *gormigrate.Migration { +func migrate20230904000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230904000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20230904100000.go b/server/backend/migration/20230904100000.go index c744432c..d2c515b9 100644 --- a/server/backend/migration/20230904100000.go +++ b/server/backend/migration/20230904100000.go @@ -19,7 +19,7 @@ type NewsSource struct { // migrate20230904100000 // migrates the crontab from kino to movie crontab -func (m TumDBMigrator) migrate20230904100000() *gormigrate.Migration { +func migrate20230904100000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230904100000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20231003000000.go b/server/backend/migration/20231003000000.go index b15eea95..1c0b696f 100644 --- a/server/backend/migration/20231003000000.go +++ b/server/backend/migration/20231003000000.go @@ -84,7 +84,7 @@ var staticData embed.FS // migrate20231003000000 // migrates the static data for the canteen rating system and adds the necessary cronjob entries -func (m TumDBMigrator) migrate20231003000000() *gormigrate.Migration { +func migrate20231003000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20231003000000", Migrate: func(tx *gorm.DB) error { diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 2e1c19aa..3d9f14eb 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -8,33 +8,20 @@ import ( "gorm.io/gorm" ) -// TumDBMigrator contains a reference to our database -type TumDBMigrator struct { - database *gorm.DB - shouldAutoMigrate bool -} - -// New creates a new TumDBMigrator with a database -func New(db *gorm.DB, shouldAutoMigrate bool) TumDBMigrator { - return TumDBMigrator{database: db, shouldAutoMigrate: shouldAutoMigrate} +func autoMigrate(db *gorm.DB) error { + err := db.AutoMigrate( + &model.TopNews{}, + &model.Crontab{}, + &model.File{}, + &model.NewsSource{}, + &model.NewsAlert{}, + &model.News{}, + &model.CanteenHeadCount{}, + ) + return err } -// Migrate starts the migration either by using AutoMigrate in development environments or manually in prod -func (m TumDBMigrator) Migrate() error { - if m.shouldAutoMigrate { - log.Info("Using automigration") - err := m.database.AutoMigrate( - &model.TopNews{}, - &model.Crontab{}, - &model.File{}, - &model.NewsSource{}, - &model.NewsAlert{}, - &model.News{}, - &model.CanteenHeadCount{}, - ) - return err - } - log.Info("Using manual migration") +func manualMigrate(db *gorm.DB) error { gormigrateOptions := &gormigrate.Options{ TableName: gormigrate.DefaultOptions.TableName, IDColumnName: gormigrate.DefaultOptions.IDColumnName, @@ -42,21 +29,29 @@ func (m TumDBMigrator) Migrate() error { UseTransaction: true, ValidateUnknownMigrations: true, } - mig := gormigrate.New(m.database, gormigrateOptions, []*gormigrate.Migration{ - m.migrate20200000000000(), - m.migrate20210709193000(), - m.migrate20220126230000(), - m.migrate20220713000000(), - m.migrate20221119131300(), - m.migrate20221210000000(), - m.migrate20230825000000(), - m.migrate20230904000000(), - m.migrate20230530000000(), - m.migrate20230904100000(), - m.migrate20230826000000(), - m.migrate20231003000000(), + mig := gormigrate.New(db, gormigrateOptions, []*gormigrate.Migration{ + migrate20200000000000(), + migrate20210709193000(), + migrate20220126230000(), + migrate20220713000000(), + migrate20221119131300(), + migrate20221210000000(), + migrate20230825000000(), + migrate20230904000000(), + migrate20230530000000(), + migrate20230904100000(), + migrate20230826000000(), + migrate20231003000000(), }) - err := mig.Migrate() - return err + return mig.Migrate() +} +// Migrate starts the migration either by using AutoMigrate in development environments or manually in prod +func Migrate(db *gorm.DB, shouldAutoMigrate bool) error { + log.WithField("shouldAutoMigrate", shouldAutoMigrate).Debug("starting migration") + if shouldAutoMigrate { + return autoMigrate(db) + } else { + return manualMigrate(db) + } } diff --git a/server/utils/db.go b/server/utils/db.go index 3c9cb471..f89a85eb 100644 --- a/server/utils/db.go +++ b/server/utils/db.go @@ -25,7 +25,7 @@ func SetupDB() *gorm.DB { // Migrate the schema // currently not activated as - if err := migration.New(db, false).Migrate(); err != nil { + if err := migration.Migrate(db, false); err != nil { log.WithError(err).Fatal("Failed to migrate database") } From 7403693da5b95317a1be40fafc2000d4294f590d Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 14:47:00 +0200 Subject: [PATCH 26/62] renamed EXIT_AFTER_MIGRATION -> CI_EXIT_AFTER_MIGRATION --- .github/workflows/main.yml | 2 +- server/utils/db.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5147183c..c0c6d603 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: run: go run main.go working-directory: ./server env: - EXIT_AFTER_MIGRATION: "true" + CI_EXIT_AFTER_MIGRATION: "true" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev build: diff --git a/server/utils/db.go b/server/utils/db.go index f89a85eb..59d7bdb4 100644 --- a/server/utils/db.go +++ b/server/utils/db.go @@ -29,7 +29,7 @@ func SetupDB() *gorm.DB { log.WithError(err).Fatal("Failed to migrate database") } - if os.Getenv("EXIT_AFTER_MIGRATION") == "true" { + if os.Getenv("CI_EXIT_AFTER_MIGRATION") == "true" { log.Info("Exiting after migration") os.Exit(0) } From e7d82530dcb0917d6a043096cbfc66d78fbee234 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 15:09:37 +0200 Subject: [PATCH 27/62] tested a different way of migrating the newsSource table --- server/backend/migration/static_data/source-schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/backend/migration/static_data/source-schema.sql b/server/backend/migration/static_data/source-schema.sql index b33181b3..12afcc11 100644 --- a/server/backend/migration/static_data/source-schema.sql +++ b/server/backend/migration/static_data/source-schema.sql @@ -467,7 +467,7 @@ create table if not exists newsSource source int auto_increment primary key, title mediumtext not null, url mediumtext null, - icon int null, + icon int not null, hook enum ('newspread', 'impulsivHook') null, constraint newsSource_ibfk_1 foreign key (icon) references files (file) on update cascade on delete set null ) collate = utf8mb4_unicode_ci From ff7672572a56669d56587cac4da0eca4ed751257 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 15:14:14 +0200 Subject: [PATCH 28/62] fixed another typo --- server/backend/migration/static_data/source-schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/backend/migration/static_data/source-schema.sql b/server/backend/migration/static_data/source-schema.sql index 12afcc11..f7fea9c7 100644 --- a/server/backend/migration/static_data/source-schema.sql +++ b/server/backend/migration/static_data/source-schema.sql @@ -469,7 +469,7 @@ create table if not exists newsSource url mediumtext null, icon int not null, hook enum ('newspread', 'impulsivHook') null, - constraint newsSource_ibfk_1 foreign key (icon) references files (file) on update cascade on delete set null + constraint newsSource_ibfk_1 foreign key (icon) references files (file) on update cascade on delete cascade ) collate = utf8mb4_unicode_ci auto_increment = 17; From 959503b7fe146f8463a4565bfe74782bca6a35c9 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 15:27:39 +0200 Subject: [PATCH 29/62] removed an automigration --- server/backend/migration/20230904100000.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/backend/migration/20230904100000.go b/server/backend/migration/20230904100000.go index 866782f3..cd785772 100644 --- a/server/backend/migration/20230904100000.go +++ b/server/backend/migration/20230904100000.go @@ -38,9 +38,6 @@ func migrate20230904100000() *gormigrate.Migration { if err := SafeEnumAdd(tx, &model.Crontab{}, "type", "movie"); err != nil { return err } - if err := tx.AutoMigrate(&NewsSource{}); err != nil { - return err - } // tu film news source is now inlined if err := tx.Delete(&NewsSource{Source: 2}).Error; err != nil { return err From f4922e72c96bbbbe8e0b11bd3b26d92e3fc08b81 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 15:36:48 +0200 Subject: [PATCH 30/62] removed now obsolete docs --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 350490e2..a1c4d7db 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,6 @@ The docker compose will start the server and a mariadb instance. The server will be available at `localhost:50051` and the mariadb instance at `localhost:3306`. Additionally, docker creates the volume `campus-db-data` to persist the data of the mariadb instances. -### Setting up the Database -The mariadb schema can be installed by executing the following command inside the mariadb container: -```bash -mysql --user=root --password=secret_root_password campus_db < /entrypoint/schema.sql -``` - ### Environment Variables The following environment variables need to be set for the server to work properly: * [REQUIRED] `DB_NAME`: The name of the database to use. From 908043ae0c520bb9b49230d0fd22bf9ff0b89d2b Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 16:01:35 +0200 Subject: [PATCH 31/62] removed the source schema from the docker-compose file to make running it possible --- docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9bc5cc4a..e876f031 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -39,7 +39,6 @@ services: - MARIADB_DATABASE=${DB_NAME} volumes: - campus-db-data:/bitnami/mariadb - - ./deployment/charts/backend/files/source-schema.sql:/docker-entrypoint-startdb.d/schema.sql healthcheck: test: ['CMD', '/opt/bitnami/scripts/mariadb/healthcheck.sh'] interval: 15s From 99d126de4634372bc642148f4e9dbf4c13aee6ce Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 17:53:34 +0200 Subject: [PATCH 32/62] added all models to the migration --- server/backend/migration/migration.go | 43 ++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 3d9f14eb..69c86e3b 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -10,13 +10,48 @@ import ( func autoMigrate(db *gorm.DB) error { err := db.AutoMigrate( - &model.TopNews{}, + &model.Cafeteria{}, + &model.CafeteriaRating{}, + &model.CafeteriaRatingAverage{}, + &model.CafeteriaRatingTag{}, + &model.CafeteriaRatingTagAverage{}, + &model.CafeteriaRatingTagOption{}, + &model.CanteenHeadCount{}, &model.Crontab{}, + &model.Device{}, + &model.Dish{}, + &model.DishNameTag{}, + &model.DishNameTagAverage{}, + &model.DishNameTagOption{}, + &model.DishNameTagOptionExcluded{}, + &model.DishNameTagOptionIncluded{}, + &model.DishRating{}, + &model.DishRatingAverage{}, + &model.DishRatingTag{}, + &model.DishRatingTagAverage{}, + &model.DishRatingTagOption{}, + &model.DishToDishNameTag{}, + &model.DishesOfTheWeek{}, + &model.PublishedExamResult{}, + &model.Feedback{}, &model.File{}, - &model.NewsSource{}, - &model.NewsAlert{}, + &model.IOSDevice{}, + //&model.IOSDeviceLastUpdated{}, + &model.IOSDeviceRequestLog{}, + &model.IOSDevicesActivityReset{}, + &model.IOSGrade{}, + //&model.IOSRemoteNotification{}, + &model.IOSScheduledUpdateLog{}, + &model.IOSSchedulingPriority{}, + &model.Kino{}, + &model.NewExamResultsSubscriber{}, &model.News{}, - &model.CanteenHeadCount{}, + &model.NewsAlert{}, + &model.NewsSource{}, + &model.Notification{}, + &model.NotificationConfirmation{}, + &model.NotificationType{}, + &model.UpdateNote{}, ) return err } From 8ef8eda28cb56da226be20ddd649c85e55508b90 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 18:01:43 +0200 Subject: [PATCH 33/62] fixed naming issues for the db models --- server/backend/migration/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 69c86e3b..b20b80ec 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -40,7 +40,7 @@ func autoMigrate(db *gorm.DB) error { &model.IOSDeviceRequestLog{}, &model.IOSDevicesActivityReset{}, &model.IOSGrade{}, - //&model.IOSRemoteNotification{}, + //&model.IOSRemoteNotification...{}, -- wtf??? &model.IOSScheduledUpdateLog{}, &model.IOSSchedulingPriority{}, &model.Kino{}, From 0a189ee194ff6fc6cc79aeae40db5e2bc33d0a48 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 18:17:24 +0200 Subject: [PATCH 34/62] enabled the auto migration in the workflow --- .github/workflows/main.yml | 22 +++++++++++++++++++--- server/utils/db.go | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0c6d603..7579d2b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,13 +20,20 @@ jobs: test_migrations: runs-on: ubuntu-latest services: - mariadb: + auto_mariadb: image: bitnami/mariadb:latest ports: - 3306:3306 env: MARIADB_ROOT_PASSWORD: super_secret_passw0rd MARIADB_DATABASE: campus_db + manual_mariadb: + image: bitnami/mariadb:latest + ports: + - 3300:3306 + env: + MARIADB_ROOT_PASSWORD: super_secret_passw0rd + MARIADB_DATABASE: campus_db steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 @@ -35,14 +42,23 @@ jobs: cache-dependency-path: | server/go.sum - name: wait for db - run: sleep 20 && docker logs $(docker ps -qa) - - name: run migration + run: sleep 20 + - name: run manual migrations run: go run main.go working-directory: ./server env: CI_EXIT_AFTER_MIGRATION: "true" + CI_AUTO_MIGRATION: "false" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev + - name: run auto migrations + run: go run main.go + working-directory: ./server + env: + CI_EXIT_AFTER_MIGRATION: "true" + CI_AUTO_MIGRATION: "true" + DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + ENVIRONMENT: dev build: runs-on: ubuntu-latest needs: [test, test_migrations] diff --git a/server/utils/db.go b/server/utils/db.go index 59d7bdb4..802852ac 100644 --- a/server/utils/db.go +++ b/server/utils/db.go @@ -25,7 +25,7 @@ func SetupDB() *gorm.DB { // Migrate the schema // currently not activated as - if err := migration.Migrate(db, false); err != nil { + if err := migration.Migrate(db, os.Getenv("CI_AUTO_MIGRATION") == "true"); err != nil { log.WithError(err).Fatal("Failed to migrate database") } From 0f0c4c32ddae158496bd07b6b31b0bedf11301ae Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 22 Oct 2023 23:38:43 +0200 Subject: [PATCH 35/62] removed model.IOSGrade --- server/backend/migration/migration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 81000403..2c477d31 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -38,10 +38,10 @@ func autoMigrate(db *gorm.DB) error { &model.Feedback{}, &model.File{}, &model.IOSDevice{}, - //&model.IOSDeviceLastUpdated{}, + //&model.IOSDeviceLastUpdated{}, -- not a gorm model &model.IOSDeviceRequestLog{}, &model.IOSDevicesActivityReset{}, - &model.IOSGrade{}, + //&model.IOSGrade{}, -- not a gorm model //&model.IOSRemoteNotification...{}, -- wtf??? &model.IOSScheduledUpdateLog{}, &model.IOSSchedulingPriority{}, From 589e528b132f182e126ddbbd33059185dd8d85de Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 00:32:11 +0200 Subject: [PATCH 36/62] tested a different way of defining Device.LastAccess --- server/model/device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/device.go b/server/model/device.go index 6542aa24..61291cb2 100755 --- a/server/model/device.go +++ b/server/model/device.go @@ -12,7 +12,7 @@ type Device struct { Member null.Int `gorm:"column:member;type:int;" json:"member"` UUID string `gorm:"column:uuid;type:varchar(50);" json:"uuid"` Created null.Time `gorm:"column:created;type:timestamp;" json:"created"` - LastAccess time.Time `gorm:"column:lastAccess;type:timestamp;default:0000-00-00 00:00:00;" json:"last_access"` + LastAccess time.Time `gorm:"column:lastAccess;type:timestamp;default:'0000-00-00 00:00:00';" json:"last_access"` LastAPI string `gorm:"column:lastApi;type:text;size:16777215;" json:"last_api"` Developer string `gorm:"column:developer;type:char;size:5;default:false;" json:"developer"` OsVersion string `gorm:"column:osVersion;type:text;size:16777215;" json:"os_version"` From cddf5399575f5075e257999f33a430610fb8d024 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 00:35:53 +0200 Subject: [PATCH 37/62] fixed the type of `Device.Developer` --- server/model/device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/device.go b/server/model/device.go index 61291cb2..d47ddd18 100755 --- a/server/model/device.go +++ b/server/model/device.go @@ -14,7 +14,7 @@ type Device struct { Created null.Time `gorm:"column:created;type:timestamp;" json:"created"` LastAccess time.Time `gorm:"column:lastAccess;type:timestamp;default:'0000-00-00 00:00:00';" json:"last_access"` LastAPI string `gorm:"column:lastApi;type:text;size:16777215;" json:"last_api"` - Developer string `gorm:"column:developer;type:char;size:5;default:false;" json:"developer"` + Developer string `gorm:"column:developer;type:enum('true','false');default:'false';" json:"developer"` OsVersion string `gorm:"column:osVersion;type:text;size:16777215;" json:"os_version"` AppVersion string `gorm:"column:appVersion;type:text;size:16777215;" json:"app_version"` Counter int32 `gorm:"column:counter;type:int;default:0;" json:"counter"` From 3034ab1b2c200aed63ae06816bb724a554780add Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 00:38:14 +0200 Subject: [PATCH 38/62] fixed the type of `Device.PkActive` --- server/model/device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/device.go b/server/model/device.go index d47ddd18..52ca2707 100755 --- a/server/model/device.go +++ b/server/model/device.go @@ -19,7 +19,7 @@ type Device struct { AppVersion string `gorm:"column:appVersion;type:text;size:16777215;" json:"app_version"` Counter int32 `gorm:"column:counter;type:int;default:0;" json:"counter"` Pk null.String `gorm:"column:pk;type:text;size:4294967295;" json:"pk"` - PkActive string `gorm:"column:pkActive;type:char;size:5;default:false;" json:"pk_active"` + PkActive string `gorm:"column:pkActive;type:enum('true', 'false');default:'false';" json:"pk_active"` GcmToken null.String `gorm:"column:gcmToken;type:text;size:65535;" json:"gcm_token"` GcmStatus null.String `gorm:"column:gcmStatus;type:varchar(200);" json:"gcm_status"` ConfirmationKey null.String `gorm:"column:confirmationKey;type:varchar(35);" json:"confirmation_key"` From 77f810485d35ba744bfaf427c9f6434096ee7946 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 00:41:07 +0200 Subject: [PATCH 39/62] fixed the type of `NotificationType.Confirmation` --- server/model/notification_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/notification_type.go b/server/model/notification_type.go index 661383d7..7a53f240 100644 --- a/server/model/notification_type.go +++ b/server/model/notification_type.go @@ -19,5 +19,5 @@ var ( type NotificationType struct { Type int64 `gorm:"primary_key;AUTO_INCREMENT;column:type;type:int;" json:"type"` Name string `gorm:"column:name;type:text;size:65535;" json:"name"` - Confirmation string `gorm:"column:confirmation;type:char;size:5;default:false;" json:"confirmation"` + Confirmation string `gorm:"column:confirmation;type:enum('true', 'false');default:'false';" json:"confirmation"` } From 77142afc183110f43e8f6b6499bfc054b07de756 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:06:15 +0200 Subject: [PATCH 40/62] added an mysqldump configuration --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7579d2b6..e401e12d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,6 +59,8 @@ jobs: CI_AUTO_MIGRATION: "true" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev + - name: export auto databases + run: mysqldump --host localhost:3300 --user root --password super_secret_passw0rd --no-data campus_db --result-file=./auto.sql && cat ./auto.sql build: runs-on: ubuntu-latest needs: [test, test_migrations] From 6d9640d98bc42e8f8830b64fef9390bd58a48ecf Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:10:41 +0200 Subject: [PATCH 41/62] added an mysqldump configuration --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e401e12d..098935ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,6 +51,8 @@ jobs: CI_AUTO_MIGRATION: "false" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev + - name: export manual database + run: mysqldump --user root --password super_secret_passw0rd --no-data campus_db --result-file=./manual.sql && cat ./manual.sql - name: run auto migrations run: go run main.go working-directory: ./server @@ -59,8 +61,6 @@ jobs: CI_AUTO_MIGRATION: "true" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev - - name: export auto databases - run: mysqldump --host localhost:3300 --user root --password super_secret_passw0rd --no-data campus_db --result-file=./auto.sql && cat ./auto.sql build: runs-on: ubuntu-latest needs: [test, test_migrations] From ede64c87032220925dcac3e0c358c1a4831c6cfa Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:13:23 +0200 Subject: [PATCH 42/62] tested different connection method --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 098935ba..ff420630 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev - name: export manual database - run: mysqldump --user root --password super_secret_passw0rd --no-data campus_db --result-file=./manual.sql && cat ./manual.sql + run: mysqldump --host localhost --port=3306 --user=root --password=super_secret_passw0rd --no-data campus_db --result-file=./manual.sql && cat ./manual.sql - name: run auto migrations run: go run main.go working-directory: ./server From d1a7c6c023e71656d1e190c1b9f7eecd177521a3 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:26:10 +0200 Subject: [PATCH 43/62] switched to atlas for shema diffing --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff420630..7a112489 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,8 +51,6 @@ jobs: CI_AUTO_MIGRATION: "false" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev - - name: export manual database - run: mysqldump --host localhost --port=3306 --user=root --password=super_secret_passw0rd --no-data campus_db --result-file=./manual.sql && cat ./manual.sql - name: run auto migrations run: go run main.go working-directory: ./server @@ -61,6 +59,9 @@ jobs: CI_AUTO_MIGRATION: "true" DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local ENVIRONMENT: dev + - uses: ariga/setup-atlas@master + - name: export diff the migrations + run: atlas schema diff --from "maria://root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --to "maria://root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local" build: runs-on: ubuntu-latest needs: [test, test_migrations] From ddfcdb721f68e9113a2b044dacb25b27f76eaa42 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:28:45 +0200 Subject: [PATCH 44/62] tested different connection strings --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a112489..d6ee6b4b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,7 @@ jobs: ENVIRONMENT: dev - uses: ariga/setup-atlas@master - name: export diff the migrations - run: atlas schema diff --from "maria://root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --to "maria://root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local" + run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" build: runs-on: ubuntu-latest needs: [test, test_migrations] From f39bdc8b1ac22277d6700cf82088cc54f8f0e8f6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 01:34:50 +0200 Subject: [PATCH 45/62] added additional formatting --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6ee6b4b..2c624d8f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,7 @@ jobs: ENVIRONMENT: dev - uses: ariga/setup-atlas@master - name: export diff the migrations - run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" + run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' build: runs-on: ubuntu-latest needs: [test, test_migrations] From 18c1468ce5756620df1a04090193a724db933b31 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:23:01 +0200 Subject: [PATCH 46/62] refactored the action --- .github/workflows/main.yml | 47 +------------------- .github/workflows/test_migration.yml | 64 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/test_migration.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c624d8f..56123e25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,54 +17,9 @@ jobs: - name: run tests run: go test -v ./... working-directory: ./server - test_migrations: - runs-on: ubuntu-latest - services: - auto_mariadb: - image: bitnami/mariadb:latest - ports: - - 3306:3306 - env: - MARIADB_ROOT_PASSWORD: super_secret_passw0rd - MARIADB_DATABASE: campus_db - manual_mariadb: - image: bitnami/mariadb:latest - ports: - - 3300:3306 - env: - MARIADB_ROOT_PASSWORD: super_secret_passw0rd - MARIADB_DATABASE: campus_db - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: | - server/go.sum - - name: wait for db - run: sleep 20 - - name: run manual migrations - run: go run main.go - working-directory: ./server - env: - CI_EXIT_AFTER_MIGRATION: "true" - CI_AUTO_MIGRATION: "false" - DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local - ENVIRONMENT: dev - - name: run auto migrations - run: go run main.go - working-directory: ./server - env: - CI_EXIT_AFTER_MIGRATION: "true" - CI_AUTO_MIGRATION: "true" - DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local - ENVIRONMENT: dev - - uses: ariga/setup-atlas@master - - name: export diff the migrations - run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' build: runs-on: ubuntu-latest - needs: [test, test_migrations] + needs: [test] steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml new file mode 100644 index 00000000..b357dbeb --- /dev/null +++ b/.github/workflows/test_migration.yml @@ -0,0 +1,64 @@ +name: CI +on: + pull_request: + branches: [ main ] +jobs: + test_migrations: + runs-on: ubuntu-latest + services: + auto_mariadb: + image: bitnami/mariadb:latest + ports: + - 3306:3306 + env: + MARIADB_ROOT_PASSWORD: super_secret_passw0rd + MARIADB_DATABASE: campus_db + manual_mariadb: + image: bitnami/mariadb:latest + ports: + - 3300:3306 + env: + MARIADB_ROOT_PASSWORD: super_secret_passw0rd + MARIADB_DATABASE: campus_db + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache-dependency-path: | + server/go.sum + - name: wait for db + run: sleep 20 + - name: run manual migrations + run: go run main.go + working-directory: ./server + env: + CI_EXIT_AFTER_MIGRATION: "true" + CI_AUTO_MIGRATION: "false" + DB_DSN: root:super_secret_passw0rd@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + ENVIRONMENT: dev + - name: run auto migrations + run: go run main.go + working-directory: ./server + env: + CI_EXIT_AFTER_MIGRATION: "true" + CI_AUTO_MIGRATION: "true" + DB_DSN: root:super_secret_passw0rd@tcp(localhost:3300)/campus_db?charset=utf8mb4&parseTime=True&loc=Local + ENVIRONMENT: dev + - uses: ariga/setup-atlas@master + - name: export diff the migrations + run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: "${{ github.event.number }}" + body-includes: Found the following differences in the sql schema + comment-author: github-actions[bot] + - run: | + curl \ + -X POST \ + ${{ github.event.pull_request.comments_url }} \ + -H "Content-Type: application/json" \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Block with all the details
" }' \ No newline at end of file From de8569b0ac6329fa0d405aeed1535e33ed535526 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:28:13 +0200 Subject: [PATCH 47/62] fixed a linting issue --- .github/workflows/test_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index b357dbeb..4775d5ba 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -61,4 +61,4 @@ jobs: ${{ github.event.pull_request.comments_url }} \ -H "Content-Type: application/json" \ -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Block with all the details
" }' \ No newline at end of file + --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Block with all the details
" }' From 3d44fc1eba42391005d3dd24e58d861c58016bc5 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:28:34 +0200 Subject: [PATCH 48/62] updated the block --- .github/workflows/test_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 4775d5ba..84daf285 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -61,4 +61,4 @@ jobs: ${{ github.event.pull_request.comments_url }} \ -H "Content-Type: application/json" \ -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Block with all the details
" }' + --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Updated block
" }' From bc1fc0f915b669baa10bdd63b01826312f036d6c Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:30:29 +0200 Subject: [PATCH 49/62] added a concurrency setting --- .github/workflows/test_migration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 84daf285..a7e5cd8c 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -2,6 +2,9 @@ name: CI on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true jobs: test_migrations: runs-on: ubuntu-latest From cd7139e85b6408e7bace3b14da3852ed4785aaa1 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:31:14 +0200 Subject: [PATCH 50/62] test concurrecny --- .github/workflows/test_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index a7e5cd8c..bedae278 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -64,4 +64,4 @@ jobs: ${{ github.event.pull_request.comments_url }} \ -H "Content-Type: application/json" \ -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one woud need to Updated block
" }' + --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to Test concurrency
" }' From 14e7ebeadbc58e2aa5df4925c5a91da975a88f56 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:32:58 +0200 Subject: [PATCH 51/62] test if `wait for db` is nessesary --- .github/workflows/test_migration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index bedae278..e4e196ad 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -30,8 +30,6 @@ jobs: go-version: '1.21' cache-dependency-path: | server/go.sum - - name: wait for db - run: sleep 20 - name: run manual migrations run: go run main.go working-directory: ./server From fdc8c6485818dea7986eb5b38fe6b950e3655428 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:44:18 +0200 Subject: [PATCH 52/62] tested a different commenting mechanism --- .github/workflows/test_migration.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index e4e196ad..5c26cf9c 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -56,10 +56,18 @@ jobs: issue-number: "${{ github.event.number }}" body-includes: Found the following differences in the sql schema comment-author: github-actions[bot] - - run: | - curl \ - -X POST \ - ${{ github.event.pull_request.comments_url }} \ - -H "Content-Type: application/json" \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - --data '{ "body": "Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to Test concurrency
" }' + - name: Create comment + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: "${{ github.event.number }}" + comment-id: "${{ steps.fc.outputs.comment-id }}" + body: | + Found the following differences in the sql schema: +
+ To get from local migrations to auto migration state one would need to + + create-or-update-comment + +
+ edit-mode: replace + reactions: eyes From 87a0f7b4ed8bee4babeed66487a1ffd245785c63 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:45:49 +0200 Subject: [PATCH 53/62] fixed linting issues --- .github/workflows/test_migration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 5c26cf9c..fa868e4e 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -65,9 +65,7 @@ jobs: Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to - create-or-update-comment -
edit-mode: replace reactions: eyes From 59eed7e7834301bde62f2685d3cfa07dbb44cd46 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:46:27 +0200 Subject: [PATCH 54/62] fixed the name of the action --- .github/workflows/test_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index fa868e4e..3bf3684e 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -1,4 +1,4 @@ -name: CI +name: Migration Test on: pull_request: branches: [ main ] From a05ac8ccaa69a84a58c3c790b172b2cc7caadb9a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:47:51 +0200 Subject: [PATCH 55/62] tested if the comment is edited --- .github/workflows/test_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 3bf3684e..08a64284 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -65,7 +65,7 @@ jobs: Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to - create-or-update-comment + create-or-update-comment, should edit the comment
edit-mode: replace reactions: eyes From 8b2c3b067e242f6d6aab446e3109f953bc91a9ea Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 11:53:17 +0200 Subject: [PATCH 56/62] plugged the diff_migrations to the commenting step --- .github/workflows/test_migration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 08a64284..0f83b1d8 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -48,7 +48,8 @@ jobs: ENVIRONMENT: dev - uses: ariga/setup-atlas@master - name: export diff the migrations - run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' + id: diff_migrations + run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT - name: Find Comment uses: peter-evans/find-comment@v2 id: fc @@ -65,7 +66,7 @@ jobs: Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to - create-or-update-comment, should edit the comment + ${{join(steps.diff_migrations.outputs.*, '\n')}}
edit-mode: replace reactions: eyes From d4c55c4334662565756b2a3901eacce74810268e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 12:00:18 +0200 Subject: [PATCH 57/62] added an eof delimiter --- .github/workflows/test_migration.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 0f83b1d8..7a9ad926 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -49,7 +49,12 @@ jobs: - uses: ariga/setup-atlas@master - name: export diff the migrations id: diff_migrations - run: atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT + run: | + export message=atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' && + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) && + echo "text<<$EOF" >> $GITHUB_OUTPUT && + echo "$message" >> $GITHUB_OUTPUT && + echo "$EOF" >> $GITHUB_OUTPUT - name: Find Comment uses: peter-evans/find-comment@v2 id: fc From 7f1ba3d6992d4092617f199d8364672083b72d8a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 12:01:52 +0200 Subject: [PATCH 58/62] tested different delimiter mechanism --- .github/workflows/test_migration.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 7a9ad926..e1b75b4c 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -50,10 +50,9 @@ jobs: - name: export diff the migrations id: diff_migrations run: | - export message=atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' && EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) && echo "text<<$EOF" >> $GITHUB_OUTPUT && - echo "$message" >> $GITHUB_OUTPUT && + atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT && echo "$EOF" >> $GITHUB_OUTPUT - name: Find Comment uses: peter-evans/find-comment@v2 From a20ed02ea789fa945c5e5fcb504e5890bf03b1e6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 12:34:16 +0200 Subject: [PATCH 59/62] tested a different formatting --- .github/workflows/test_migration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index e1b75b4c..03637ee1 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -70,7 +70,11 @@ jobs: Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to + + ```sql ${{join(steps.diff_migrations.outputs.*, '\n')}} + ``` +
edit-mode: replace reactions: eyes From 3ef038f322bbb435063fc748dcd5ace720ff6861 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 13:45:13 +0200 Subject: [PATCH 60/62] added a better diff --- .github/workflows/test_migration.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 03637ee1..8a0e7b94 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -51,7 +51,10 @@ jobs: id: diff_migrations run: | EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) && - echo "text<<$EOF" >> $GITHUB_OUTPUT && + echo "local_to_auto<<$EOF" >> $GITHUB_OUTPUT && + atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT && + echo "$EOF" >> $GITHUB_OUTPUT + echo "auto_to_local<<$EOF" >> $GITHUB_OUTPUT && atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT && echo "$EOF" >> $GITHUB_OUTPUT - name: Find Comment @@ -72,7 +75,15 @@ jobs: To get from local migrations to auto migration state one would need to ```sql - ${{join(steps.diff_migrations.outputs.*, '\n')}} + ${{ steps.diff_migrations.outputs.local_to_auto }} + ``` + + +
+ To get from auto migrations to local migration state one would need to + + ```sql + ${{ steps.diff_migrations.outputs.auto_to_local }} ```
From 26034dde042fb6da300c0477119cb35e34603298 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 13:46:06 +0200 Subject: [PATCH 61/62] tested if the formatting works --- .github/workflows/test_migration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index 8a0e7b94..b9f346fc 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -73,19 +73,19 @@ jobs: Found the following differences in the sql schema:
To get from local migrations to auto migration state one would need to - + ```sql ${{ steps.diff_migrations.outputs.local_to_auto }} ``` - +
To get from auto migrations to local migration state one would need to - + ```sql ${{ steps.diff_migrations.outputs.auto_to_local }} ``` - +
edit-mode: replace reactions: eyes From 5e05f6274715dc7586ac435b60baf1eac528c4ab Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 23 Oct 2023 13:53:31 +0200 Subject: [PATCH 62/62] reformatted the message --- .github/workflows/test_migration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_migration.yml b/.github/workflows/test_migration.yml index b9f346fc..a8b9b51b 100644 --- a/.github/workflows/test_migration.yml +++ b/.github/workflows/test_migration.yml @@ -70,9 +70,10 @@ jobs: issue-number: "${{ github.event.number }}" comment-id: "${{ steps.fc.outputs.comment-id }}" body: | - Found the following differences in the sql schema: + :eyes: Found the following differences in the sql schema: +
- To get from local migrations to auto migration state one would need to + Needed get from local to auto migration state ```sql ${{ steps.diff_migrations.outputs.local_to_auto }} @@ -80,7 +81,7 @@ jobs:
- To get from auto migrations to local migration state one would need to + Needed from auto to local migration state ```sql ${{ steps.diff_migrations.outputs.auto_to_local }} @@ -88,4 +89,3 @@ jobs:
edit-mode: replace - reactions: eyes