From 39f122d539ae088cc287034bfd93e406af55a1e6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 18 Sep 2023 00:42:35 +0200 Subject: [PATCH 1/3] fixed `static_data` not being included into the binary (#194) --- .../backend/cafeteriaRatingDBInitializer.go | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/server/backend/cafeteriaRatingDBInitializer.go b/server/backend/cafeteriaRatingDBInitializer.go index 500a93d4..454aca80 100644 --- a/server/backend/cafeteriaRatingDBInitializer.go +++ b/server/backend/cafeteriaRatingDBInitializer.go @@ -2,9 +2,8 @@ package backend import ( "database/sql" + "embed" "encoding/json" - "os" - "path/filepath" "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/guregu/null" @@ -30,13 +29,16 @@ type nameTag struct { CanBeIncluded []string `json:"canbeincluded"` } +//go:embed static_data +var staticData embed.FS + /* Writes all available tags from the json file into tables in order to make them easier to use. Will be executed once while the server is started. */ func initTagRatingOptions(db *gorm.DB) { - updateTagTable("backend/static_data/dishRatingTags.json", db, DISH) - updateTagTable("backend/static_data/cafeteriaRatingTags.json", db, CAFETERIA) + updateTagTable("static_data/dishRatingTags.json", db, DISH) + updateTagTable("static_data/cafeteriaRatingTags.json", db, CAFETERIA) updateNameTagOptions(db) addEntriesForCronJob(db, "averageRatingComputation", 300) addEntriesForCronJob(db, "dishNameDownload", 302400) //run twice every week @@ -70,8 +72,7 @@ If a tag with the exact german and english name does not exist yet, it will be c Old tags won't be removed to prevent problems with foreign keys. */ func updateNameTagOptions(db *gorm.DB) { - absPathDishNames, _ := filepath.Abs("backend/static_data/dishNameTags.json") - tagsNames := generateNameTagListFromFile(absPathDishNames) + tagsNames := generateNameTagListFromFile("static_data/dishNameTags.json") for _, v := range tagsNames.MultiLanguageNameTags { var parentId int32 res := db.Model(&model.DishNameTagOption{}). @@ -154,8 +155,7 @@ If an entry with the same German and English name exists, the entry won't be add The TagType is used to identify the corresponding model */ func updateTagTable(path string, db *gorm.DB, tagType modelType) { - absPathDish, _ := filepath.Abs(path) - tagsDish := generateRatingTagListFromFile(absPathDish) + tagsDish := generateRatingTagListFromFile(path) insertModel := getTagModel(tagType, db) for _, v := range tagsDish.MultiLanguageTags { var count int64 @@ -200,42 +200,29 @@ func getTagModel(tagType modelType, db *gorm.DB) *gorm.DB { } func generateNameTagListFromFile(path string) multiLanguageNameTags { - file := readFromFile(path) + file, err := staticData.ReadFile(path) + if err != nil { + log.WithError(err).Error("Error including json.") + } var tags multiLanguageNameTags - errjson := json.NewDecoder(file).Decode(&tags) + errjson := json.Unmarshal(file, &tags) if errjson != nil { - log.WithError(errjson).Error("Error while reading the file.") + log.WithError(errjson).Error("Error parsing nameTagList to json.") } - defer func(jsonFile *os.File) { - if err := jsonFile.Close(); err != nil { - log.WithError(err).Error("Error in parsing json.") - } - }(file) return tags } func generateRatingTagListFromFile(path string) multiLanguageTags { - file := readFromFile(path) + file, err := staticData.ReadFile(path) + if err != nil { + log.WithError(err).Error("Error including json.") + } + var tags multiLanguageTags - errjson := json.NewDecoder(file).Decode(&tags) + errjson := json.Unmarshal(file, &tags) if errjson != nil { - log.WithError(errjson).Error("Error while reading or parsing the file.") + log.WithError(errjson).Error("Error parsing ratingTagList to json.") } - defer func(jsonFile *os.File) { - if err := jsonFile.Close(); err != nil { - log.WithError(err).Error("Error in parsing json.") - } - }(file) return tags } - -func readFromFile(path string) *os.File { - jsonFile, err := os.Open(path) - - if err != nil { - log.WithError(err).Error("Unable to open file with path: ", path) - } - - return jsonFile -} From 92b5a0a5637e2aa0e1da734ec310087278be9807 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 18 Sep 2023 00:44:21 +0200 Subject: [PATCH 2/3] fixed GOPATH always being on the path during regeneration script executions --- server/api/generate.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/generate.bash b/server/api/generate.bash index 1e0c93e7..8f356c6d 100755 --- a/server/api/generate.bash +++ b/server/api/generate.bash @@ -6,6 +6,7 @@ echo making sure that this script is run from $BASEDIR pushd $BASEDIR > /dev/null echo updating the generated files +export PATH="$PATH:$(go env GOPATH)/bin" buf mod update || exit 1 buf generate || exit 1 From d9919663117b44289e1bd09ade334192d6e8d32b Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 18 Sep 2023 01:11:01 +0200 Subject: [PATCH 3/3] refactored all usages of sql.NullXYZ to instead use the null.XYZ syntax and helper fucntions for constructing the fileds (#229) --- server/backend/cafeteriaRatingDBInitializer.go | 3 +-- server/backend/cron/news.go | 13 ++++++------- server/backend/migration/20210709193000.go | 8 +++----- server/backend/migration/20221119131300.go | 10 ++-------- server/backend/migration/20221210000000.go | 4 +--- server/backend/migration/20230825000000.go | 4 +--- server/backend/migration/20230904000000.go | 4 +--- server/backend/newsAlerts_test.go | 8 ++++---- server/backend/news_test.go | 12 ++++++------ server/model/files.go | 12 ++++++------ server/model/iosDeviceLastUpdated.go | 9 +++++---- 11 files changed, 36 insertions(+), 51 deletions(-) diff --git a/server/backend/cafeteriaRatingDBInitializer.go b/server/backend/cafeteriaRatingDBInitializer.go index 454aca80..c4879559 100644 --- a/server/backend/cafeteriaRatingDBInitializer.go +++ b/server/backend/cafeteriaRatingDBInitializer.go @@ -1,7 +1,6 @@ package backend import ( - "database/sql" "embed" "encoding/json" @@ -57,7 +56,7 @@ func addEntriesForCronJob(db *gorm.DB, cronName string, interval int32) { errCreate := db.Model(&model.Crontab{}). Create(&model.Crontab{ Interval: interval, - Type: null.String{NullString: sql.NullString{String: cronName, Valid: true}}, + Type: null.StringFrom(cronName), LastRun: 0, }).Error if errCreate != nil { diff --git a/server/backend/cron/news.go b/server/backend/cron/news.go index 02198907..473357a1 100644 --- a/server/backend/cron/news.go +++ b/server/backend/cron/news.go @@ -2,7 +2,6 @@ package cron import ( "crypto/md5" - "database/sql" "errors" "fmt" "regexp" @@ -91,7 +90,7 @@ func (c *CronService) parseNewsFeed(source model.NewsSource) error { if !skipNews(existingNewsLinksForSource, item.Link) { // pick the first enclosure that is an image (if any) var pickedEnclosure *gofeed.Enclosure - var enclosureUrl = null.String{NullString: sql.NullString{Valid: true, String: ""}} + var enclosureUrl = null.StringFrom("") for _, enclosure := range item.Enclosures { if strings.HasSuffix(enclosure.URL, "jpg") || strings.HasSuffix(enclosure.URL, "jpeg") || @@ -101,13 +100,13 @@ func (c *CronService) parseNewsFeed(source model.NewsSource) error { break } } - var fileId = null.Int{NullInt64: sql.NullInt64{Valid: false}} + var fileId = null.Int{} if pickedEnclosure != nil { fileId, err = c.saveImage(pickedEnclosure.URL) if err != nil { log.WithError(err).Error("can't save news image") } - enclosureUrl = null.String{NullString: sql.NullString{String: pickedEnclosure.URL, Valid: true}} + enclosureUrl = null.StringFrom(pickedEnclosure.URL) } bm := bluemonday.StrictPolicy() sanitizedDesc := bm.Sanitize(item.Description) @@ -155,8 +154,8 @@ func (c *CronService) saveImage(url string) (null.Int, error) { file := model.Files{ Name: targetFileName, Path: ImageDirectory, - URL: sql.NullString{String: url, Valid: true}, - Downloaded: sql.NullBool{Bool: false, Valid: true}, + URL: null.StringFrom(url), + Downloaded: null.BoolFrom(false), } err := c.db.Transaction(func(tx *gorm.DB) error { if err := tx.Create(&file).Error; err != nil { @@ -169,7 +168,7 @@ func (c *CronService) saveImage(url string) (null.Int, error) { return null.Int{}, err } // creating this int is annoying but i'm too afraid to use real ORM in the model - return null.Int{NullInt64: sql.NullInt64{Int64: int64(file.File), Valid: true}}, nil + return null.IntFrom(int64(file.File)), nil } // skipNews returns true if link is in existingLinks or link is invalid diff --git a/server/backend/migration/20210709193000.go b/server/backend/migration/20210709193000.go index c8e3967a..98c98255 100644 --- a/server/backend/migration/20210709193000.go +++ b/server/backend/migration/20210709193000.go @@ -1,8 +1,6 @@ package migration import ( - "database/sql" - "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" "github.com/guregu/null" @@ -18,8 +16,8 @@ func (m TumDBMigrator) migrate20210709193000() *gormigrate.Migration { ID: "20210709193000", Migrate: func(tx *gorm.DB) error { type Files struct { - URL sql.NullString `gorm:"column:url;default:null;" json:"url"` // URL of the file source (if any) - Downloaded sql.NullBool `gorm:"column:downloaded;type:boolean;default:1;" json:"downloaded"` // true when file is ready to be served, false when still being downloaded + URL null.String `gorm:"column:url;default:null;" json:"url"` // URL of the file source (if any) + Downloaded null.Bool `gorm:"column:downloaded;type:boolean;default:1;" json:"downloaded"` // true when file is ready to be served, false when still being downloaded } if err := tx.AutoMigrate( &Files{}, @@ -29,7 +27,7 @@ func (m TumDBMigrator) migrate20210709193000() *gormigrate.Migration { } return tx.Create(&model.Crontab{ Interval: 300, - Type: null.String{NullString: sql.NullString{String: "fileDownload", Valid: true}}, + Type: null.StringFrom("fileDownload"), }).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/server/backend/migration/20221119131300.go b/server/backend/migration/20221119131300.go index 393bf9ee..ffdbb938 100644 --- a/server/backend/migration/20221119131300.go +++ b/server/backend/migration/20221119131300.go @@ -1,7 +1,6 @@ package migration import ( - "database/sql" _ "embed" "encoding/json" @@ -53,7 +52,7 @@ func (m TumDBMigrator) migrate20221119131300() *gormigrate.Migration { err := tx.Create(&model.Crontab{ Interval: 60, - Type: null.String{NullString: sql.NullString{String: cron.IOSNotifications, Valid: true}}, + Type: null.StringFrom(cron.IOSNotifications), }).Error if err != nil { @@ -62,12 +61,7 @@ func (m TumDBMigrator) migrate20221119131300() *gormigrate.Migration { } return tx.Create(&model.Crontab{ - Type: null.String{ - NullString: sql.NullString{ - String: cron.IOSActivityReset, - Valid: true, - }, - }, + Type: null.StringFrom(cron.IOSActivityReset), Interval: 86400, }).Error }, diff --git a/server/backend/migration/20221210000000.go b/server/backend/migration/20221210000000.go index 4baac263..54dc7f3f 100644 --- a/server/backend/migration/20221210000000.go +++ b/server/backend/migration/20221210000000.go @@ -1,8 +1,6 @@ package migration import ( - "database/sql" - "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" "github.com/guregu/null" @@ -30,7 +28,7 @@ func (m TumDBMigrator) migrate20221210000000() *gormigrate.Migration { return tx.Create(&model.Crontab{ Interval: 60 * 5, // Every 5 minutes - Type: null.String{NullString: sql.NullString{String: "canteenHeadCount", Valid: true}}, + Type: null.StringFrom("canteenHeadCount"), }).Error }, diff --git a/server/backend/migration/20230825000000.go b/server/backend/migration/20230825000000.go index 2e49d370..c426ed13 100644 --- a/server/backend/migration/20230825000000.go +++ b/server/backend/migration/20230825000000.go @@ -1,8 +1,6 @@ package migration import ( - "database/sql" - "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" "github.com/guregu/null" @@ -26,7 +24,7 @@ func (m TumDBMigrator) migrate20230825000000() *gormigrate.Migration { } return tx.Create(&model.Crontab{ Interval: 60 * 10, // Every 10 minutes - Type: null.String{NullString: sql.NullString{String: "chat", Valid: true}}, + Type: null.StringFrom("chat"), }).Error }, } diff --git a/server/backend/migration/20230904000000.go b/server/backend/migration/20230904000000.go index b397022b..d9c0a227 100644 --- a/server/backend/migration/20230904000000.go +++ b/server/backend/migration/20230904000000.go @@ -1,8 +1,6 @@ package migration import ( - "database/sql" - "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" "github.com/guregu/null" @@ -31,7 +29,7 @@ func (m TumDBMigrator) migrate20230904000000() *gormigrate.Migration { } return tx.Create(&model.Crontab{ Interval: 60 * 10, // Every 10 minutes - Type: null.String{NullString: sql.NullString{String: "ticketsales", Valid: true}}, + Type: null.StringFrom("ticketsales"), }).Error }, } diff --git a/server/backend/newsAlerts_test.go b/server/backend/newsAlerts_test.go index 68c7e1eb..ffbb0579 100644 --- a/server/backend/newsAlerts_test.go +++ b/server/backend/newsAlerts_test.go @@ -60,11 +60,11 @@ func (s *NewsAlertSuite) Test_GetTopNewsOne() { Name: "Tournament_app_02-02.png", Path: "newsalerts/", Downloads: 0, - URL: sql.NullString{Valid: false}, - Downloaded: sql.NullBool{Bool: true, Valid: true}, + URL: null.String{}, + Downloaded: null.Bool{}, }, - Name: null.String{NullString: sql.NullString{String: "Exzellenzuniversität", Valid: true}}, - Link: null.String{NullString: sql.NullString{String: "https://tum.de", Valid: true}}, + Name: null.StringFrom("Exzellenzuniversität"), + Link: null.StringFrom("https://tum.de"), Created: time.Time.Add(time.Now(), time.Hour*-4), From: time.Time.Add(time.Now(), time.Hour*-2), To: time.Time.Add(time.Now(), time.Hour*2), diff --git a/server/backend/news_test.go b/server/backend/news_test.go index 6345bf7a..df6d87bd 100644 --- a/server/backend/news_test.go +++ b/server/backend/news_test.go @@ -52,8 +52,8 @@ func file(id int32) *model.Files { Name: fmt.Sprintf("src_%d.png", id), Path: "news/sources", Downloads: 1, - URL: sql.NullString{Valid: false}, - Downloaded: sql.NullBool{Bool: true, Valid: true}, + URL: null.String{}, + Downloaded: null.BoolFrom(true), } } @@ -61,10 +61,10 @@ func source1() *model.NewsSource { return &model.NewsSource{ Source: 1, Title: "Amazing News 1", - URL: null.String{NullString: sql.NullString{String: "https://example.com/amazing1", Valid: true}}, + URL: null.StringFrom("https://example.com/amazing1"), FilesID: file(2).File, Files: *file(2), - Hook: null.String{NullString: sql.NullString{String: "", Valid: true}}, + Hook: null.StringFrom(""), } } @@ -72,10 +72,10 @@ func source2() *model.NewsSource { return &model.NewsSource{ Source: 2, Title: "Amazing News 2", - URL: null.String{NullString: sql.NullString{String: "https://example.com/amazing2", Valid: true}}, + URL: null.StringFrom("https://example.com/amazing2"), FilesID: file(2).File, Files: *file(2), - Hook: null.String{NullString: sql.NullString{String: "hook", Valid: true}}, + Hook: null.StringFrom("hook"), } } diff --git a/server/model/files.go b/server/model/files.go index 01c64983..7ece1387 100644 --- a/server/model/files.go +++ b/server/model/files.go @@ -17,12 +17,12 @@ var ( // Files struct is a row record of the files table in the tca database type Files struct { - File int32 `gorm:"primary_key;AUTO_INCREMENT;column:file;type:int;" json:"file"` - Name string `gorm:"column:name;type:text;size:16777215;" json:"name"` - Path string `gorm:"column:path;type:text;size:16777215;" json:"path"` - Downloads int32 `gorm:"column:downloads;type:int;default:0;" json:"downloads"` - URL sql.NullString `gorm:"column:url;default:null;" json:"url"` // URL of the files source (if any) - Downloaded sql.NullBool `gorm:"column:downloaded;type:boolean;default:1;" json:"downloaded"` // true when file is ready to be served, false when still being downloaded + File int32 `gorm:"primary_key;AUTO_INCREMENT;column:file;type:int;" json:"file"` + Name string `gorm:"column:name;type:text;size:16777215;" json:"name"` + Path string `gorm:"column:path;type:text;size:16777215;" json:"path"` + Downloads int32 `gorm:"column:downloads;type:int;default:0;" json:"downloads"` + URL null.String `gorm:"column:url;default:null;" json:"url"` // URL of the files source (if any) + Downloaded null.Bool `gorm:"column:downloaded;type:boolean;default:1;" json:"downloaded"` // true when file is ready to be served, false when still being downloaded } // TableName sets the insert table name for this struct type diff --git a/server/model/iosDeviceLastUpdated.go b/server/model/iosDeviceLastUpdated.go index d06c185e..be79699c 100644 --- a/server/model/iosDeviceLastUpdated.go +++ b/server/model/iosDeviceLastUpdated.go @@ -1,16 +1,17 @@ package model import ( - "database/sql" "fmt" + + "github.com/guregu/null" ) // IOSDeviceLastUpdated used as a result of a query that joins // IOSDevice and IOSDeviceRequestLog tables. type IOSDeviceLastUpdated struct { - DeviceID string `json:"deviceId"` - LastUpdated sql.NullTime `json:"lastUpdated"` - PublicKey string `json:"publicKey"` + DeviceID string `json:"deviceId"` + LastUpdated null.Time `json:"lastUpdated"` + PublicKey string `json:"publicKey"` } func (device *IOSDeviceLastUpdated) String() string {