Skip to content

Commit

Permalink
Mariadb to mysql migration (#356)
Browse files Browse the repository at this point in the history
* changed from mariadb to mysql
* removed a race-condition from the testing logic
  • Loading branch information
CommanderStorm authored May 12, 2024
1 parent 37c993e commit bda8b29
Show file tree
Hide file tree
Showing 27 changed files with 367 additions and 269 deletions.
24 changes: 10 additions & 14 deletions .github/workflows/test_migration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,20 @@ jobs:
test_migrations:
runs-on: ubuntu-latest
services:
auto_mariadb:
image: bitnami/mariadb:latest
auto_mysql:
image: mysql:8
ports:
- 3306:3306
env:
MARIADB_ROOT_PASSWORD: super_secret_passw0rd
MARIADB_DATABASE: campus_db
MARIADB_CHARACTER_SET: utf8mb4
MARIADB_COLLATE: utf8mb4_unicode_ci
manual_mariadb:
image: bitnami/mariadb:latest
MYSQL_ROOT_PASSWORD: super_secret_passw0rd
MYSQL_DATABASE: campus_db
manual_mysql:
image: mysql:8
ports:
- 3300:3306
env:
MARIADB_ROOT_PASSWORD: super_secret_passw0rd
MARIADB_DATABASE: campus_db
MARIADB_CHARACTER_SET: utf8mb4
MARIADB_COLLATE: utf8mb4_unicode_ci
MYSQL_ROOT_PASSWORD: super_secret_passw0rd
MYSQL_DATABASE: campus_db
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
Expand Down Expand Up @@ -58,10 +54,10 @@ jobs:
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) &&
echo "local_to_auto<<$EOF" >> $GITHUB_OUTPUT &&
atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT &&
atlas schema diff --from "mysql://root:super_secret_passw0rd@localhost:3300/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --to "mysql://root:super_secret_passw0rd@localhost:3306/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT &&
echo "$EOF" >> $GITHUB_OUTPUT
echo "auto_to_local<<$EOF" >> $GITHUB_OUTPUT &&
atlas schema diff --from "maria://root:super_secret_passw0rd@localhost:3306/campus_db" --to "maria://root:super_secret_passw0rd@localhost:3300/campus_db" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT &&
atlas schema diff --from "mysql://root:super_secret_passw0rd@localhost:3306/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --to "mysql://root:super_secret_passw0rd@localhost:3300/campus_db?charset=utf8mb4&parseTime=True&loc=Local" --format '{{ sql . " " }}' >> $GITHUB_OUTPUT &&
echo "$EOF" >> $GITHUB_OUTPUT
- name: Find Comment
uses: peter-evans/find-comment@v3
Expand Down
4 changes: 2 additions & 2 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
"mode": "auto",
"program": "${workspaceFolder}/server/main.go",
"env": {
"DB_DSN": "gorm:GORM_USER_PASSWORD@tcp(localhost:3306)/campus_backend",
"DB_NAME": "campus_backend"
"DB_DSN": "gorm:GORM_USER_PASSWORD@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local",
"DB_NAME": "campus_db"
}
},
{
Expand Down
40 changes: 9 additions & 31 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,10 @@ The API is publicly available for anyone, but most notably, it's the main backen
### Installing Requirements

The backend uses MySQL as its backend for storing data.
In the following, we provide instructions for installing [MariaDB](https://mariadb.org/) as the DB server of choice.

#### Fedora

```bash
sudo dnf install mariadb-server

# Start the MariaDB server
sudo systemctl start mariadb

# Optional: Enable autostart
sudo systemctl enable mariadb
```

More details are available here: https://docs.fedoraproject.org/en-US/quick-docs/installing-mysql-mariadb/

#### Debian/Ubuntu
While it is possible to install [mysql](https://mysql.com/) natively (instructions are on their website), we recommend the following:

```bash
sudo apt install mariadb-server

# Start the MariaDB server
sudo systemctl start mariadb

# Optional: Enable autostart
sudo systemctl enable mariadb
docker run
```

### Setting up the DB
Expand All @@ -61,26 +39,26 @@ To start the server there are environment variables, as well as command line opt

```bash
cd server
export DB_DSN="Your gorm DB connection string for example: gorm:GORM_USER_PASSWORD@tcp(localhost:3306)/campus_backend"
export DB_DSN="The DB-name from above string for example: campus_backend"
export DB_DSN="Your gorm DB connection string for example: gorm:GORM_USER_PASSWORD@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local"
export DB_NAME="The DB-name from above string for example: campus_backend"
go run ./main.go
```

#### Environment Variables

There are a few environment variables available:

* [REQUIRED] `DB_DSN`: The [GORM](https://gorm.io/) [DB connection string](https://gorm.io/docs/connecting_to_the_database.html#MySQL) for connecting to the MySQL DB. Example: `gorm@tcp(localhost:3306)/campus_backend`
* [REQUIRED] `DB_DSN`: The name of the database from above connection string. Example: `campus_backend`
* [REQUIRED] `DB_DSN`: The [GORM](https://gorm.io/) [DB connection string](https://gorm.io/docs/connecting_to_the_database.html#MySQL) for connecting to the MySQL DB. Example: `gorm@tcp(localhost:3306)/campus_db?charset=utf8mb4&parseTime=True&loc=Local`
* [REQUIRED] `DB_NAME`: The name of the database from above connection string. Example: `campus_backend`
* [OPTIONAL] `SENTRY_DSN`: The Sentry [Data Source Name](https://sentry-docs-git-patch-1.sentry.dev/product/sentry-basics/dsn-explainer/) for reporting issues and crashes.

## Running the Server (Docker)
```bash
docker compose -f docker-compose.local.yml up -d
```
The docker compose will start the server and a mariadb instance (=> without the grpc-web layer and without routing/certificates to worry about)
The server will be available at `localhost:50051` and the mariadb instance at `localhost:3306`.
Additionally, docker creates the volume `campus-db-data` to persist the data of the mariadb instances.
The docker compose will start the server and a mysql instance (=> without the grpc-web layer and without routing/certificates to worry about)
The server will be available at `localhost:50051` and the mysql instance at `localhost:3306`.
Additionally, docker creates the volume `campus-db-data` to persist the data of the mysql instances.

### Environment Variables
The following environment variables need to be set for the server to work properly:
Expand Down
40 changes: 20 additions & 20 deletions docker-compose.local.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,28 @@

services:
db:
image: mysql:8
restart: no # in dev, this avoids having to crawl through the restart and makes a crash more obvious ^^
ports:
- "${DB_PORT:-3306}:3306"
environment:
- MYSQL_ROOT_PASSWORD=${DB_USER_PASSWORD}
- MYSQL_DATABASE=${DB_NAME}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
volumes:
- mysql-data:/var/lib/mysql
healthcheck:
test: /usr/bin/mysql --user=root --password=${DB_USER_PASSWORD} --execute "SHOW DATABASES"
interval: 2s
timeout: 20s
retries: 10
backend:
build:
context: server/
args:
version: dev # compiled with the git sha in prod
restart: always
restart: no # in dev, this avoids having to crawl through the restart and makes a crash more obvious ^^
ports:
- "50051:50051"
environment:
Expand All @@ -26,26 +43,9 @@ services:
depends_on:
db:
condition: service_healthy
db:
image: bitnami/mariadb:latest
restart: unless-stopped
ports:
- "${DB_PORT:-3306}:3306"
environment:
- MARIADB_ROOT_PASSWORD=${DB_USER_PASSWORD}
- MARIADB_DATABASE=${DB_NAME}
- MARIADB_CHARACTER_SET=utf8mb4
- MARIADB_COLLATE=utf8mb4_unicode_ci
volumes:
- campus-db-data:/bitnami/mariadb
healthcheck:
test: ['CMD', '/opt/bitnami/scripts/mariadb/healthcheck.sh']
interval: 15s
timeout: 5s
retries: 6

volumes:
campus-db-data:
mysql-data:
driver: local
backend-storage:
driver: local
59 changes: 34 additions & 25 deletions server/backend/feedback.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err
feedback := &model.Feedback{EmailId: id.String(), Recipient: "[email protected]"}

// download images
dbPath := path.Join("feedback", feedback.EmailId)
dbPath := path.Join(cron.StorageDir, "feedback", feedback.EmailId)
var uploadedFilenames []*string
for {
req, err := stream.Recv()
Expand All @@ -43,7 +43,7 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err
}
if err != nil {
log.WithError(err).Error("Error receiving feedback")
deleteUploaded(feedback.EmailId)
deleteUploaded(dbPath)
return status.Error(codes.Internal, "Error receiving feedback")
}
mergeFeedback(feedback, req)
Expand All @@ -57,9 +57,6 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err
}
feedback.ImageCount = int32(len(uploadedFilenames))
// validate feedback
feedback.Feedback = strings.TrimSpace(feedback.Feedback)
feedback.Feedback = strings.ReplaceAll(feedback.Feedback, " ", " ")
feedback.Feedback = strings.ToValidUTF8(feedback.Feedback, "?")
if feedback.Feedback == "" && feedback.ImageCount == 0 {
return status.Error(codes.InvalidArgument, "Please attach an image or feedback for us")
}
Expand All @@ -68,6 +65,7 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err
fiveMinutesAgo := now.Add(time.Minute * -5).Unix()
lastFeedback, feedbackExisted := s.feedbackEmailLastReuestAt.LoadOrStore(feedback.ReplyToEmail.String, now.Unix())
if feedbackExisted && lastFeedback.(int64) >= fiveMinutesAgo {
deleteUploaded(dbPath)
return status.Error(codes.ResourceExhausted, fmt.Sprintf("You have already send a feedback recently. Please wait %d seconds", lastFeedback.(int64)-fiveMinutesAgo))
}
}
Expand All @@ -92,11 +90,11 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err
}
return tx.Create(feedback).Error
}); err != nil {
deleteUploaded(dbPath)
if errors.Is(err, gorm.ErrDuplicatedKey) {
return status.Error(codes.AlreadyExists, "Feedback already exists")
}
log.WithError(err).Error("Error creating feedback")
deleteUploaded(feedback.EmailId)
return status.Error(codes.Internal, "Error creating feedback")
}

Expand All @@ -109,78 +107,89 @@ func (s *CampusServer) CreateFeedback(stream pb.Campus_CreateFeedbackServer) err

// deleteUploaded deletes all uploaded images from the filesystem
func deleteUploaded(dbPath string) {
if err := os.RemoveAll(cron.StorageDir + dbPath); err != nil {
if err := os.RemoveAll(dbPath); err != nil {
log.WithError(err).WithField("path", dbPath).Error("Error deleting uploaded images from filesystem")
}
}

func handleImageUpload(content []byte, imageCounter int, dbPath string) *string {
filename, realFilePath := inferFileName(mimetype.Detect(content), dbPath, imageCounter)
func handleImageUpload(content []byte, imageCounter int, dir string) *string {
filename := inferFileName(mimetype.Detect(content), imageCounter)
if filename == nil {
return nil // the filetype is not accepted by us
}
targetFilePath := path.Join(dir, *filename)

if err := os.MkdirAll(path.Dir(*realFilePath), 0755); err != nil {
log.WithError(err).WithField("dbPath", dbPath).Error("Error creating directory for feedback")
if err := os.MkdirAll(dir, 0755); err != nil {
log.WithError(err).WithField("dir", dir).Error("Error creating directory for feedback")
return nil
}
out, err := os.Create(*realFilePath)
targetFile, err := os.Create(targetFilePath)
if err != nil {
log.WithError(err).WithField("path", dbPath).Error("Error creating file for feedback")
log.WithError(err).WithField("path", targetFilePath).Error("Error creating file for feedback")
return nil
}
defer func(out *os.File) {
err := out.Close()
defer func(targetFile *os.File) {
err := targetFile.Close()
if err != nil {
log.WithError(err).WithField("path", dbPath).Error("Error while closing file")
log.WithError(err).WithField("path", dir).Error("Error while closing file")
}
}(out)
if _, err := io.Copy(out, bytes.NewReader(content)); err != nil {
log.WithError(err).WithField("path", dbPath).Error("Error while writing file")
if err := os.Remove(*realFilePath); err != nil {
log.WithError(err).WithField("path", dbPath).Warn("Could not clean up file")
}(targetFile)
if _, err := io.Copy(targetFile, bytes.NewReader(content)); err != nil {
log.WithError(err).WithField("path", targetFilePath).Error("Error while writing file")
if err := os.Remove(targetFilePath); err != nil {
log.WithError(err).WithField("path", targetFilePath).Warn("Could not clean up file")
}
return nil
}
return filename
}

func inferFileName(mime *mimetype.MIME, dbPath string, counter int) (*string, *string) {
func inferFileName(mime *mimetype.MIME, counter int) *string {
allowedExt := []string{".jpeg", ".jpg", ".png", ".webp", ".md", ".txt", ".pdf"}
if !slices.Contains(allowedExt, mime.Extension()) {
return nil, nil
return nil
}

filename := fmt.Sprintf("%d%s", counter, mime.Extension())
realFilePath := path.Join(cron.StorageDir, dbPath, filename)
return &filename, &realFilePath
return &filename
}

func mergeFeedback(feedback *model.Feedback, req *pb.CreateFeedbackRequest) {
if req.Recipient.Enum() != nil {
feedback.Recipient = receiverFromTopic(req.Recipient)
}
sanitiseString(&req.OsVersion)
if req.OsVersion != "" {
feedback.OsVersion = null.StringFrom(req.OsVersion)
}
sanitiseString(&req.AppVersion)
if req.AppVersion != "" {
feedback.AppVersion = null.StringFrom(req.AppVersion)
}
if req.Location != nil && req.Location.Longitude != 0 && req.Location.Latitude != 0 {
feedback.Longitude = null.FloatFrom(req.Location.Longitude)
feedback.Latitude = null.FloatFrom(req.Location.Latitude)
}
sanitiseString(&req.Message)
if req.Message != "" {
feedback.Feedback = req.Message
}
sanitiseString(&req.FromEmail)
if req.FromEmail != "" {
feedback.ReplyToEmail = null.StringFrom(req.FromEmail)
}
sanitiseString(&req.FromName)
if req.FromName != "" {
feedback.ReplyToName = null.StringFrom(req.FromEmail)
}
}

func sanitiseString(s *string) {
*s = strings.TrimSpace(*s)
*s = strings.ReplaceAll(*s, " ", " ")
*s = strings.ToValidUTF8(*s, "?")
}

func receiverFromTopic(topic pb.CreateFeedbackRequest_Recipient) string {
switch topic {
case pb.CreateFeedbackRequest_TUM_DEV:
Expand Down
Loading

0 comments on commit bda8b29

Please sign in to comment.