From 71d76e049e46071643b1fc50cabcca47b4949d65 Mon Sep 17 00:00:00 2001 From: Dean Lofts Date: Fri, 23 Aug 2024 18:51:17 +1000 Subject: [PATCH] sqlite backup --- Gemfile | 1 + Gemfile.lock | 18 ++++++ README.md | 40 +++++++++++- app/jobs/backup_database_job.rb | 38 ++++++++++++ config/initializers/aws_s3.rb | 12 ++++ config/sidekiq_scheduler.yml | 6 +- lib/tasks/restore.rake | 27 ++++++++ terraform/README.md | 18 ++++++ terraform/{ => droplet}/.terraform.lock.hcl | 0 terraform/{ => droplet}/main.tf | 0 terraform/{ => droplet}/outputs.tf | 0 terraform/{ => droplet}/provider.tf | 0 terraform/{ => droplet}/variables.tf | 0 terraform/spaces/.terraform.lock.hcl | 26 ++++++++ terraform/spaces/main.tf | 69 +++++++++++++++++++++ 15 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 app/jobs/backup_database_job.rb create mode 100644 config/initializers/aws_s3.rb create mode 100644 lib/tasks/restore.rake rename terraform/{ => droplet}/.terraform.lock.hcl (100%) rename terraform/{ => droplet}/main.tf (100%) rename terraform/{ => droplet}/outputs.tf (100%) rename terraform/{ => droplet}/provider.tf (100%) rename terraform/{ => droplet}/variables.tf (100%) create mode 100644 terraform/spaces/.terraform.lock.hcl create mode 100644 terraform/spaces/main.tf diff --git a/Gemfile b/Gemfile index 5a573f3..6f6930d 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ gem 'mini_magick' gem 'sidekiq' gem 'sidekiq-scheduler' gem 'vite_rails' +gem 'aws-sdk-s3' # Use Kredis to get higher-level data types in Redis [https://github.com/rails/kredis] # gem "kredis" diff --git a/Gemfile.lock b/Gemfile.lock index fe68ab9..e14daad 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,6 +77,22 @@ GEM tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.968.0) + aws-sdk-core (3.201.5) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.88.0) + aws-sdk-core (~> 3, >= 3.201.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.159.0) + aws-sdk-core (~> 3, >= 3.201.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.9.1) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) bcrypt (3.1.20) bigdecimal (3.1.8) @@ -150,6 +166,7 @@ GEM jbuilder (2.12.0) actionview (>= 5.0.0) activesupport (>= 5.0.0) + jmespath (1.6.2) logger (1.6.0) loofah (2.22.0) crass (~> 1.0.2) @@ -358,6 +375,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + aws-sdk-s3 bootsnap capybara chartkick diff --git a/README.md b/README.md index f2024d1..3ba2e0c 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,47 @@ Note: The Dockerfile uses a multi-stage build process to create a lean productio The project uses SQLite by default. For production, consider using PostgreSQL or MySQL. +### Backup and Restore Process + +Linkarooie includes an automated backup system to ensure that your SQLite database is securely stored and easily recoverable. This process is managed using a combination of scheduled jobs and DigitalOcean Spaces for storage. + +#### Automated Backups + +The `BackupDatabaseJob` is scheduled to run daily at 2 AM, ensuring that your SQLite database is backed up regularly. The backup process involves the following steps: + +1. **Database Dump**: The job creates a dump of the current SQLite database, storing it in the `db/backups` directory with a timestamp and environment identifier. +2. **Upload to DigitalOcean Spaces**: The backup file is then uploaded to a DigitalOcean Spaces bucket, where it is securely stored with versioning enabled. This ensures that previous versions of the backup are retained for a short period, allowing you to restore from a specific point in time if needed. +3. **Cleanup**: Optionally, the local backup file is deleted after it has been successfully uploaded to DigitalOcean Spaces. + +#### Restoring from a Backup + +In the event that you need to restore your database from a backup, you can use the provided Rake task. This task allows you to specify the backup file you want to restore from and automatically loads it into the SQLite database. + +**Restoration Steps:** + +1. **Run the Restore Task**: Use the following command, specifying the path to your backup file: + + ```bash + rake db:restore BACKUP_FILE=path/to/your_backup_file.sql + ``` + +2. **Process Overview**: + + * The task will first drop all existing tables in the database to ensure a clean restoration. + * It will then load the specified backup file into the database. + * Upon completion, your database will be restored to the state it was in when the backup was created. + +3. **Error Handling**: If the backup file is not provided or if any errors occur during the restoration process, the task will output helpful messages to guide you in resolving the issue. + +#### Important Notes + +* **Environment-Specific Backups**: Backups are created separately for each environment (development, production, test), and the backup files are named accordingly. +* **DigitalOcean Spaces Configuration**: Ensure that your DigitalOcean API credentials and bucket details are correctly configured in your environment variables for the backup and restore processes to function properly. +* **Testing Restores**: Regularly test the restore process in a development environment to ensure that your backups are reliable and that the restore process works as expected. + ### Geolocation -For now this isn't optional but I intend to make it to. [API key required](https://ipapi.com) but it is free. +Currently, geolocation functionality is mandatory, but I plan to make it optional in future updates. To enable geolocation, you will need an [API key from ipapi](https://ipapi.com), which is free to obtain. ## Customization diff --git a/app/jobs/backup_database_job.rb b/app/jobs/backup_database_job.rb new file mode 100644 index 0000000..9d66a5d --- /dev/null +++ b/app/jobs/backup_database_job.rb @@ -0,0 +1,38 @@ +class BackupDatabaseJob < ApplicationJob + queue_as :default + + def perform + environment = Rails.env + backup_file = "db/backups/#{environment}_backup_#{Time.now.strftime('%Y%m%d%H%M%S')}.sqlite3" + + begin + # Ensure the backup directory exists + FileUtils.mkdir_p("db/backups") + + # Dump the SQLite database for the current environment + database_path = Rails.configuration.database_configuration[environment]["database"] + `sqlite3 #{database_path} .dump > #{backup_file}` + + # Upload to DigitalOcean Spaces + upload_to_spaces(backup_file) + + # Optionally, delete the local backup file after upload + File.delete(backup_file) if File.exist?(backup_file) + + Rails.logger.info "BackupDatabaseJob: Backup created and uploaded successfully: #{backup_file}" + rescue => e + Rails.logger.error "BackupDatabaseJob: Failed to create or upload backup: #{e.message}" + raise + end + end + + private + + def upload_to_spaces(file_path) + bucket_name = ENV['SPACES_BUCKET_NAME'] || 'sqlite-backup-bucket' + file_name = File.basename(file_path) + + obj = S3_CLIENT.bucket(bucket_name).object("backups/#{file_name}") + obj.upload_file(file_path) + end +end diff --git a/config/initializers/aws_s3.rb b/config/initializers/aws_s3.rb new file mode 100644 index 0000000..1f018cb --- /dev/null +++ b/config/initializers/aws_s3.rb @@ -0,0 +1,12 @@ +require 'aws-sdk-s3' + +Aws.config.update({ + region: ENV['SPACES_REGION'] || 'syd1', + credentials: Aws::Credentials.new( + ENV['SPACES_ACCESS_KEY_ID'], + ENV['SPACES_SECRET_ACCESS_KEY'] + ), + endpoint: "https://#{ENV['SPACES_REGION'] || 'syd1'}.digitaloceanspaces.com" +}) + +S3_CLIENT = Aws::S3::Resource.new diff --git a/config/sidekiq_scheduler.yml b/config/sidekiq_scheduler.yml index 1a874c5..3ac8575 100644 --- a/config/sidekiq_scheduler.yml +++ b/config/sidekiq_scheduler.yml @@ -1,3 +1,7 @@ aggregate_metrics: cron: '0 1 * * *' # Run at 1 AM every day - class: AggregateMetricsJob \ No newline at end of file + class: AggregateMetricsJob + +backup_database: +cron: '0 2 * * *' # Runs daily at 2 AM +class: "BackupDatabaseJob" diff --git a/lib/tasks/restore.rake b/lib/tasks/restore.rake new file mode 100644 index 0000000..5be4e84 --- /dev/null +++ b/lib/tasks/restore.rake @@ -0,0 +1,27 @@ +namespace :db do + desc "Restore the SQLite database from a SQL dump" + task restore: :environment do + backup_file = ENV['BACKUP_FILE'] + + unless backup_file + puts "ERROR: You must provide the path to the backup file." + puts "Usage: rake db:restore BACKUP_FILE=path/to/your_backup_file.sql" + exit 1 + end + + begin + puts "Restoring database from #{backup_file}..." + + # Drop the current database tables + ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS #{ActiveRecord::Base.connection.tables.join(', ')}") + + # Load the backup SQL file + system("sqlite3 #{Rails.configuration.database_configuration[Rails.env]['database']} < #{backup_file}") + + puts "Database restored successfully." + rescue => e + puts "ERROR: Failed to restore the database: #{e.message}" + exit 1 + end + end +end diff --git a/terraform/README.md b/terraform/README.md index 64b9c09..bd417c9 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -9,10 +9,28 @@ terraform apply -var="do_token=YOUR_DIGITALOCEAN_TOKEN" ssh root@ ``` +- Terraform for Spaces. + +> Note: `export DO_TOKEN=` and also `SPACES_ACCESS_KEY_ID` and `SPACES_SECRET_ACCESS_KEY` before running this. + +```bash +terraform apply -var="do_token=$DO_TOKEN" \ + -var="spaces_access_id=$SPACES_ACCESS_KEY_ID" \ + -var="spaces_secret_key=$SPACES_SECRET_ACCESS_KEY" +``` + * Create the instance with Terraform * Collect the droplet IP address * Check for access `ssh root@` +- Or for spaces. + +```bash +spaces_bucket_domain_name = "sqlite-backup-bucket.syd1.digitaloceanspaces.com" +spaces_bucket_name = "sqlite-backup-bucket" +spaces_bucket_region = "syd1" +``` + ## GitHub Secrets Ensure you have the following secrets set in your GitHub repository: diff --git a/terraform/.terraform.lock.hcl b/terraform/droplet/.terraform.lock.hcl similarity index 100% rename from terraform/.terraform.lock.hcl rename to terraform/droplet/.terraform.lock.hcl diff --git a/terraform/main.tf b/terraform/droplet/main.tf similarity index 100% rename from terraform/main.tf rename to terraform/droplet/main.tf diff --git a/terraform/outputs.tf b/terraform/droplet/outputs.tf similarity index 100% rename from terraform/outputs.tf rename to terraform/droplet/outputs.tf diff --git a/terraform/provider.tf b/terraform/droplet/provider.tf similarity index 100% rename from terraform/provider.tf rename to terraform/droplet/provider.tf diff --git a/terraform/variables.tf b/terraform/droplet/variables.tf similarity index 100% rename from terraform/variables.tf rename to terraform/droplet/variables.tf diff --git a/terraform/spaces/.terraform.lock.hcl b/terraform/spaces/.terraform.lock.hcl new file mode 100644 index 0000000..d3c51f6 --- /dev/null +++ b/terraform/spaces/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/digitalocean/digitalocean" { + version = "2.40.0" + constraints = "~> 2.0" + hashes = [ + "h1:Y7VkuuqOBgv+1jgL/4Hi247K2BskXwXtR/Uk9ssK/e4=", + "zh:00235830abae70642ebefc4d9c00e5eb978e28b74abc6b34f16b078f242aa217", + "zh:09d77785f768bd568f85a121d3d79316083befe903ce4ccd5567689a23236fb0", + "zh:0c9c4e19b411702d316a6bd044903e2ec506a69d38495ed32cc31e3f3f26acae", + "zh:12b34c88faad5b6149e9a3ad1396680588e1bae263b20d6b19835460f111c190", + "zh:15f041fc57ea46673a828919efe2ef3f05f7c4b863b7d7881336b93e92bd1159", + "zh:45e01972de2fab1687a09ea8fb3e4519be11c93ef93a63f28665630850858a20", + "zh:4e18bf5c1d2ec1ec6b6a9f4b58045309006f510edf770168fc18e273e6a09289", + "zh:575528b7e36e3489d2309e0c6cb9bd9952595cac5459b914f2d2827de1a1e4fc", + "zh:67462192212f810875d556462c79f574a8f5713b7a869ba4fce25953bfcf2dd2", + "zh:7024637b31e8276b653265fdf3f479220182edde4b300b034562b4c287faefa5", + "zh:a7904721b2680be8330dde98dd826be15c67eb274da7876f042cbcd6592ac970", + "zh:b225d4b67037a19392b0ab00d1f5fc9e729db4dfc32d18d4b36225693270ef52", + "zh:bd1e8768819d6113b2ec16f939196a1f2ae6d2803824fde463a20d06e071b212", + "zh:c5da40dc0749548ee2e1943776fb41b952c994e50bbc404251df20a81f730242", + "zh:dabc3387392aaba297739e1e97fadf059258fc3efb4dff2f499dbc407b6e088d", + "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2", + ] +} diff --git a/terraform/spaces/main.tf b/terraform/spaces/main.tf new file mode 100644 index 0000000..a49d713 --- /dev/null +++ b/terraform/spaces/main.tf @@ -0,0 +1,69 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +provider "digitalocean" { + token = var.do_token + spaces_access_id = var.spaces_access_id + spaces_secret_key = var.spaces_secret_key +} + +variable "do_token" { + description = "DigitalOcean API token" +} + +variable "spaces_access_id" { + description = "Access Key ID for DigitalOcean Spaces" +} + +variable "spaces_secret_key" { + description = "Secret Access Key for DigitalOcean Spaces" +} + +variable "region" { + description = "DigitalOcean region" + default = "syd1" +} + +resource "digitalocean_spaces_bucket" "sqlite_backup" { + name = "sqlite-backup-bucket" + region = var.region + + versioning { + enabled = true + } + + lifecycle_rule { + id = "cleanup-old-backups" + enabled = true + prefix = "backup/" + expiration { + days = 30 + } + noncurrent_version_expiration { + days = 7 + } + } + + force_destroy = false +} + +output "spaces_bucket_name" { + value = digitalocean_spaces_bucket.sqlite_backup.name + description = "The name of the DigitalOcean Space bucket created." +} + +output "spaces_bucket_region" { + value = digitalocean_spaces_bucket.sqlite_backup.region + description = "The region of the DigitalOcean Space bucket created." +} + +output "spaces_bucket_domain_name" { + value = digitalocean_spaces_bucket.sqlite_backup.bucket_domain_name + description = "The full domain name of the DigitalOcean Space bucket." +}