Generate sitemaps with Controller and Actions instead of sitemap_gene…

…rator (#268)

Related to #265

Implement sitemap generation with a primary index referencing a per-city index, and the per-city index listing out a sitemap per-city-per-day containing all service requests for that day.

Gemfile

@@ -39,7 +39,6 @@ gem 'rails', '~> 7.1'
 gem 'sass-rails'
 gem 'scenic'
 gem 'sentry-raven'
-gem 'sitemap_generator'
 gem 'slim'
 gem 'slim-rails'
 gem 'terser'

Gemfile.lock

@@ -443,8 +443,6 @@ GEM
       websocket (~> 1.0)
     sentry-raven (2.13.0)
       faraday (>= 0.7.6, < 1.0)
-    sitemap_generator (6.3.0)
-      builder (~> 3.0)
     slim (5.2.1)
       temple (~> 0.10.0)
       tilt (>= 2.1.0)
@@ -552,7 +550,6 @@ DEPENDENCIES
   scenic
   selenium-webdriver
   sentry-raven
-  sitemap_generator
   slim
   slim-rails
   slim_lint

app/controllers/sitemaps_controller.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class SitemapsController < ApplicationController
+  def index
+    @cities = City.all
+
+    respond_to do |format|
+      format.xml
+    end
+  end
+
+  def city_index
+    @city = City.find_by!(slug: params[:slug])
+
+    oldest_service_request = @city.service_requests.order(:created_at, :id).limit(1).pick(:created_at)&.to_date || Date.current
+    @dates = oldest_service_request..Date.current
+
+    respond_to do |format|
+      format.xml
+    end
+  end
+
+  def city_day
+    @city = City.find_by!(slug: params[:slug])
+    @date = Date.parse(params[:date])
+    @service_requests = @city.service_requests.where(requested_datetime: @date.all_day)
+
+    respond_to do |format|
+      format.xml
+    end
+  end
+
+  def static
+    @links = [
+      root_url,
+      about_url,
+    ] + City.all.map { |city| city_url(city) }
+
+    respond_to do |format|
+      format.xml
+    end
+  end
+
+  private
+
+  def format_lastmod(updated)
+    updated.utc.strftime("%Y-%m-%d")
+  end
+  helper_method :format_lastmod
+end

app/models/service_request.rb

@@ -59,6 +59,10 @@ def raw_data=(json)
                             }
   end
 
+  def raw_data
+    super || {}
+  end
+
   def slug
     description = raw_data['description'].presence || ''
     slug = description.gsub(/(-|_)/, ' ').squish.encode.to_slug.normalize.to_s.downcase

app/views/sitemaps/city_day.xml.builder

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+xml.instruct! :xml, version: "1.0", encoding: "UTF-8"
+xml.urlset xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" do
+  @service_requests.each do |service_request|
+    xml.url do
+      xml.loc service_request_url(service_request, format: :xml)
+      xml.lastmod format_lastmod(service_request.updated_at)
+    end
+  end
+end

app/views/sitemaps/city_index.xml.builder

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+xml.instruct! :xml, version: "1.0", encoding: "UTF-8"
+xml.sitemapindex xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" do
+  @dates.each do |date|
+    xml.sitemap do
+      xml.loc city_day_sitemap_url(@city.slug, date.strftime("%Y-%m-%d"), format: :xml)
+      xml.lastmod date.strftime("%Y-%m-%dT%H:%M:%S%:z")
+    end
+  end
+end

app/views/sitemaps/index.xml.builder

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+xml.instruct! :xml, version: "1.0", encoding: "UTF-8"
+xml.sitemapindex xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" do
+  xml.sitemap do
+    xml.loc static_sitemap_url(format: :xml)
+  end
+
+  @cities.each do |city|
+    xml.sitemap do
+      xml.loc city_sitemap_index_url(city.slug, format: :xml)
+    end
+  end
+end

app/views/sitemaps/static.xml.builder

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+xml.instruct! :xml, version: "1.0", encoding: "UTF-8"
+xml.urlset xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" do
+  @links.each do |link|
+    xml.url do
+      xml.loc link
+    end
+  end
+end

config/initializers/good_job.rb

@@ -14,11 +14,6 @@
         class: 'RefreshJob',
         description: "Update open311 statuses",
       },
-      sitemap: {
-        cron: '0 7 * * *',
-        class: 'SitemapJob',
-        description: "Update Sitemap",
-      },
     }
 
     GoodJob::Engine.middleware.use(Rack::Auth::Basic) do |provided_username, provided_password|

config/routes.rb

@@ -16,7 +16,10 @@
 
   mount GoodJob::Engine => 'good_job'
 
-  get '/sitemap.xml.gz', to: redirect("https://#{Rails.application.secrets.s3_bucket_name}.s3.amazonaws.com/sitemaps/sitemap.xml.gz")
+  get '/sitemap.xml', to: 'sitemaps#index', defaults: { format: 'xml' }, as: :sitemap
+  get '/sitemaps/static.xml', to: 'sitemaps#static', defaults: { format: 'xml' }, as: :static_sitemap
+  get '/sitemaps/:slug.xml', to: 'sitemaps#city_index', defaults: { format: 'xml' }, as: :city_sitemap_index
+  get '/sitemaps/:slug/:date.xml', to: 'sitemaps#city_day', defaults: { format: 'xml' }, as: :city_day_sitemap
 
   resolve "ServiceRequest" do |service_request, options|
     service_request.parameterize.merge(controller: 'cities/requests', action: 'show').merge(options)

config/secrets.yml

@@ -12,10 +12,6 @@
 
 defaults: &defaults
   default_host: <%= ENV.fetch('DEFAULT_HOST', 'localhost:3000') %>
-  upload_host: <%= ENV.fetch('UPLOAD_HOST', 'http://localhost:3000') %>
-  s3_bucket_name: <%= ENV['S3_BUCKET_NAME'] %>
-  aws_access_key_id: <%= ENV['AWS_ACCESS_KEY_ID'] %>
-  aws_secret_access_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
 
 development:
   <<: *defaults

db/migrate/20240730192533_add_order_index_to_service_requests.rb

@@ -0,0 +1,7 @@
+class AddOrderIndexToServiceRequests < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :service_requests, [:city_id, :created_at, :id], algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_07_30_143512) do
+ActiveRecord::Schema[7.1].define(version: 2024_07_30_192533) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
   enable_extension "pgcrypto"
@@ -133,6 +133,7 @@
     t.datetime "updated_at", null: false
     t.bigint "city_id"
     t.geography "geometry", limit: {:srid=>4326, :type=>"geometry", :geographic=>true}
+    t.index ["city_id", "created_at", "id"], name: "index_service_requests_on_city_id_and_created_at_and_id"
     t.index ["city_id", "requested_datetime"], name: "index_service_requests_on_city_id_and_requested_datetime", order: { requested_datetime: "DESC NULLS LAST" }
     t.index ["city_id", "service_request_id"], name: "index_service_requests_on_city_id_and_service_request_id", unique: true
     t.index ["city_id"], name: "index_service_requests_on_city_id"

public/robots.txt

@@ -8,4 +8,4 @@
 #
 
 User-agent: *
-Sitemap: https://status.open311.org/sitemap.xml.gz
+Sitemap: https://status.open311.org/sitemap.xml

spec/system/sitemaps_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe 'Sitemaps' do
+  let(:city) { City.first }
+  let(:service_request) { create(:service_request, city:) }
+
+  before do
+    City.load!
+  end
+
+  it 'visits the primary sitemap index' do
+    visit '/sitemap.xml'
+    expect(page).to have_content static_sitemap_url(format: :xml)
+    expect(page).to have_content city_sitemap_index_url(City.first.slug, format: :xml)
+  end
+
+  it 'visits the per-city index' do
+    visit "/sitemaps/#{city.slug}.xml"
+    expect(page).to have_content city_day_sitemap_url(city.slug, date: Date.current, format: :xml)
+  end
+
+  it 'visits the per-city-per-day sitemap' do
+    visit "/sitemaps/#{city.slug}/#{service_request.requested_datetime.to_date}.xml"
+    expect(page).to have_content(service_request_url(service_request))
+  end
+
+  it 'visits the static sitemap' do
+    visit '/sitemaps/static.xml'
+    expect(page).to have_content(root_url)
+  end
+end