add ui

.gitignore

@@ -6,10 +6,13 @@
 /pkg/
 /spec/reports/
 /tmp/
+/tmp/
+web/node_modules
 
 *.gem
 
 vendor/bundle
 
 .rspec_status
-.DS_Store
+.DS_Store
+.env

Gemfile

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "pry-byebug", "~> 3.10"
+
 gem "rake", "~> 13.0"
 gem "rspec"
 gem "rubocop", "~> 1.55"
@@ -20,3 +21,10 @@ gem 'foreman', '~> 0.87.2'
 # group :test do
 #   gem 'database_cleaner', '~> 2.0', '>= 2.0.2'
 # end
+
+# web
+gem 'rackup', '~> 2.1'
+gem 'sinatra', '~> 4.0'
+gem 'sinatra-contrib'
+gem 'puma', '~> 6.4', '>= 6.4.2'
+gem 'kaminari', '~> 1.2', '>= 1.2.2'

Gemfile.lock

@@ -4,6 +4,7 @@ PATH
     outboxer (0.1.11)
       activerecord (~> 7.0)
       kaminari (~> 1.2)
+      rack-flash3 (~> 1.0, >= 1.0.5)
 
 GEM
   remote: https://rubygems.org/
@@ -47,11 +48,13 @@ GEM
     database_cleaner-core (2.0.1)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    dotenv (3.1.2)
     drb (2.2.0)
       ruby2_keywords
     erubi (1.12.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
+    ffi (1.16.3)
     foreman (0.87.2)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
@@ -69,32 +72,51 @@ GEM
       kaminari-core (= 1.2.2)
     kaminari-core (1.2.2)
     language_server-protocol (3.17.0.3)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     method_source (1.0.0)
     minitest (5.20.0)
+    multi_json (1.15.0)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
     mutex_m (0.2.0)
     mysql2 (0.5.6)
-    nokogiri (1.16.2-arm64-darwin)
+    nio4r (2.7.0)
+    nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.2-x86_64-darwin)
+    nokogiri (1.16.0-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.2-x86_64-linux)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
     parallel (1.23.0)
     parser (3.2.2.3)
       ast (~> 2.4.1)
       racc
-    pg (1.5.6)
+    pg (1.5.3)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
+    puma (6.4.2)
+      nio4r (~> 2.0)
     racc (1.7.1)
-    rack (3.0.8)
+    rack (3.0.10)
+    rack-flash3 (1.0.5)
+      rack
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
     rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
@@ -104,9 +126,14 @@ GEM
       nokogiri (~> 1.14)
     rainbow (3.1.1)
     rake (13.0.6)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
     redis-client (0.19.1)
       connection_pool
     regexp_parser (2.8.1)
+    rerun (0.14.0)
+      listen (~> 3.0)
     rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -147,10 +174,24 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    sinatra-contrib (4.0.0)
+      multi_json (>= 0.0.2)
+      mustermann (~> 3.0)
+      rack-protection (= 4.0.0)
+      sinatra (= 4.0.0)
+      tilt (~> 2.0)
+    tilt (2.3.0)
     timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
+    webrick (1.8.1)
     yard (0.9.34)
 
 PLATFORMS
@@ -161,17 +202,24 @@ PLATFORMS
 DEPENDENCIES
   activerecord (~> 7.1, >= 7.1.2)
   database_cleaner (~> 2.0, >= 2.0.2)
+  dotenv (>= 3.1.2)
   factory_bot (~> 6.4, >= 6.4.6)
   foreman (~> 0.87.2)
+  kaminari (~> 1.2, >= 1.2.2)
   mysql2 (~> 0.5.6)
   outboxer!
   pg (~> 1.5)
   pry-byebug (~> 3.10)
+  puma (~> 6.4, >= 6.4.2)
+  rackup (~> 2.1)
   rake (~> 13.0)
+  rerun (~> 0.14.0)
   rspec
   rubocop (~> 1.55)
   sidekiq (~> 7.2)
   simplecov (~> 0.22.0)
+  sinatra (~> 4.0)
+  sinatra-contrib
   yard (~> 0.9.34)
 
 BUNDLED WITH

Procfile

@@ -1,3 +1,4 @@
-sidekiq: bundle exec sidekiq -r ./config/sidekiq.rb
-sidekiq_ui: rackup config.ru
+# outboxer_message_backlogger: bundle exec ruby script/message/backlogger.rb
+sidekiq: bundle exec sidekiq -C config/sidekiq.yml -r ./config/sidekiq.rb
 outboxer_message_publisher: bin/outboxer_message_publisher
+outboxer_app: bundle exec rerun "rackup -p 3000"

README.md

@@ -35,7 +35,7 @@ bin/rails g outboxer:schema
 bin/rake db:migrate
 ```
 
-###  3. backlog message when event created
+###  3. after event created, backlog message
 
 ```ruby
 class Event < ActiveRecord::Base
@@ -49,7 +49,7 @@ class Event < ActiveRecord::Base
 end
 ```
 
-### 4. define event created job
+### 4. add event created job
 
 ```ruby
 class EventCreatedJob
@@ -69,7 +69,7 @@ end
 bin/rails g outboxer:message_publisher
 ```
 
-### 6. update publish block to queue event created job
+### 6. update publish block to perform event created job async
 
 ```ruby
 Outboxer::Publisher.publish do |message|
@@ -86,6 +86,46 @@ end
 bin/outboxer_message_publisher
 ```
 
+### 7. manage messages
+
+manage backlogged, queued, publishing and failed messages with the web ui
+
+<img width="1257" alt="Screenshot 2024-05-20 at 8 47 57 pm" src="https://github.com/fast-programmer/outboxer/assets/394074/0446bc7e-9d5f-4fe1-b210-ff394bdacdd6">
+
+#### rails
+
+##### config/routes.rb
+
+```ruby
+require 'outboxer/web'
+
+Rails.application.routes.draw do
+  mount Outboxer::Web, at: '/outboxer'
+end
+```
+
+#### rack
+
+##### config.ru
+
+```ruby
+require 'outboxer/web'
+
+map '/outboxer' do
+  run Outboxer::Web
+end
+```
+
+### 8. monitor message publisher
+
+understanding how much memory and cpu is required by the message publisher
+
+<img width="310" alt="Screenshot 2024-05-20 at 10 41 57 pm" src="https://github.com/fast-programmer/outboxer/assets/394074/1222ad47-15e3-44d1-bb45-6abc6b3e4325">
+
+```bash
+run bin/outboxer_message_publishermon
+```
+
 ## Motivation
 
 Outboxer was created to help Rails teams migrate to eventually consistent event driven architecture quickly, using existing tools and infrastructure.

app/jobs/event/created_job.rb

@@ -3,11 +3,7 @@ class CreatedJob
     include Sidekiq::Job
 
     def perform(args)
-      Sidekiq.logger.info "Event::CreatedJob#perform args: #{args.inspect} started"
-
-      sleep 1.5
-
-      Sidekiq.logger.info "Event::CreatedJob#perform args: #{args.inspect} finished"
+      Sidekiq.logger.info "Event::CreatedJob#perform args: #{args.inspect}"
     end
   end
 end

bin/outboxer_message_publisher

@@ -18,9 +18,13 @@ logger = Outboxer::Logger.new($stdout, level: Logger::INFO)
 db_config = Outboxer::Database.config(environment: environment)
 Outboxer::Database.connect(config: db_config, logger: logger)
 
-Signal.trap('INT') { Outboxer::Message.stop_publishing }
+['INT', 'TERM'].each do |signal|
+  Signal.trap(signal) do
+    Outboxer::Message.stop_publishing
+  end
+end
 
-Outboxer::Message.publish(threads: 100, queue: 100, poll: 1, logger: logger) do |message|
+Outboxer::Message.publish(threads: 5, queue: 1000, poll: 1, logger: logger) do |message|
   case message[:messageable_type]
   when 'Event'
     Event::CreatedJob.perform_async({ 'id' => message[:messageable_id] })

bin/outboxer_message_publishermon

@@ -55,7 +55,7 @@ loop do
 
   pids = outboxer_message_publisher_pids.sort
 
-  puts "Outboxer Report"
+  puts "Outboxer Message Publisher Monitor"
   puts Time.now.utc
   puts "\n---- Overview ----"
   puts format("%-15s: %-10s", "Backlogged", total_backlogged)

config.ru

@@ -1,8 +1,19 @@
+require 'bundler/setup'
+
+require 'dotenv/load'
+
+require 'securerandom'
+require 'sinatra'
 require 'sidekiq'
 require 'sidekiq/web'
+require 'outboxer/web'
 
-require 'securerandom'
+use Rack::Session::Cookie, secret: ENV['SESSION_SECRET'], same_site: true, max_age: 86400
 
-use Rack::Session::Cookie, secret: SecureRandom.hex(32), same_site: true, max_age: 86400
+map '/outboxer' do
+  run Outboxer::Web
+end
 
-run Sidekiq::Web
+map '/sidekiq' do
+  run Sidekiq::Web
+end

config/database.yml

@@ -6,7 +6,7 @@ development:
   username: outboxer_developer
   password: outboxer_password
   database: outboxer_development
-  pool: <%= ENV.fetch('DATABASE_POOL') { 5 } %>
+  pool: <%= ENV.fetch('DATABASE_POOL') { 26 } %>
 
 test:
   adapter: <%= ENV.fetch('DATABASE_ADAPTER') { 'postgresql' } %>
@@ -16,4 +16,4 @@ test:
   username: outboxer_developer
   password: outboxer_password
   database: outboxer_test
-  pool: <%= ENV.fetch('DATABASE_POOL') { 5 } %>
+  pool: <%= ENV.fetch('DATABASE_POOL') { 26 } %>

config/sidekiq.rb

@@ -3,8 +3,7 @@
 require 'active_record'
 require 'sidekiq'
 
-# require_relative '../app/models/event'
-require_relative '../app/jobs/event_created_job'
+require_relative '../app/jobs/event/created_job'
 
 environment = ENV['RAILS_ENV'] || 'development'
 
@@ -20,6 +19,6 @@
   config.redis = { url: redis_url }
 end
 
-# Sidekiq.configure_client do |config|
-#   config.redis = { url: redis_url }
-# end
+Sidekiq.configure_client do |config|
+  config.redis = { url: redis_url }
+end

config/sidekiq.yml

@@ -0,0 +1,5 @@
+:concurrency: 10
+:queues:
+  - default
+:redis:
+  :url: "redis://localhost:6379/0"