From 74145f16ade4aefff767ec0748075d9b7765b3e0 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Fri, 21 Jun 2019 15:38:35 -0500 Subject: [PATCH 01/41] Setup a docker development environment --- .dockerignore | 12 +++++++++ Dockerfile | 28 ++++++++++++++++++++ README.mkdn | 41 +++++++++++++++++++++++++++++ Rakefile | 7 +++-- docker-compose.yml | 41 +++++++++++++++++++++++++++++ docker/startup.sh | 27 +++++++++++++++++++ spec/config/shards.yml | 9 ++++--- spec/octopus/octopus_spec.rb | 2 +- spec/support/database_connection.rb | 9 ++++++- spec/support/database_models.rb | 2 +- 10 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 docker/startup.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e64ea9d9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +*.gem +*.sqlite3 +.bundle +.rvmrc +.ruby-version +Gemfile.lock +gemfiles/*.lock +pkg/* +*.rbc +tmp/* +.*.sw[a-z] +database.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9b6afc7e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM ruby:2.5 + +RUN apt-get update && apt-get install -y \ + mysql-client \ + postgresql + +WORKDIR /usr/src/app + +# Pull in a full profile for gem/bundler +SHELL ["/bin/bash", "-l", "-c"] + +RUN gem install --no-document bundler -v 1.16.6 + +# Copy only what's needed for bundler +COPY Gemfile ar-octopus.gemspec /usr/src/app/ +COPY lib/octopus/version.rb /usr/src/app/lib/octopus/version.rb + +RUN bundle install + +# Uncomment if you want to copy the octopus repo +# into the Docker image itself. docker-compose is +# set up to use a bind mount of your local directory +# from the host +#COPY . /usr/src/app + +# This just keeps the container running. Replace +# this with a rails server if you want +CMD ["/bin/sleep", "infinity"] diff --git a/README.mkdn b/README.mkdn index 99d57a65..fa132b37 100644 --- a/README.mkdn +++ b/README.mkdn @@ -229,6 +229,47 @@ bundle install cucumber ``` +### Using Docker Compose + +For a more consistent build/test/development environment, use [Docker Compose](https://docs.docker.com/compose/install/). + +```bash +[me@host octopus]$ docker-compose -f docker-compose.yml up +Creating network "octopus_default" with the default driver +Creating octopus_postgres_1 ... +Creating octopus_mysql_1 ... +Creating octopus_postgres_1 +. . . +octopus_1 | mysqld is alive +octopus_1 | MySQL is ready +octopus_1 | + bundle exec rake db:prepare +. . . +octopus_1 | + bundle exec rake spec +. . . +octopus_1 | Finished in 7 minutes 44 seconds (files took 0.71416 seconds to load) +octopus_1 | 359 examples, 0 failures, 2 pending + +(Ctrl+C to shut down the stack) +``` + +Your workstation's octopus clone is mounted inside the docker container, so +you may still use your favorite development tools. The `octopus_1` container +is available for re-running the test suite. To start a new shell in another +terminal, run: + +```bash +[me@host octopus]$ docker-compose exec octopus /bin/bash +root@f6ae68df6fc8:/usr/src/app# + # run bundle exec rake spec again +``` + +To completely destory the stack and start over: + +```bash +[me@host octopus]$ docker-compose -f docker-compose.yml down +[me@host octopus]$ docker-compose -f docker-compose.yml up +``` + If you are having issues running the octopus spec suite, verify your database users and passwords match those inside the config files and your permissions are correct. ## Contributors: diff --git a/Rakefile b/Rakefile index e3d5f728..0aa741ac 100644 --- a/Rakefile +++ b/Rakefile @@ -13,15 +13,17 @@ namespace :db do task :build_databases do pg_spec = { :adapter => 'postgresql', - :host => 'localhost', + :host => (ENV['POSTGRES_HOST'] || ''), :username => (ENV['POSTGRES_USER'] || 'postgres'), + :password => (ENV['POSTGRES_PASSWORD'] || ''), :encoding => 'utf8', } mysql_spec = { :adapter => 'mysql2', - :host => 'localhost', + :host => (ENV['MYSQL_HOST'] || ''), :username => (ENV['MYSQL_USER'] || 'root'), + :password => (ENV['MYSQL_PASSWORD'] || ''), :encoding => 'utf8', } @@ -64,6 +66,7 @@ namespace :db do # the model be a descendent of ActiveRecord::Base. class BlankModel < ActiveRecord::Base; end + puts "Preparing shard #{shard_symbol}" BlankModel.using(shard_symbol).connection.initialize_schema_migrations_table BlankModel.using(shard_symbol).connection.initialize_metadata_table if Octopus.atleast_rails50? diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b74c4a00 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +version: '3' + +services: + octopus: + build: . + # Your local clone binds to /usr/src/app + volumes: + - .:/usr/src/app + environment: + POSTGRES_HOST: postgres # image name below + POSTGRES_USER: postgres + POSTGRES_PASSWORD: testpassword # password below + MYSQL_HOST: mysql + MYSQL_USER: root + MYSQL_PASSWORD: testpassword + depends_on: + - mysql + - postgres + # Wait for databases to start up, run test suite + command: ./docker/startup.sh + mysql: + image: mysql:8 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: testpassword + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + interval: 10s + timeout: 20s + retries: 5 + postgres: + image: postgres:11 + restart: always + environment: + POSTGRES_PASSWORD: testpassword + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 diff --git a/docker/startup.sh b/docker/startup.sh new file mode 100755 index 00000000..de7810d1 --- /dev/null +++ b/docker/startup.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +# Wait for our two databases to startup +# (docker-compose v3 removes the health check stuff) +until PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -c '\q'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 10 +done + +>&2 echo "Postgres is ready" + +until mysqladmin -u$MYSQL_USER -h $MYSQL_HOST -p$MYSQL_PASSWORD ping; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 10 +done + +>&2 echo "MySQL is ready" + +set -x +bundle exec rake db:prepare +bundle exec rake appraisal:install +bundle exec rake spec +set +x +echo Octopus is ready for you. Run "docker exec octopus /bin/bash" +/bin/sleep infinity diff --git a/spec/config/shards.yml b/spec/config/shards.yml index c4bc2831..ca3c2a16 100644 --- a/spec/config/shards.yml +++ b/spec/config/shards.yml @@ -1,7 +1,8 @@ mysql: &mysql adapter: mysql2 username: <%= ENV['MYSQL_USER'] || 'root' %> - host: localhost + password: <%= ENV['MYSQL_PASSWORD'] || '' %> + host: <%= ENV['MYSQL_HOST'] || 'localhost' %> mysql_unavailable: &mysql_unavailable adapter: mysql2 @@ -18,10 +19,10 @@ octopus: &octopus postgresql_shard: adapter: postgresql username: <%= ENV['POSTGRES_USER'] || 'postgres' %> - password: + password: <%= ENV['POSTGRES_PASSWORD'] || '' %> database: octopus_shard_1 encoding: unicode - host: localhost + host: <%= ENV['POSTGRES_HOST'] || 'localhost' %> sqlite_shard: adapter: sqlite3 @@ -49,7 +50,7 @@ octopus: &octopus database: octopus_shard_4 <<: *mysql - protocol_shard: postgres://<%= ENV['POSTGRES_USER'] || 'postgres' %>@localhost:5432/octopus_shard_2 + protocol_shard: postgres://<%= ENV['POSTGRES_USER'] || 'postgres' %>:<%= ENV['POSTGRES_PASSWORD'] || '' %>@<%= ENV['POSTGRES_HOST'] || 'localhost' %>:5432/octopus_shard_2 octopus_with_default_migration_group: <<: *octopus diff --git a/spec/octopus/octopus_spec.rb b/spec/octopus/octopus_spec.rb index db2077bf..6c6d8cce 100644 --- a/spec/octopus/octopus_spec.rb +++ b/spec/octopus/octopus_spec.rb @@ -40,7 +40,7 @@ expect { User.using(:crazy_shard).create!(:name => 'Joaquim') }.to raise_error(RuntimeError) Octopus.setup do |config| - config.shards = { :crazy_shard => { :adapter => 'mysql2', :database => 'octopus_shard_5', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => '' } } + config.shards = { :crazy_shard => { :adapter => 'mysql2', :database => 'octopus_shard_5', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => "#{ENV['MYSQL_PASSWORD'] || ''}", :host => "#{ENV['MYSQL_HOST'] || 'localhost'}" } } end expect { User.using(:crazy_shard).create!(:name => 'Joaquim') }.not_to raise_error diff --git a/spec/support/database_connection.rb b/spec/support/database_connection.rb index 20b5f2cf..71c9cb90 100644 --- a/spec/support/database_connection.rb +++ b/spec/support/database_connection.rb @@ -1,4 +1,11 @@ require 'logger' -ActiveRecord::Base.establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_1', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => '') +ActiveRecord::Base.establish_connection( + :adapter => 'mysql2', + :database => 'octopus_shard_1', + :username => "#{ENV['MYSQL_USER'] || 'root'}", + :password => "#{ENV['MYSQL_PASSWORD'] || ''}", + :host => "#{ENV['MYSQL_HOST'] || 'localhost'}" +) + ActiveRecord::Base.logger = Logger.new(File.open('database.log', 'a')) diff --git a/spec/support/database_models.rb b/spec/support/database_models.rb index 787ecebc..5cdc59d7 100644 --- a/spec/support/database_models.rb +++ b/spec/support/database_models.rb @@ -28,7 +28,7 @@ class Cat < ActiveRecord::Base # This class sets its own connection class CustomConnection < ActiveRecord::Base self.table_name = 'custom' - octopus_establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_2', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => '') + octopus_establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_2', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => "#{ENV['MYSQL_PASSWORD'] || ''}", :host => "#{ENV['MYSQL_HOST'] || 'localhost'}") end # This items belongs to a client From 48090f09fe8766dc241bc426825fce2b052d01e9 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Tue, 25 Jun 2019 13:32:20 -0500 Subject: [PATCH 02/41] Start on setting up sample_app -- not working Running cucumber gives the error: cannot load such file -- test/unit (LoadError) --- Dockerfile | 2 +- docker/startup.sh | 5 +- gemfiles/rails42.gemfile | 2 +- lib/octopus/abstract_adapter.rb | 2 +- sample_app/Gemfile | 5 +- sample_app/Gemfile.lock | 159 +++++++++++++++++++------------- sample_app/config/database.yml | 18 ++-- sample_app/script/ci_build | 14 +++ 8 files changed, 126 insertions(+), 81 deletions(-) create mode 100755 sample_app/script/ci_build diff --git a/Dockerfile b/Dockerfile index 9b6afc7e..1e5e673e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN gem install --no-document bundler -v 1.16.6 COPY Gemfile ar-octopus.gemspec /usr/src/app/ COPY lib/octopus/version.rb /usr/src/app/lib/octopus/version.rb -RUN bundle install +RUN bundle install --path=.bundle # Uncomment if you want to copy the octopus repo # into the Docker image itself. docker-compose is diff --git a/docker/startup.sh b/docker/startup.sh index de7810d1..db7bb567 100755 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -19,9 +19,12 @@ done >&2 echo "MySQL is ready" set -x +bundle install bundle exec rake db:prepare bundle exec rake appraisal:install bundle exec rake spec +#Not working yet +#./sample_app/script/ci_build set +x -echo Octopus is ready for you. Run "docker exec octopus /bin/bash" +echo Octopus is ready for you. Run \"docker-compose exec octopus /bin/bash\" /bin/sleep infinity diff --git a/gemfiles/rails42.gemfile b/gemfiles/rails42.gemfile index 24e5e488..9be86420 100644 --- a/gemfiles/rails42.gemfile +++ b/gemfiles/rails42.gemfile @@ -3,5 +3,5 @@ source "https://rubygems.org" gem "activerecord", "~> 4.2.0" -gem "mysql2", "0.4.10" + gemspec path: "../" diff --git a/lib/octopus/abstract_adapter.rb b/lib/octopus/abstract_adapter.rb index f6ce7956..646f2272 100644 --- a/lib/octopus/abstract_adapter.rb +++ b/lib/octopus/abstract_adapter.rb @@ -19,7 +19,7 @@ def method_missing(meth, *args, &block) end def octopus_shard - @config[:octopus_shard] + @config && @config[:octopus_shard] end def initialize(*args) diff --git a/sample_app/Gemfile b/sample_app/Gemfile index 7f667363..49331897 100644 --- a/sample_app/Gemfile +++ b/sample_app/Gemfile @@ -5,10 +5,11 @@ gem 'rails', '3.2.13' # Bundle edge Rails instead: # gem 'rails', :git => 'git://github.com/rails/rails.git' -gem 'pg' -gem 'sqlite3' +gem 'pg', '~> 0.11' +gem 'sqlite3', '~> 1.3.5' gem 'ar-octopus', '0.6.0' gem 'pry' +gem 'json', '~> 1.8.6' group :test do gem 'capybara' diff --git a/sample_app/Gemfile.lock b/sample_app/Gemfile.lock index 54b6f2ba..5483f1eb 100644 --- a/sample_app/Gemfile.lock +++ b/sample_app/Gemfile.lock @@ -28,70 +28,88 @@ GEM activesupport (3.2.13) i18n (= 0.6.1) multi_json (~> 1.0) - addressable (2.3.5) + addressable (2.6.0) + public_suffix (>= 2.0.2, < 4.0) ar-octopus (0.6.0) activerecord (>= 3.0.0, < 4.0) activesupport (>= 3.0.0, < 4.0) - arel (3.0.2) - aruba (0.5.3) - childprocess (>= 0.3.6) - cucumber (>= 1.1.1) - rspec-expectations (>= 2.7.0) + arel (3.0.3) + aruba (0.14.10) + childprocess (>= 0.6.3, < 1.1.0) + contracts (~> 0.9) + cucumber (>= 1.3.19) + ffi (~> 1.9) + rspec-expectations (>= 2.99) + thor (~> 0.19) + backports (3.15.0) builder (3.0.4) - capybara (2.1.0) - mime-types (>= 1.16) + capybara (2.18.0) + addressable + mini_mime (>= 0.1.3) nokogiri (>= 1.3.3) rack (>= 1.0.0) rack-test (>= 0.5.4) - xpath (~> 2.0) - childprocess (0.3.9) - ffi (~> 1.0, >= 1.0.11) - coderay (1.0.9) - cucumber (1.3.3) + xpath (>= 2.0, < 4.0) + childprocess (1.0.1) + rake (< 13.0) + coderay (1.1.2) + contracts (0.16.0) + cucumber (3.1.2) builder (>= 2.1.2) - diff-lcs (>= 1.1.3) - gherkin (~> 2.12.0) - multi_json (~> 1.7.5) - multi_test (~> 0.0.1) - cucumber-rails (1.3.1) - capybara (>= 1.1.2) - cucumber (>= 1.2.0) - nokogiri (>= 1.5.0) - rails (~> 3.0) - database_cleaner (1.0.1) - diff-lcs (1.2.4) + cucumber-core (~> 3.2.0) + cucumber-expressions (~> 6.0.1) + cucumber-wire (~> 0.0.1) + diff-lcs (~> 1.3) + gherkin (~> 5.1.0) + multi_json (>= 1.7.5, < 2.0) + multi_test (>= 0.1.2) + cucumber-core (3.2.1) + backports (>= 3.8.0) + cucumber-tag_expressions (~> 1.1.0) + gherkin (~> 5.0) + cucumber-expressions (6.0.1) + cucumber-rails (1.4.5) + capybara (>= 1.1.2, < 3) + cucumber (>= 1.3.8, < 4) + mime-types (>= 1.16, < 4) + nokogiri (~> 1.5) + railties (>= 3, < 5.1) + cucumber-tag_expressions (1.1.1) + cucumber-wire (0.0.1) + database_cleaner (1.7.0) + diff-lcs (1.3) erubis (2.7.0) - ffi (1.9.0) - gherkin (2.12.0) - multi_json (~> 1.3) + ffi (1.11.1) + gherkin (5.1.0) hike (1.2.3) i18n (0.6.1) journey (1.0.4) - json (1.8.0) - launchy (2.3.0) + json (1.8.6) + launchy (2.4.3) addressable (~> 2.3) - mail (2.5.4) + mail (2.5.5) mime-types (~> 1.16) treetop (~> 1.4.8) - method_source (0.8.1) - mime-types (1.23) - mini_portile (0.5.1) - multi_json (1.7.7) - multi_test (0.0.1) - nokogiri (1.6.0) - mini_portile (~> 0.5.0) - pg (0.15.1) - polyglot (0.3.3) - pry (0.9.12.2) - coderay (~> 1.0.5) - method_source (~> 0.8) - slop (~> 3.4) - rack (1.4.5) - rack-cache (1.2) + method_source (0.9.2) + mime-types (1.25.1) + mini_mime (1.0.1) + mini_portile2 (2.4.0) + multi_json (1.13.1) + multi_test (0.1.2) + nokogiri (1.10.3) + mini_portile2 (~> 2.4.0) + pg (0.21.0) + polyglot (0.3.5) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.1.0) + rack (1.4.7) + rack-cache (1.9.0) rack (>= 0.4) - rack-ssl (1.3.3) + rack-ssl (1.3.4) rack - rack-test (0.6.2) + rack-test (0.6.3) rack (>= 1.0) rails (3.2.13) actionmailer (= 3.2.13) @@ -108,35 +126,40 @@ GEM rake (>= 0.8.7) rdoc (~> 3.4) thor (>= 0.14.6, < 2.0) - rake (10.1.0) + rake (12.3.2) rdoc (3.12.2) json (~> 1.4) - rspec-core (2.14.0) - rspec-expectations (2.14.0) - diff-lcs (>= 1.1.3, < 2.0) - rspec-mocks (2.14.0) - rspec-rails (2.14.0) + rspec-core (3.8.1) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.4) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-rails (3.8.2) actionpack (>= 3.0) activesupport (>= 3.0) railties (>= 3.0) - rspec-core (~> 2.14.0) - rspec-expectations (~> 2.14.0) - rspec-mocks (~> 2.14.0) - slop (3.4.5) - sprockets (2.2.2) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.2) + sprockets (2.2.3) hike (~> 1.2) multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) - sqlite3 (1.3.7) - thor (0.18.1) + sqlite3 (1.3.13) + thor (0.20.3) tilt (1.4.1) - treetop (1.4.14) + treetop (1.4.15) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.37) - xpath (2.0.0) - nokogiri (~> 1.3) + tzinfo (0.3.55) + xpath (3.2.0) + nokogiri (~> 1.8) PLATFORMS ruby @@ -147,9 +170,13 @@ DEPENDENCIES capybara cucumber-rails database_cleaner + json (~> 1.8.6) launchy - pg + pg (~> 0.11) pry rails (= 3.2.13) rspec-rails - sqlite3 + sqlite3 (~> 1.3.5) + +BUNDLED WITH + 1.17.3 diff --git a/sample_app/config/database.yml b/sample_app/config/database.yml index d26cc790..d75ece45 100644 --- a/sample_app/config/database.yml +++ b/sample_app/config/database.yml @@ -2,27 +2,27 @@ # gem install sqlite3-ruby (not necessary on OS X Leopard) development: adapter: postgresql - username: postgres - password: + username: <%= ENV['POSTGRES_USER'] || 'postgres' %> + password: <%= ENV['POSTGRES_PASSWORD'] || '' %> database: octopus_sample_app_development encoding: unicode - host: localhost + host: <%= ENV['POSTGRES_HOST'] || 'localhost' %> # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". # Do not set this db to the same as development or production. test: &test adapter: postgresql - username: postgres - password: + username: <%= ENV['POSTGRES_USER'] || 'postgres' %> + password: <%= ENV['POSTGRES_PASSWORD'] || '' %> database: octopus_sample_app_test encoding: unicode - host: localhost + host: <%= ENV['POSTGRES_HOST'] || 'localhost' %> production: adapter: postgresql - username: postgres - password: + username: <%= ENV['POSTGRES_USER'] || 'postgres' %> + password: <%= ENV['POSTGRES_PASSWORD'] || '' %> database: octopus_sample_app_production encoding: unicode - host: localhost + host: <%= ENV['POSTGRES_HOST'] || 'localhost' %> diff --git a/sample_app/script/ci_build b/sample_app/script/ci_build new file mode 100755 index 00000000..38ba2d93 --- /dev/null +++ b/sample_app/script/ci_build @@ -0,0 +1,14 @@ +#!/bin/bash +dir=`readlink -f $0` +dir="`dirname $dir`/.." +RAILS_ROOT=`readlink -f $dir` +cd $RAILS_ROOT + +bundle install --path=$RAILS_ROOT/.bundle + +for stage in test development production; do + PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -c "create database octopus_sample_app_$stage" +done + +# Not working... +bundle exec cucumber From 8e2671d20dd6ff48c82e58ac2eedbee8b592765f Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Tue, 25 Jun 2019 13:55:15 -0500 Subject: [PATCH 03/41] Ooops, revert a couple files. These weren't supposed to make it into this PR --- gemfiles/rails42.gemfile | 2 +- lib/octopus/abstract_adapter.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gemfiles/rails42.gemfile b/gemfiles/rails42.gemfile index 9be86420..24e5e488 100644 --- a/gemfiles/rails42.gemfile +++ b/gemfiles/rails42.gemfile @@ -3,5 +3,5 @@ source "https://rubygems.org" gem "activerecord", "~> 4.2.0" - +gem "mysql2", "0.4.10" gemspec path: "../" diff --git a/lib/octopus/abstract_adapter.rb b/lib/octopus/abstract_adapter.rb index 646f2272..f6ce7956 100644 --- a/lib/octopus/abstract_adapter.rb +++ b/lib/octopus/abstract_adapter.rb @@ -19,7 +19,7 @@ def method_missing(meth, *args, &block) end def octopus_shard - @config && @config[:octopus_shard] + @config[:octopus_shard] end def initialize(*args) From c3ac7ca16912ced8387fdfeaa9cace095f581503 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:22:34 -0500 Subject: [PATCH 04/41] Add DB2 server --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b74c4a00..3e0cffe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,13 @@ services: MYSQL_HOST: mysql MYSQL_USER: root MYSQL_PASSWORD: testpassword + DB2_HOST: db2 + DB2_USER: db2inst1 + DB2_PASSWORD: testpassword depends_on: - mysql - postgres + - db2 # Wait for databases to start up, run test suite command: ./docker/startup.sh mysql: @@ -39,3 +43,10 @@ services: interval: 10s timeout: 5s retries: 5 + db2: + image: ibmcom/db2express-c + restart: always + environment: + DB2INST1_PASSWORD: testpassword + LICENSE: accept + command: db2start && sleep infinity From ce42246ab4b93c95241bb4d18bf9169c980cb7ac Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:23:11 -0500 Subject: [PATCH 05/41] Trying to add ibm_db gem to gemspec This is failing with the stacktrace: Caused by: LoadError: cannot load such file -- arel/visitors/bind_visitor /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `require' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `block in require' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:257:in `load_dependency' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `require' /usr/src/app/.bundle/ruby/2.5.0/gems/ibm_db-4.0.0/lib/active_record/connection_adapters/ibm_db_adapter.rb:13:in `' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `require' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `block in require' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:257:in `load_dependency' /usr/src/app/.bundle/ruby/2.5.0/gems/activesupport-5.2.3/lib/active_support/dependencies.rb:291:in `require' /usr/src/app/lib/octopus/proxy_config.rb:245:in `initialize_adapter' --- ar-octopus.gemspec | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ar-octopus.gemspec b/ar-octopus.gemspec index 6e8c1017..c0d34a99 100644 --- a/ar-octopus.gemspec +++ b/ar-octopus.gemspec @@ -34,6 +34,10 @@ Gem::Specification.new do |s| s.add_development_dependency 'rubocop' s.add_development_dependency 'sqlite3', '~> 1.3.6' s.add_development_dependency 'pry-byebug' + # ibm_db <=4.0.0 breaks on arel version 9.0. Rails 5.0 uses arel 8.0 + #s.add_development_dependency 'activerecord', '>= 4.2.0', '<= 5.0.7.2' + #s.add_development_dependency 'activesupport', '>= 4.2.0', '<= 5.0.7.2' + s.add_development_dependency 'ibm_db', '4.0.0' s.license = 'MIT' end From 75a1ef5a4f3b2d6785c3f2d7a2e311d6f636cec7 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:25:19 -0500 Subject: [PATCH 06/41] Adding ibm_db versions for each rails version --- Appraisals | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Appraisals b/Appraisals index 223fb4dc..a09d0180 100644 --- a/Appraisals +++ b/Appraisals @@ -1,13 +1,16 @@ appraise "rails42" do gem "activerecord", "~> 4.2.0" + gem "ibm_db", "~> 3.0.5" end appraise "rails5" do gem "activerecord", "~> 5.0.0" + gem "ibm_db", "~> 4.0.0" end appraise "rails51" do gem "activerecord", "~> 5.1.0" + gem "ibm_db", "~> 4.0.0" end appraise "rails52" do From 77a315578053c043ae651e0e1575cc5f742744ae Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:36:54 -0500 Subject: [PATCH 07/41] Install the DB2 ODBC driver The gem install does this, but the download is slow. Watching the wget progress is more satisfying. --- Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1e5e673e..081c262b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,16 @@ FROM ruby:2.5 RUN apt-get update && apt-get install -y \ mysql-client \ - postgresql + postgresql \ + vim + +# Install the IBM DB2 CLI driver. The ibm_db gem install +# will also do this, but it has no status output and takes +# several minutes. +ENV IBM_DB_HOME=/opt/ibm/clidriver +RUN wget http://public.dhe.ibm.com/ibmdl/export/pub/software/data/db2/drivers/odbc_cli/linuxx64_odbc_cli.tar.gz -O /tmp/linuxx64_odbc_cli.tar.gz \ + && mkdir -p /opt/ibm \ + && tar -C /opt/ibm -xzvf /tmp/linuxx64_odbc_cli.tar.gz WORKDIR /usr/src/app From faaca64f237c71fb81caf52aa8182d6ff92da55f Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:38:03 -0500 Subject: [PATCH 08/41] Add a DB2 shard --- spec/config/shards.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spec/config/shards.yml b/spec/config/shards.yml index ca3c2a16..04aa13f6 100644 --- a/spec/config/shards.yml +++ b/spec/config/shards.yml @@ -28,6 +28,15 @@ octopus: &octopus adapter: sqlite3 database: /tmp/database.sqlite3 + db2_shard: + adapter: ibm_db + username: <%= ENV['DB2_USER'] || 'db2inst1' %> + password: <%= ENV['DB2_PASSWORD'] || '' %> + host: <%= ENV['DB2_HOST'] || 'db2' %> + port: <%= ENV['DB2_PORT'] || '50000' %> + database: octopus6 + schema: <%= ENV['DB2_SCHEMA'] || 'db2inst1' %> + history_shards: aug2009: database: octopus_shard_2 From 28e919026d1d5691b73d31f285db9fc814a2a9ef Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:40:44 -0500 Subject: [PATCH 09/41] Appriasal run for ibm_db --- gemfiles/rails42.gemfile | 3 ++- gemfiles/rails51.gemfile | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gemfiles/rails42.gemfile b/gemfiles/rails42.gemfile index 24e5e488..e15d484b 100644 --- a/gemfiles/rails42.gemfile +++ b/gemfiles/rails42.gemfile @@ -3,5 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 4.2.0" -gem "mysql2", "0.4.10" +gem "ibm_db", "~> 3.0.5" + gemspec path: "../" diff --git a/gemfiles/rails51.gemfile b/gemfiles/rails51.gemfile index e2f8f85c..144640da 100644 --- a/gemfiles/rails51.gemfile +++ b/gemfiles/rails51.gemfile @@ -3,5 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 5.1.0" +gem "ibm_db", "~> 4.0.0" gemspec path: "../" From 3e9f50b32060c8deccbc8c79038a55d027f14423 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:43:39 -0500 Subject: [PATCH 10/41] Check that @config exists. Fixes #413 Error message is: ActiveRecord::StatementInvalid: NoMethodError: undefined method `[]' for nil:NilClass: from octopus/lib/octopus/abstract_adapter.rb:23:in `octopus_shard' from octopus/lib/octopus/abstract_adapter.rb:12:in `instrument' from vendor/bundle/ruby/2.3.0/gems/activerecord-4.2.11.1/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' This has shown to surface in the ibm_db and sqlserver adapters. --- lib/octopus/abstract_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/octopus/abstract_adapter.rb b/lib/octopus/abstract_adapter.rb index f6ce7956..646f2272 100644 --- a/lib/octopus/abstract_adapter.rb +++ b/lib/octopus/abstract_adapter.rb @@ -19,7 +19,7 @@ def method_missing(meth, *args, &block) end def octopus_shard - @config[:octopus_shard] + @config && @config[:octopus_shard] end def initialize(*args) From 4fd11290f99e92d413ceb29cd5a787b9f40995bd Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 14:43:39 -0500 Subject: [PATCH 11/41] Check that @config exists. Fixes #413 Error message is: ActiveRecord::StatementInvalid: NoMethodError: undefined method `[]' for nil:NilClass: from octopus/lib/octopus/abstract_adapter.rb:23:in `octopus_shard' from octopus/lib/octopus/abstract_adapter.rb:12:in `instrument' from vendor/bundle/ruby/2.3.0/gems/activerecord-4.2.11.1/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' This has shown to surface in the ibm_db and sqlserver adapters. --- lib/octopus/abstract_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/octopus/abstract_adapter.rb b/lib/octopus/abstract_adapter.rb index f6ce7956..646f2272 100644 --- a/lib/octopus/abstract_adapter.rb +++ b/lib/octopus/abstract_adapter.rb @@ -19,7 +19,7 @@ def method_missing(meth, *args, &block) end def octopus_shard - @config[:octopus_shard] + @config && @config[:octopus_shard] end def initialize(*args) From 5f3bc57e4d8ccadef12c44cd3ed10749eacf6521 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 26 Jun 2019 15:42:29 -0500 Subject: [PATCH 12/41] Monkey patch IBM_DBAdapter to add @config This allows the InstrumenterDecorator to properly log the shard name --- lib/octopus.rb | 1 + lib/octopus/adapter_paches.rb | 5 +++++ lib/octopus/proxy_config.rb | 11 ++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 lib/octopus/adapter_paches.rb diff --git a/lib/octopus.rb b/lib/octopus.rb index ffea3f78..72f2c196 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -191,6 +191,7 @@ def self.fully_replicated(&_block) require 'octopus/persistence' require 'octopus/log_subscriber' require 'octopus/abstract_adapter' +require 'octopus/adapter_patches' require 'octopus/singular_association' require 'octopus/finder_methods' require 'octopus/query_cache_for_shards' unless Octopus.rails4? diff --git a/lib/octopus/adapter_paches.rb b/lib/octopus/adapter_paches.rb new file mode 100644 index 00000000..bac63582 --- /dev/null +++ b/lib/octopus/adapter_paches.rb @@ -0,0 +1,5 @@ +# Monkey patch unsupported database adapters that have not +# used an internal @config hash +class ActiveRecord::ConnectionAdapters::IBM_DBAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter + attr_accessor :config +end diff --git a/lib/octopus/proxy_config.rb b/lib/octopus/proxy_config.rb index 93a2e5bc..e94b5e16 100644 --- a/lib/octopus/proxy_config.rb +++ b/lib/octopus/proxy_config.rb @@ -224,7 +224,11 @@ def connection_pool_for(config, adapter) spec = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(name, config.dup, adapter) end - ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + if patched_adapter?(config) + pool.connection.config = config.dup + end + pool end def resolve_string_connection(spec) @@ -240,6 +244,11 @@ def structurally_slave_group?(config) config.is_a?(Hash) && config.values.any? { |v| structurally_slave? v } end + def patched_adapter?(config) + # Test and add more adapters here? + ["ibm_db"].member?(config && config['adapter']) + end + def initialize_adapter(adapter) begin require "active_record/connection_adapters/#{adapter}_adapter" From fa21f02ac658df8befc5bc873dcfc0bca14385ac Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Fri, 28 Jun 2019 09:06:56 -0500 Subject: [PATCH 13/41] Ooof, typo in filename --- lib/octopus/adapter_patches.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 lib/octopus/adapter_patches.rb diff --git a/lib/octopus/adapter_patches.rb b/lib/octopus/adapter_patches.rb new file mode 100644 index 00000000..bac63582 --- /dev/null +++ b/lib/octopus/adapter_patches.rb @@ -0,0 +1,5 @@ +# Monkey patch unsupported database adapters that have not +# used an internal @config hash +class ActiveRecord::ConnectionAdapters::IBM_DBAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter + attr_accessor :config +end From 9da3a89bcd6c478ee544fadbccab8752c371ad94 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Fri, 28 Jun 2019 16:23:41 -0500 Subject: [PATCH 14/41] Adding db2 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e0cffe4..ba702239 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: timeout: 5s retries: 5 db2: - image: ibmcom/db2express-c + build: docker/db2 restart: always environment: DB2INST1_PASSWORD: testpassword From 08ad9e0d9be042f811db5da73eeaa5d0f9f29abc Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Fri, 28 Jun 2019 16:24:19 -0500 Subject: [PATCH 15/41] Adding DB2 --- Rakefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Rakefile b/Rakefile index 0aa741ac..21c3bee5 100644 --- a/Rakefile +++ b/Rakefile @@ -3,6 +3,7 @@ require 'rspec/core/rake_task' require 'rubocop/rake_task' require 'appraisal' +$stdout.sync = true RSpec::Core::RakeTask.new RuboCop::RakeTask.new @@ -27,11 +28,22 @@ namespace :db do :encoding => 'utf8', } + db2_spec = { + :adapter => 'ibm_db', + :host => (ENV['DB2_HOST'] || 'db2'), + :username => (ENV['DB2_USER'] || 'db2inst1'), + :password => (ENV['DB2_PASSWORD'] || ''), + :port => (ENV['DB2_PORT'] || '50000'), + :schema => (ENV['DB2_SCHEMA'] || 'db2inst1'), + :database => 'octopus6', + :encoding => 'utf8', + } ` rm -f /tmp/database.sqlite3 ` require 'active_record' # Connects to PostgreSQL + puts "Connecting to PostgreSQL" ActiveRecord::Base.establish_connection(pg_spec.merge('database' => 'postgres', 'schema_search_path' => 'public')) (1..2).map do |i| # drop the old database (if it exists) @@ -41,6 +53,7 @@ namespace :db do end # Connect to MYSQL + puts "Connecting to MySQL" ActiveRecord::Base.establish_connection(mysql_spec) (1..5).map do |i| # drop the old database (if it exists) @@ -48,6 +61,9 @@ namespace :db do # create new database ActiveRecord::Base.connection.create_database("octopus_shard_#{i}") end + + puts "Connecting to DB2" + ActiveRecord::Base.establish_connection(db2_spec) end desc 'Create tables on tests databases' From ee8c9d22c71ab75685de5a8dc1fca10e2750bcbf Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Fri, 28 Jun 2019 16:24:47 -0500 Subject: [PATCH 16/41] Setting up DB2 server image --- Dockerfile | 5 +++++ docker/db2-ld.conf | 1 + docker/db2/Dockerfile | 12 ++++++++++++ docker/db2/create_databases.sh | 6 ++++++ docker/db2dsdriver.cfg | 9 +++++++++ docker/startup.sh | 11 +++++++++++ lib/octopus/adapter_paches.rb | 5 ----- 7 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 docker/db2-ld.conf create mode 100644 docker/db2/Dockerfile create mode 100755 docker/db2/create_databases.sh create mode 100644 docker/db2dsdriver.cfg delete mode 100644 lib/octopus/adapter_paches.rb diff --git a/Dockerfile b/Dockerfile index 081c262b..3d953b39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,11 @@ RUN wget http://public.dhe.ibm.com/ibmdl/export/pub/software/data/db2/drivers/od && mkdir -p /opt/ibm \ && tar -C /opt/ibm -xzvf /tmp/linuxx64_odbc_cli.tar.gz +# Set up DB2 libraries and catalogs +COPY docker/db2-ld.conf /etc/ld.so.conf.d/ +COPY docker/db2dsdriver.cfg /opt/ibm/clidriver/cfg/ +RUN ldconfig + WORKDIR /usr/src/app # Pull in a full profile for gem/bundler diff --git a/docker/db2-ld.conf b/docker/db2-ld.conf new file mode 100644 index 00000000..7f6f5db5 --- /dev/null +++ b/docker/db2-ld.conf @@ -0,0 +1 @@ +/opt/ibm/clidriver/lib diff --git a/docker/db2/Dockerfile b/docker/db2/Dockerfile new file mode 100644 index 00000000..8396f77b --- /dev/null +++ b/docker/db2/Dockerfile @@ -0,0 +1,12 @@ +FROM ibmcom/db2express-c:latest + +SHELL ["/bin/bash", "-l", "-c"] +COPY create_databases.sh / +# Bake the Octopus databases into the DB2 image, +# since they take a very long time to create. +# NOTE: this is the only way to properly source +# the db2 profile environment to use the +# DB2 CLI command line. +RUN su - db2inst1 -c /create_databases.sh + +CMD db2start && sleep infinity diff --git a/docker/db2/create_databases.sh b/docker/db2/create_databases.sh new file mode 100755 index 00000000..baaf19e0 --- /dev/null +++ b/docker/db2/create_databases.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +source /home/db2inst1/sqllib/db2profile +db2start +echo "Creating databases, this may take around 6 - 8 minutes..." +time db2 create database octopus6 diff --git a/docker/db2dsdriver.cfg b/docker/db2dsdriver.cfg new file mode 100644 index 00000000..91b8ea4d --- /dev/null +++ b/docker/db2dsdriver.cfg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docker/startup.sh b/docker/startup.sh index db7bb567..4d13e590 100755 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -18,7 +18,18 @@ done >&2 echo "MySQL is ready" +until ! /opt/ibm/clidriver/bin/db2cli validate -database $DB2_DATABASE:$DB2_HOST:$DB2_PORT -connect -user $DB2_USERNAME -passwd $DB2_PASSWORD | grep -q FAILED ; do + >&2 echo "DB2 is unavailable - sleeping" + sleep 10 +done + +>&2 echo "DB2 is ready" + set -x +cd /usr/src/ibm_db/IBM_DB_Adapter/ibm_db/ext +ruby extconf.rb +make +cd /usr/src/app bundle install bundle exec rake db:prepare bundle exec rake appraisal:install diff --git a/lib/octopus/adapter_paches.rb b/lib/octopus/adapter_paches.rb deleted file mode 100644 index bac63582..00000000 --- a/lib/octopus/adapter_paches.rb +++ /dev/null @@ -1,5 +0,0 @@ -# Monkey patch unsupported database adapters that have not -# used an internal @config hash -class ActiveRecord::ConnectionAdapters::IBM_DBAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter - attr_accessor :config -end From 4a488a94361d4abc1a35d8ffcb60bd1b98f435b9 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 09:05:17 -0500 Subject: [PATCH 17/41] Add conditional check to allow the ibm_db gem to be optional --- lib/octopus.rb | 4 ++++ lib/octopus/adapter_patches.rb | 7 +++++-- spec/config/shards.yml | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/octopus.rb b/lib/octopus.rb index 72f2c196..a68b112a 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -122,6 +122,10 @@ def self.atleast_rails52? ActiveRecord::VERSION::MAJOR > 5 || (ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR > 1) end + def self.ibm_db_support? + defined?(IBM_DB) && defined?(ActiveRecord::ConnectionAdapters::IBM_DBAdapter) + end + attr_writer :logger def self.logger diff --git a/lib/octopus/adapter_patches.rb b/lib/octopus/adapter_patches.rb index bac63582..6553b230 100644 --- a/lib/octopus/adapter_patches.rb +++ b/lib/octopus/adapter_patches.rb @@ -1,5 +1,8 @@ # Monkey patch unsupported database adapters that have not # used an internal @config hash -class ActiveRecord::ConnectionAdapters::IBM_DBAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter - attr_accessor :config + +if Octopus.ibm_db_support? + class ActiveRecord::ConnectionAdapters::IBM_DBAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter + attr_accessor :config + end end diff --git a/spec/config/shards.yml b/spec/config/shards.yml index 04aa13f6..0d0634be 100644 --- a/spec/config/shards.yml +++ b/spec/config/shards.yml @@ -28,6 +28,7 @@ octopus: &octopus adapter: sqlite3 database: /tmp/database.sqlite3 + <% if Octopus.ibm_db_support? %> db2_shard: adapter: ibm_db username: <%= ENV['DB2_USER'] || 'db2inst1' %> @@ -36,6 +37,7 @@ octopus: &octopus port: <%= ENV['DB2_PORT'] || '50000' %> database: octopus6 schema: <%= ENV['DB2_SCHEMA'] || 'db2inst1' %> + <% end %> history_shards: aug2009: From 66c61c6397dfa0488e21f44b72a85c70c6ab3410 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 09:26:30 -0500 Subject: [PATCH 18/41] Optionally load the ibm_db gem --- lib/octopus.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/octopus.rb b/lib/octopus.rb index a68b112a..335693ca 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -5,6 +5,13 @@ require 'yaml' require 'erb' +# Optionally load the ibm_db gem +begin + require 'ibm_db' + require 'active_record/connection_adapters/ibm_db_adapter' +rescue LoadError +end + module Octopus def self.env @env ||= 'octopus' From a70aad5cb51e960e77f4f2b6fe58e69ba79a56d9 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 11:09:53 -0500 Subject: [PATCH 19/41] ibm_db support only works in rails 4.2 and 5.0 --- Appraisals | 2 +- Rakefile | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Appraisals b/Appraisals index a09d0180..ead15c33 100644 --- a/Appraisals +++ b/Appraisals @@ -5,12 +5,12 @@ end appraise "rails5" do gem "activerecord", "~> 5.0.0" + # For now, ibm_db only works on rails 5.0 gem "ibm_db", "~> 4.0.0" end appraise "rails51" do gem "activerecord", "~> 5.1.0" - gem "ibm_db", "~> 4.0.0" end appraise "rails52" do diff --git a/Rakefile b/Rakefile index 21c3bee5..65f15ba4 100644 --- a/Rakefile +++ b/Rakefile @@ -28,16 +28,6 @@ namespace :db do :encoding => 'utf8', } - db2_spec = { - :adapter => 'ibm_db', - :host => (ENV['DB2_HOST'] || 'db2'), - :username => (ENV['DB2_USER'] || 'db2inst1'), - :password => (ENV['DB2_PASSWORD'] || ''), - :port => (ENV['DB2_PORT'] || '50000'), - :schema => (ENV['DB2_SCHEMA'] || 'db2inst1'), - :database => 'octopus6', - :encoding => 'utf8', - } ` rm -f /tmp/database.sqlite3 ` require 'active_record' @@ -62,8 +52,6 @@ namespace :db do ActiveRecord::Base.connection.create_database("octopus_shard_#{i}") end - puts "Connecting to DB2" - ActiveRecord::Base.establish_connection(db2_spec) end desc 'Create tables on tests databases' @@ -76,6 +64,10 @@ namespace :db do require "#{File.dirname(__FILE__)}/spec/support/database_connection" shard_symbols = [:master, :brazil, :canada, :russia, :alone_shard, :postgresql_shard, :sqlite_shard] + if Octopus.ibm_db_support? + shard_symbols += [:db2_shard] + end + shard_symbols << :protocol_shard shard_symbols.each do |shard_symbol| # Rails 3.1 needs to do some introspection around the base class, which requires @@ -86,6 +78,16 @@ namespace :db do BlankModel.using(shard_symbol).connection.initialize_schema_migrations_table BlankModel.using(shard_symbol).connection.initialize_metadata_table if Octopus.atleast_rails50? + # Since it's too slow to drop/create DB2 databases, + # drop all the tables instead + if shard_symbol == :db2_shard + [:users,:clients,:cats,:items,:computers,:keyboards, :roles, + :permissions, :permissions_roles, :assignments, :programmers, + :projects, :comments, :parts, :yummy, :adverts, :custom].each do |table| + BlankModel.using(shard_symbol).connection.drop_table(table) + end + end + BlankModel.using(shard_symbol).connection.create_table(:users) do |u| u.string :name u.integer :number From 5ef3e27405596d3e04697c37efb87b3cba22e60d Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 12:24:01 -0500 Subject: [PATCH 20/41] Remove ibm_db gem, this is optional now --- ar-octopus.gemspec | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ar-octopus.gemspec b/ar-octopus.gemspec index c0d34a99..6e8c1017 100644 --- a/ar-octopus.gemspec +++ b/ar-octopus.gemspec @@ -34,10 +34,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'rubocop' s.add_development_dependency 'sqlite3', '~> 1.3.6' s.add_development_dependency 'pry-byebug' - # ibm_db <=4.0.0 breaks on arel version 9.0. Rails 5.0 uses arel 8.0 - #s.add_development_dependency 'activerecord', '>= 4.2.0', '<= 5.0.7.2' - #s.add_development_dependency 'activesupport', '>= 4.2.0', '<= 5.0.7.2' - s.add_development_dependency 'ibm_db', '4.0.0' s.license = 'MIT' end From 82205de96ce86e33c01cf2a43f7e5daf0d5de9b5 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 12:24:32 -0500 Subject: [PATCH 21/41] Remove ibm_db from rails 5.1 --- gemfiles/rails5.gemfile | 1 + gemfiles/rails51.gemfile | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/rails5.gemfile b/gemfiles/rails5.gemfile index a7a9bb00..6f6e2aa3 100644 --- a/gemfiles/rails5.gemfile +++ b/gemfiles/rails5.gemfile @@ -3,5 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 5.0.0" +gem "ibm_db", "~> 4.0.0" gemspec path: "../" diff --git a/gemfiles/rails51.gemfile b/gemfiles/rails51.gemfile index 144640da..e2f8f85c 100644 --- a/gemfiles/rails51.gemfile +++ b/gemfiles/rails51.gemfile @@ -3,6 +3,5 @@ source "https://rubygems.org" gem "activerecord", "~> 5.1.0" -gem "ibm_db", "~> 4.0.0" gemspec path: "../" From 1c658c1c6320ad539700ba4f0a9366596b0f3c18 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 1 Jul 2019 12:25:14 -0500 Subject: [PATCH 22/41] First DB2 test is working --- spec/octopus/db2_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 spec/octopus/db2_spec.rb diff --git a/spec/octopus/db2_spec.rb b/spec/octopus/db2_spec.rb new file mode 100644 index 00000000..2ca2aeb6 --- /dev/null +++ b/spec/octopus/db2_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +describe 'when using db2' do + before(:each) do + OctopusHelper.clean_connection_proxy + skip "DB2 support not loaded" unless Octopus.ibm_db_support? + end + + it 'should create an object' do + Octopus.using(:db2_shard) do + Cat.create!(:name => "Kitty") + expect(Cat.count).to eq(1) + end + end + +end From 7f48d8587716d78441128d0669bcf5981fe92c56 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Tue, 2 Jul 2019 15:54:29 -0500 Subject: [PATCH 23/41] Note on DB2 --- Appraisals | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Appraisals b/Appraisals index ead15c33..ae909143 100644 --- a/Appraisals +++ b/Appraisals @@ -1,5 +1,8 @@ appraise "rails42" do gem "activerecord", "~> 4.2.0" + #Causes issue https://github.com/ibmdb/ruby-ibmdb/issues/31 + # Wait for this PR to be accepted or pull in a local gem for yourself: + # https://github.com/ibmdb/ruby-ibmdb/pull/38/files gem "ibm_db", "~> 3.0.5" end From c6e267f466cc7d5ce47d3a87230a34dfa109272b Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Tue, 2 Jul 2019 15:55:11 -0500 Subject: [PATCH 24/41] We don't need this with Appraisal --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d953b39..e484eb26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,11 +25,11 @@ SHELL ["/bin/bash", "-l", "-c"] RUN gem install --no-document bundler -v 1.16.6 +# Uncomment if you don't want to use Appraisal # Copy only what's needed for bundler -COPY Gemfile ar-octopus.gemspec /usr/src/app/ -COPY lib/octopus/version.rb /usr/src/app/lib/octopus/version.rb - -RUN bundle install --path=.bundle +#COPY Gemfile ar-octopus.gemspec /usr/src/app/ +#COPY lib/octopus/version.rb /usr/src/app/lib/octopus/version.rb +#RUN bundle install --path=.bundle # Uncomment if you want to copy the octopus repo # into the Docker image itself. docker-compose is From 3495359da0b80f1b0d15d43c95bdb33d24d8091f Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Tue, 2 Jul 2019 15:55:59 -0500 Subject: [PATCH 25/41] Tests for DB2 --- Rakefile | 9 ++++-- spec/config/shards.yml | 67 +++++++++++++++++++++++++++++++++++----- spec/octopus/db2_spec.rb | 49 +++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 13 deletions(-) diff --git a/Rakefile b/Rakefile index 65f15ba4..9e135916 100644 --- a/Rakefile +++ b/Rakefile @@ -65,7 +65,7 @@ namespace :db do shard_symbols = [:master, :brazil, :canada, :russia, :alone_shard, :postgresql_shard, :sqlite_shard] if Octopus.ibm_db_support? - shard_symbols += [:db2_shard] + shard_symbols += [:db2_1, :db2_2, :db2_3, :db2_4] end shard_symbols << :protocol_shard @@ -80,11 +80,14 @@ namespace :db do # Since it's too slow to drop/create DB2 databases, # drop all the tables instead - if shard_symbol == :db2_shard + if BlankModel.using(shard_symbol).connection.adapter_name == "IBM_DB" [:users,:clients,:cats,:items,:computers,:keyboards, :roles, :permissions, :permissions_roles, :assignments, :programmers, :projects, :comments, :parts, :yummy, :adverts, :custom].each do |table| - BlankModel.using(shard_symbol).connection.drop_table(table) + begin + BlankModel.using(shard_symbol).connection.drop_table(table) + rescue ActiveRecord::StatementInvalid + end end end diff --git a/spec/config/shards.yml b/spec/config/shards.yml index 0d0634be..2e26f1b5 100644 --- a/spec/config/shards.yml +++ b/spec/config/shards.yml @@ -10,6 +10,17 @@ mysql_unavailable: &mysql_unavailable host: 192.0.2.1 connect_timeout: 3 +<% if Octopus.ibm_db_support? %> +db2: &db2 + adapter: ibm_db + username: <%= ENV['DB2_USER'] || 'db2inst1' %> + password: <%= ENV['DB2_PASSWORD'] || '' %> + host: <%= ENV['DB2_HOST'] || 'db2' %> + port: <%= ENV['DB2_PORT'] || '50000' %> + schema: <%= ENV['DB2_SCHEMA'] || 'db2inst1' %> + database: octopus6 +<% end %> + octopus: &octopus shards: alone_shard: @@ -27,16 +38,20 @@ octopus: &octopus sqlite_shard: adapter: sqlite3 database: /tmp/database.sqlite3 - + <% if Octopus.ibm_db_support? %> - db2_shard: - adapter: ibm_db - username: <%= ENV['DB2_USER'] || 'db2inst1' %> - password: <%= ENV['DB2_PASSWORD'] || '' %> - host: <%= ENV['DB2_HOST'] || 'db2' %> - port: <%= ENV['DB2_PORT'] || '50000' %> + db2_1: + <<: *db2 database: octopus6 - schema: <%= ENV['DB2_SCHEMA'] || 'db2inst1' %> + db2_2: + <<: *db2 + database: octopus7 + db2_3: + <<: *db2 + database: octopus8 + db2_4: + <<: *db2 + database: octopus9 <% end %> history_shards: @@ -239,3 +254,39 @@ modify_config: adapter: modify_config database: octopus_shard_1 host: localhost + +<% if Octopus.ibm_db_support? %> +db2_case1: + replicated: true + fully_replicated: false + shards: + narnia: + <<: *db2 + database: octopus6 + slaves: + archenland: + <<: *db2 + database: octopus6 + calormen: + <<: *db2 + database: octopus7 + +db2_case2: + replicated: true + fully_replicated: false + # Removes the name `master` in the shard list + master_shard: narnia + shards: + narnia: + <<: *db2 + database: octopus6 + archenland: + <<: *db2 + database: octopus7 + calormen: + <<: *db2 + database: octopus8 + telmar: + <<: *db2 + database: octopus9 +<% end %> diff --git a/spec/octopus/db2_spec.rb b/spec/octopus/db2_spec.rb index 2ca2aeb6..cf193fc1 100644 --- a/spec/octopus/db2_spec.rb +++ b/spec/octopus/db2_spec.rb @@ -7,10 +7,55 @@ end it 'should create an object' do - Octopus.using(:db2_shard) do - Cat.create!(:name => "Kitty") + Octopus.using(:db2_1) do + Cat.create!(:name => "Test") expect(Cat.count).to eq(1) end end + it 'should shard with slave groups' do + OctopusHelper.using_environment :db2_case1 do + #allow(Octopus).to receive(:env).and_return('db2_case1') + Octopus.using(:narnia) do + Cat.create(:name => "Aslan") + end + expect(Cat.using(:narnia).count).to eq(1) + + # first hit round robins to calormen + expect(Cat.using(:shard => :narnia, :slave_group => :slaves).count).to eq(0) + + # second hit round robins to archenland (shared db with narnia) + expect(Cat.using(:shard => :narnia, :slave_group => :slaves).count).to eq(1) + end + end + + it 'should work with plain shards' do + OctopusHelper.using_environment :db2_case2 do + Octopus.using(:narnia) do + Cat.create(:name => "Aslan") + User.create(:name => "Peter") + end + Octopus.using(:archenland) do + Cat.create(:name => "Aslan") + User.create(:name => "Shasta") + end + Octopus.using(:calormen) do + Cat.create(:name => "Aslan") + User.create(:name => "Tisroc") + end + Octopus.using(:telmar) do + Cat.create(:name => "Aslan") + User.create(:name => "Miraz") + end + + # All shards have the same Cat + expect( Octopus.using_all { Cat.where(name: "Aslan").first }.count).to eq(4) + + # Each shard has a unique User + expect(User.using(:narnia).first.name).to eq("Peter") + expect(User.using(:archenland).first.name).to eq("Shasta") + expect(User.using(:calormen).first.name).to eq("Tisroc") + expect(User.using(:telmar).first.name).to eq("Miraz") + end + end end From 1c5b3738b3b18c29fc5280e4c818a955ff3ba137 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 3 Jul 2019 10:00:05 -0500 Subject: [PATCH 26/41] Add db2 port/db --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ba702239..25ad13f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: DB2_HOST: db2 DB2_USER: db2inst1 DB2_PASSWORD: testpassword + DB2_PORT: 50000 + DB2_DATABASE: octopus6 depends_on: - mysql - postgres From 67dc84478a38eb84e9fb909c45884f5cabc17276 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 3 Jul 2019 10:00:24 -0500 Subject: [PATCH 27/41] Adding expect, in case I change up the entrypoint/cmd --- docker/db2/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/db2/Dockerfile b/docker/db2/Dockerfile index 8396f77b..c5a9f154 100644 --- a/docker/db2/Dockerfile +++ b/docker/db2/Dockerfile @@ -1,5 +1,7 @@ FROM ibmcom/db2express-c:latest +RUN yum makecache fast && yum install -y expect + SHELL ["/bin/bash", "-l", "-c"] COPY create_databases.sh / # Bake the Octopus databases into the DB2 image, From a96f52300958e61ae3ac7beb79514fe6a7a9f8d8 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 3 Jul 2019 10:00:51 -0500 Subject: [PATCH 28/41] Create 4 total dbs for shard/replication testing --- docker/db2/create_databases.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/db2/create_databases.sh b/docker/db2/create_databases.sh index baaf19e0..446ea492 100755 --- a/docker/db2/create_databases.sh +++ b/docker/db2/create_databases.sh @@ -2,5 +2,8 @@ set -e source /home/db2inst1/sqllib/db2profile db2start -echo "Creating databases, this may take around 6 - 8 minutes..." -time db2 create database octopus6 +echo "Creating databases, this may take around 6 - 8 minutes per database..." +for db in octopus6 octopus7 octopus8 octopus9; do + echo "create databse $db" + time db2 create database $db +done From 3dcd867aeb0fb247af4559c57952887dfc1c1ae4 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 3 Jul 2019 10:01:26 -0500 Subject: [PATCH 29/41] db:prepare on rails5, run tests against all Appraisals --- docker/startup.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docker/startup.sh b/docker/startup.sh index 4d13e590..bbb2dc41 100755 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -18,7 +18,9 @@ done >&2 echo "MySQL is ready" -until ! /opt/ibm/clidriver/bin/db2cli validate -database $DB2_DATABASE:$DB2_HOST:$DB2_PORT -connect -user $DB2_USERNAME -passwd $DB2_PASSWORD | grep -q FAILED ; do +echo /opt/ibm/clidriver/bin/db2cli validate -database $DB2_DATABASE:$DB2_HOST:$DB2_PORT -connect -user $DB2_USER -passwd $DB2_PASSWORD + +until ! /opt/ibm/clidriver/bin/db2cli validate -database $DB2_DATABASE:$DB2_HOST:$DB2_PORT -connect -user $DB2_USER -passwd $DB2_PASSWORD | grep -q FAILED ; do >&2 echo "DB2 is unavailable - sleeping" sleep 10 done @@ -26,16 +28,18 @@ done >&2 echo "DB2 is ready" set -x -cd /usr/src/ibm_db/IBM_DB_Adapter/ibm_db/ext -ruby extconf.rb -make cd /usr/src/app -bundle install -bundle exec rake db:prepare -bundle exec rake appraisal:install -bundle exec rake spec -#Not working yet -#./sample_app/script/ci_build +bundle install --path=.bundle +bundle exec appraisal install + +# The db migration only works on rails 5.0. +# See bug https://github.com/ibmdb/ruby-ibmdb/issues/31 +bundle exec appraisal rails5 rake db:prepare + +# Run the full spec across all rails versions. +# (This takes a while) +bundle exec rake appraisal spec + set +x echo Octopus is ready for you. Run \"docker-compose exec octopus /bin/bash\" /bin/sleep infinity From ea41105e8b4cad771521af204c7d4af0b7a35167 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 10:59:43 -0500 Subject: [PATCH 30/41] Skip migrations with DB2. Random bug in ibm_db. Running tests individually works, but running the whole migration_spec.rb file will recreate the issue _almost_ every time, in a random spec. Here's the full error and stack trace: Octopus::Migration should send the query to the correct shard Failure/Error: delegate :adapter_name, :add_transaction_record, :case_sensitive_modifier, :type_cast, :to_sql, :quote, :quote_column_name, :quote_table_name, :quote_table_name_for_assignment, :supports_migrations?, :table_alias_for, :table_exists?, :in_clause_length, :supports_ddl_transactions?, :sanitize_limit, :prefetch_primary_key?, :current_database, :combine_bind_parameters, :empty_insert_statement_value, :assume_migrated_upto_version, :schema_cache, :substitute_at, :internal_string_options_for_primary_key, :lookup_cast_type_from_column, :supports_advisory_locks?, :get_advisory_lock, :initialize_internal_metadata_table, :release_advisory_lock, :prepare_binds_for_database, :cacheable_query, :column_name_for_operation, :prepared_statements, :transaction_state, :create_table, to: :select_connection RuntimeError: An unexpected error occurred during retrieval of table metadata: uncaught throw :"Fetch Failure: " # /usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db/lib/active_record/connection_adapters/ibm_db_adapter.rb:1858:in `rescue in tables' # /usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db/lib/active_record/connection_adapters/ibm_db_adapter.rb:1845:in `tables' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/connection_adapters/abstract/schema_statements.rb:55:in `table_exists?' # ./lib/octopus/proxy.rb:25:in `table_exists?' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/schema_migration.rb:20:in `block in table_exists?' # ./gemfiles/.bundle/ruby/2.5.0/gems/activesupport-5.0.7.2/lib/active_support/deprecation/reporting.rb:36:in `silence' # ./gemfiles/.bundle/ruby/2.5.0/gems/activesupport-5.0.7.2/lib/active_support/deprecation/instance_delegator.rb:20:in `silence' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/schema_migration.rb:20:in `table_exists?' # /usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db/lib/active_record/connection_adapters/ibm_db_adapter.rb:48:in `create_table' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/connection_adapters/abstract/schema_statements.rb:1008:in `initialize_schema_migrations_table' # ./lib/octopus/proxy.rb:200:in `initialize_schema_migrations_table' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/migration.rb:1127:in `initialize' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/migration.rb:1017:in `new' # ./gemfiles/.bundle/ruby/2.5.0/gems/activerecord-5.0.7.2/lib/active_record/migration.rb:1017:in `run' # ./lib/octopus/migration.rb:187:in `block in run_with_octopus' # ./lib/octopus/proxy.rb:86:in `block (2 levels) in run_queries_on_shard' # ./lib/octopus/proxy.rb:345:in `using_shard' # ./lib/octopus/proxy.rb:85:in `block in run_queries_on_shard' # ./lib/octopus/proxy.rb:329:in `keeping_connection_proxy' # ./lib/octopus/proxy.rb:84:in `run_queries_on_shard' # ./lib/octopus/proxy.rb:93:in `block in send_queries_to_multiple_shards' # ./lib/octopus/proxy.rb:92:in `map' # ./lib/octopus/proxy.rb:92:in `send_queries_to_multiple_shards' # ./lib/octopus/migration.rb:186:in `run_with_octopus' # ./spec/support/octopus_helper.rb:46:in `migrate_to_version' # ./spec/support/octopus_helper.rb:37:in `migrating_to_version' # ./spec/octopus/migration_spec.rb:86:in `block (2 levels) in ' # ------------------ # --- Caused by: --- # UncaughtThrowError: # uncaught throw :"\xA3\xC9\x8E+H\xEA\xC9\vX\x82g\x8B\xAAU\x00\x00\bY\r\x8C\xAAU\x00\x00\x1A\xD0\xC1\xE2\xCF\xE9\x1Dd0\xB3m\x8B\xAAU\x00\x00\x80T\r\x8C\xAAU\x00\x00\x01\x99\xD1o," # /usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db/lib/active_record/connection_adapters/ibm_db_adapter.rb:1845:in `fetch_assoc' --- spec/octopus/migration_spec.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/spec/octopus/migration_spec.rb b/spec/octopus/migration_spec.rb index 999db255..79f84212 100644 --- a/spec/octopus/migration_spec.rb +++ b/spec/octopus/migration_spec.rb @@ -10,6 +10,18 @@ def get_all_versions end describe Octopus::Migration do + before(:each) do + # Strange bug with ibm_db adapter. Listing tables randomly raises + # this exception: + # + # An unexpected error occurred during retrieval of table metadata: uncaught throw :"Fetch Failure: " + # ruby-ibmdb/IBM_DB_Adapter/ibm_db/lib/active_record/connection_adapters/ibm_db_adapter.rb:1858:in `rescue in tables' + # + # See the full stack trace in git commit. + skip "DB2 bug with migrations (see comments)" if Octopus.ibm_db_support? + end + + it 'should run just in the master shard' do OctopusHelper.migrating_to_version 1 do expect(User.using(:master).find_by_name('Master')).not_to be_nil From 091f9ef81d1088a6e7b29a14c6ce70ce7407b05b Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 11:26:25 -0500 Subject: [PATCH 31/41] DB2 doesn't have boolean types --- spec/octopus/model_spec.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/spec/octopus/model_spec.rb b/spec/octopus/model_spec.rb index 5f827e52..f628f33f 100644 --- a/spec/octopus/model_spec.rb +++ b/spec/octopus/model_spec.rb @@ -373,14 +373,27 @@ u = User.using(:brazil).find_by_name('Teste') u.toggle(:admin) u.save - expect(User.using(:brazil).find_by_name('Teste').admin).to be true + # DB2 doesn't have a native boolean type, so this is 1 + # instead of true. (ibm_db hijacks all of ActiveRecord) + if Octopus.ibm_db_support? + expect(User.using(:brazil).find_by_name('Teste').admin).to be 1 + else + expect(User.using(:brazil).find_by_name('Teste').admin).to be true + end end it 'toggle!' do _ = User.using(:brazil).create!(:name => 'Teste', :admin => false) u = User.using(:brazil).find_by_name('Teste') u.toggle!(:admin) - expect(User.using(:brazil).find_by_name('Teste').admin).to be true + # DB2 doesn't have a native boolean type, so this is 1 + # instead of true (ibm_db hijacks all of ActiveRecord) + if Octopus.ibm_db_support? + expect(User.using(:brazil).find_by_name('Teste').admin).to be 1 + else + expect(User.using(:brazil).find_by_name('Teste').admin).to be true + end + end it 'count' do From 65d981d7174e93dd8ad097a6c0f631cb9a341e7f Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 11:35:32 -0500 Subject: [PATCH 32/41] Adding Appriasals just for DB2. This keeps the test suite going for most users that won't be using the ibm_db gem --- Appraisals | 13 +++++++++++-- docker/startup.sh | 2 +- gemfiles/rails42.gemfile | 1 - gemfiles/rails42_db2.gemfile | 8 ++++++++ gemfiles/rails5.gemfile | 2 +- gemfiles/rails50.gemfile | 8 ++++++++ gemfiles/rails50_db2.gemfile | 7 +++++++ 7 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 gemfiles/rails42_db2.gemfile create mode 100644 gemfiles/rails50.gemfile create mode 100644 gemfiles/rails50_db2.gemfile diff --git a/Appraisals b/Appraisals index ae909143..9cb88850 100644 --- a/Appraisals +++ b/Appraisals @@ -1,15 +1,24 @@ appraise "rails42" do gem "activerecord", "~> 4.2.0" +end + +appraise "rails42_db2" do + gem "activerecord", "~> 4.2.0" #Causes issue https://github.com/ibmdb/ruby-ibmdb/issues/31 # Wait for this PR to be accepted or pull in a local gem for yourself: # https://github.com/ibmdb/ruby-ibmdb/pull/38/files gem "ibm_db", "~> 3.0.5" end -appraise "rails5" do +appraise "rails50_db2" do + gem "activerecord", "~> 5.0.0" +end + +appraise "rails50" do gem "activerecord", "~> 5.0.0" # For now, ibm_db only works on rails 5.0 - gem "ibm_db", "~> 4.0.0" + #gem "ibm_db", "~> 4.0.0" + gem "ibm_db", path: "/usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db" end appraise "rails51" do diff --git a/docker/startup.sh b/docker/startup.sh index bbb2dc41..a1eee176 100755 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -34,7 +34,7 @@ bundle exec appraisal install # The db migration only works on rails 5.0. # See bug https://github.com/ibmdb/ruby-ibmdb/issues/31 -bundle exec appraisal rails5 rake db:prepare +bundle exec appraisal rails50_db2 rake db:prepare # Run the full spec across all rails versions. # (This takes a while) diff --git a/gemfiles/rails42.gemfile b/gemfiles/rails42.gemfile index e15d484b..9be86420 100644 --- a/gemfiles/rails42.gemfile +++ b/gemfiles/rails42.gemfile @@ -3,6 +3,5 @@ source "https://rubygems.org" gem "activerecord", "~> 4.2.0" -gem "ibm_db", "~> 3.0.5" gemspec path: "../" diff --git a/gemfiles/rails42_db2.gemfile b/gemfiles/rails42_db2.gemfile new file mode 100644 index 00000000..e15d484b --- /dev/null +++ b/gemfiles/rails42_db2.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activerecord", "~> 4.2.0" +gem "ibm_db", "~> 3.0.5" + +gemspec path: "../" diff --git a/gemfiles/rails5.gemfile b/gemfiles/rails5.gemfile index 6f6e2aa3..4da86a01 100644 --- a/gemfiles/rails5.gemfile +++ b/gemfiles/rails5.gemfile @@ -3,6 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 5.0.0" -gem "ibm_db", "~> 4.0.0" +gem "ibm_db", path: "/usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db" gemspec path: "../" diff --git a/gemfiles/rails50.gemfile b/gemfiles/rails50.gemfile new file mode 100644 index 00000000..4da86a01 --- /dev/null +++ b/gemfiles/rails50.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activerecord", "~> 5.0.0" +gem "ibm_db", path: "/usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db" + +gemspec path: "../" diff --git a/gemfiles/rails50_db2.gemfile b/gemfiles/rails50_db2.gemfile new file mode 100644 index 00000000..a7a9bb00 --- /dev/null +++ b/gemfiles/rails50_db2.gemfile @@ -0,0 +1,7 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activerecord", "~> 5.0.0" + +gemspec path: "../" From c63e2a0e32817a3f44dc58b1168032ea945e92d9 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 12:01:23 -0500 Subject: [PATCH 33/41] Ooops, wrong names --- Appraisals | 5 ++--- gemfiles/rails50.gemfile | 1 - gemfiles/rails50_db2.gemfile | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Appraisals b/Appraisals index 9cb88850..2ed0f363 100644 --- a/Appraisals +++ b/Appraisals @@ -12,13 +12,12 @@ end appraise "rails50_db2" do gem "activerecord", "~> 5.0.0" + # For now, ibm_db only works on rails 5.0 + gem "ibm_db", "~> 4.0.0" end appraise "rails50" do gem "activerecord", "~> 5.0.0" - # For now, ibm_db only works on rails 5.0 - #gem "ibm_db", "~> 4.0.0" - gem "ibm_db", path: "/usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db" end appraise "rails51" do diff --git a/gemfiles/rails50.gemfile b/gemfiles/rails50.gemfile index 4da86a01..a7a9bb00 100644 --- a/gemfiles/rails50.gemfile +++ b/gemfiles/rails50.gemfile @@ -3,6 +3,5 @@ source "https://rubygems.org" gem "activerecord", "~> 5.0.0" -gem "ibm_db", path: "/usr/src/ruby-ibmdb/IBM_DB_Adapter/ibm_db" gemspec path: "../" diff --git a/gemfiles/rails50_db2.gemfile b/gemfiles/rails50_db2.gemfile index a7a9bb00..6f6e2aa3 100644 --- a/gemfiles/rails50_db2.gemfile +++ b/gemfiles/rails50_db2.gemfile @@ -3,5 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 5.0.0" +gem "ibm_db", "~> 4.0.0" gemspec path: "../" From 86058e1dfc7397e77e94652a6e98f4cf8eebb00a Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 12:09:00 -0500 Subject: [PATCH 34/41] Boolean coersion works in rails 4.2 with DB2 --- spec/octopus/model_spec.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/octopus/model_spec.rb b/spec/octopus/model_spec.rb index f628f33f..1e2d8573 100644 --- a/spec/octopus/model_spec.rb +++ b/spec/octopus/model_spec.rb @@ -373,9 +373,9 @@ u = User.using(:brazil).find_by_name('Teste') u.toggle(:admin) u.save - # DB2 doesn't have a native boolean type, so this is 1 - # instead of true. (ibm_db hijacks all of ActiveRecord) - if Octopus.ibm_db_support? + # For some reason, boolean coersion works in rails 4.2 + # with DB2, but not rails 5.0 + if Octopus.ibm_db_support? && Octopus.rails50? expect(User.using(:brazil).find_by_name('Teste').admin).to be 1 else expect(User.using(:brazil).find_by_name('Teste').admin).to be true @@ -386,9 +386,9 @@ _ = User.using(:brazil).create!(:name => 'Teste', :admin => false) u = User.using(:brazil).find_by_name('Teste') u.toggle!(:admin) - # DB2 doesn't have a native boolean type, so this is 1 - # instead of true (ibm_db hijacks all of ActiveRecord) - if Octopus.ibm_db_support? + # For some reason, boolean coersion works in rails 4.2 + # with DB2, but not rails 5.0 + if Octopus.ibm_db_support? && Octopus.rails50? expect(User.using(:brazil).find_by_name('Teste').admin).to be 1 else expect(User.using(:brazil).find_by_name('Teste').admin).to be true From f241079b524b4a8c7a0dba2ef328c87b4b7192ad Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Thu, 18 Jul 2019 14:13:09 -0500 Subject: [PATCH 35/41] Issue with stock ibm_db 4.0.0 gem https://github.com/ibmdb/ruby-ibmdb/pull/89 Until that issue is officially resolved by IBM, I'm using my local fork for testing Octopus. --- Appraisals | 10 +++++++++- gemfiles/rails50_db2.gemfile | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Appraisals b/Appraisals index 2ed0f363..c7dc477a 100644 --- a/Appraisals +++ b/Appraisals @@ -13,7 +13,15 @@ end appraise "rails50_db2" do gem "activerecord", "~> 5.0.0" # For now, ibm_db only works on rails 5.0 - gem "ibm_db", "~> 4.0.0" + #gem "ibm_db", "~> 4.0.0" + # 4.0.0 breaks on empty_insert_statement_value issue: + # https://github.com/ibmdb/ruby-ibmdb/pull/89 + # I've merged this into my own branch for testing until + # IBM fixes this issue officially + gem "ibm_db", + git: "https://github.com/calh/ruby-ibmdb.git", + branch: "v4.0.1", + glob: "IBM_DB_Adapter/ibm_db/IBM_DB.gemspec" end appraise "rails50" do diff --git a/gemfiles/rails50_db2.gemfile b/gemfiles/rails50_db2.gemfile index 6f6e2aa3..83d842d5 100644 --- a/gemfiles/rails50_db2.gemfile +++ b/gemfiles/rails50_db2.gemfile @@ -3,6 +3,6 @@ source "https://rubygems.org" gem "activerecord", "~> 5.0.0" -gem "ibm_db", "~> 4.0.0" +gem "ibm_db", git: "https://github.com/calh/ruby-ibmdb.git", branch: "v4.0.1", glob: "IBM_DB_Adapter/ibm_db/IBM_DB.gemspec" gemspec path: "../" From 9a8b293fc46cf792a813bf053cba5034c2bd0db6 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 31 Jul 2019 12:36:48 -0500 Subject: [PATCH 36/41] Fix for DB2 hanging indefinitely on disconnect with Passenger This is an oddball bug, deep in the DB2 C library. Disconnecting from the database during the Passenger fork process causes it to hang indefinitely --- lib/octopus.rb | 5 +++++ lib/octopus/proxy.rb | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/octopus.rb b/lib/octopus.rb index 335693ca..2da2f615 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -133,6 +133,11 @@ def self.ibm_db_support? defined?(IBM_DB) && defined?(ActiveRecord::ConnectionAdapters::IBM_DBAdapter) end + # Are we running any version of Phusion Passenger? + def self.passenger? + defined?(PhusionPassenger) + end + attr_writer :logger def self.logger diff --git a/lib/octopus/proxy.rb b/lib/octopus/proxy.rb index b18f5ab1..cfbc909c 100644 --- a/lib/octopus/proxy.rb +++ b/lib/octopus/proxy.rb @@ -159,9 +159,13 @@ def clear_active_connections! end def clear_all_connections! - with_each_healthy_shard(&:disconnect!) + # Don't disconnect on DB2, the client is buggy and hangs sometimes + unless Octopus.ibm_db_support? + with_each_healthy_shard(&:disconnect!) + end - if Octopus.atleast_rails52? + # If we're running on Passenger or > Rails 5.2, reconnect + if Octopus.passenger? || Octopus.atleast_rails52? # On Rails 5.2 it is no longer safe to re-use connection pools after they have been discarded # This happens on webservers with forking, for example Phusion Passenger. # Therefor after we clear all connections we reinitialize the shards to get fresh and not discarded ConnectionPool objects From 8b59e8326c95627d62386fcb7ddb183fd6aad510 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 31 Jul 2019 13:41:42 -0500 Subject: [PATCH 37/41] Better condition for skipping disconnects on DB2 --- lib/octopus/proxy.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/octopus/proxy.rb b/lib/octopus/proxy.rb index cfbc909c..606f6e7f 100644 --- a/lib/octopus/proxy.rb +++ b/lib/octopus/proxy.rb @@ -160,9 +160,7 @@ def clear_active_connections! def clear_all_connections! # Don't disconnect on DB2, the client is buggy and hangs sometimes - unless Octopus.ibm_db_support? - with_each_healthy_shard(&:disconnect!) - end + with_each_healthy_shard { |v| v.disconnect! if v.connection.config["adapter"] != "ibm_db" } # If we're running on Passenger or > Rails 5.2, reconnect if Octopus.passenger? || Octopus.atleast_rails52? From 98de909d9801a48560a074723971487cc8faa6c1 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 31 Jul 2019 16:14:01 -0500 Subject: [PATCH 38/41] Better cross-adapter method of checking the db type --- lib/octopus/proxy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/octopus/proxy.rb b/lib/octopus/proxy.rb index 606f6e7f..69dea500 100644 --- a/lib/octopus/proxy.rb +++ b/lib/octopus/proxy.rb @@ -160,7 +160,7 @@ def clear_active_connections! def clear_all_connections! # Don't disconnect on DB2, the client is buggy and hangs sometimes - with_each_healthy_shard { |v| v.disconnect! if v.connection.config["adapter"] != "ibm_db" } + with_each_healthy_shard { |v| v.disconnect! if v.spec.config["adapter"] != "ibm_db" } # If we're running on Passenger or > Rails 5.2, reconnect if Octopus.passenger? || Octopus.atleast_rails52? From 92b4cafe6c47e84237d39ac1f50e084f28244c06 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Mon, 5 Aug 2019 12:01:17 -0500 Subject: [PATCH 39/41] Skip the connected? test with DB2 Since we don't disconnect from DB2, this spec fails. --- spec/octopus/proxy_spec.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/octopus/proxy_spec.rb b/spec/octopus/proxy_spec.rb index 6dc15277..b7b88ec5 100644 --- a/spec/octopus/proxy_spec.rb +++ b/spec/octopus/proxy_spec.rb @@ -291,6 +291,9 @@ end it 'is consistent with connected?' do + if Octopus.ibm_db_support? + skip "DB2 bug/workaround with disconnects causes this test to fail" + end expect(Item.connected?).to be true expect(ActiveRecord::Base.connected?).to be true From a4d2a068d103f3e646e80202ce7b0608b4af2a0b Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 28 Aug 2019 14:36:25 -0500 Subject: [PATCH 40/41] Add a per-environment master_shard option In many setups, the default master shard might be a different database server in each environment. A single `master_shard` config setting doesn't fit. This patch first checks the shards.yml for `octopus` => Rails env => master_shard for a setting, and still falls back to `octopus` => `master_shard` if it doesn't exist --- lib/octopus.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/octopus.rb b/lib/octopus.rb index 2da2f615..7b7a0098 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -44,7 +44,7 @@ def self.load_balancer end def self.master_shard - ((config && config[:master_shard]) || :master).to_sym + ((config && ( config[rails_env][:master_shard] || config[:master_shard] )) || :master).to_sym end # Public: Whether or not Octopus is configured and should hook into the From 1d0722031e349f5fd09a47a91609c5ad76c2d6d1 Mon Sep 17 00:00:00 2001 From: Cal Heldenbrand Date: Wed, 28 Aug 2019 15:05:13 -0500 Subject: [PATCH 41/41] Better fix for per-environment master shards I cowboyed up a bit too fast on the first fix. This moves the per-environment master_shard selection to its own yaml key --- lib/octopus.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/octopus.rb b/lib/octopus.rb index 7b7a0098..2c7081e1 100644 --- a/lib/octopus.rb +++ b/lib/octopus.rb @@ -44,7 +44,15 @@ def self.load_balancer end def self.master_shard - ((config && ( config[rails_env][:master_shard] || config[:master_shard] )) || :master).to_sym + shard = :master + return shard unless config + + if config[:master_shards] && config[:master_shards][rails_env] + shard = config[:master_shards][rails_env].to_sym + elsif config[:master_shard] + shard = config[:master_shard].to_sym + end + shard end # Public: Whether or not Octopus is configured and should hook into the