From b1337b903b139b720a1141142cacb7ad146be704 Mon Sep 17 00:00:00 2001 From: GitLab Date: Sat, 2 Aug 2014 23:44:31 +0200 Subject: [PATCH 1/7] Supports multiple runners in dedicated directories --- bin/runner | 2 +- lib/build.rb | 5 +++-- lib/config.rb | 5 +++-- lib/network.rb | 8 ++++++-- lib/runner.rb | 15 ++++++++------- lib/setup.rb | 2 +- lib/support/init.d/gitlab_ci_runner | 18 +++++++++--------- 7 files changed, 31 insertions(+), 24 deletions(-) mode change 100644 => 100755 lib/support/init.d/gitlab_ci_runner diff --git a/bin/runner b/bin/runner index 0698591..c80151c 100755 --- a/bin/runner +++ b/bin/runner @@ -4,5 +4,5 @@ require 'bundler/setup' require_relative '../lib/runner' -GitlabCi::Runner.new +GitlabCi::Runner.new(ARGV.fetch(0, '0')) exit diff --git a/lib/build.rb b/lib/build.rb index 4788187..1a4a038 100644 --- a/lib/build.rb +++ b/lib/build.rb @@ -13,7 +13,8 @@ class Build attr_accessor :id, :commands, :ref, :tmp_file_path, :output, :before_sha, :run_at, :post_message - def initialize(data) + def initialize(runner, data) + @runner = runner @output = "" @post_message = "" @commands = data[:commands].to_a @@ -182,7 +183,7 @@ def repo_exists? end def config - @config ||= Config.new + @config ||= Config.new(@runner) end def project_dir diff --git a/lib/config.rb b/lib/config.rb index eff6334..25fe204 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -6,7 +6,8 @@ module GitlabCi class Config attr_reader :config - def initialize + def initialize(runner) + @runner = runner if File.exists?(config_path) @config = YAML.load_file(config_path) else @@ -23,7 +24,7 @@ def url end def builds_dir - @builds_path ||= File.join(ROOT_PATH, 'tmp', 'builds') + @builds_path ||= File.join(ROOT_PATH, 'tmp', 'builds', 'runner-' + @runner.to_s) end def write(key, value) diff --git a/lib/network.rb b/lib/network.rb index 8bdda8a..a8ed071 100644 --- a/lib/network.rb +++ b/lib/network.rb @@ -7,6 +7,10 @@ module GitlabCi class Network include HTTParty + def initialize(runner) + @runner = runner + end + # check for available build from coordinator # and pick a pending one # { @@ -95,7 +99,7 @@ def register_runner(token) private def broadcast message - print "#{Time.now.to_s} | #{message}" + print "#{Time.now.to_s} | #{@runner} | #{message}" end def api_url @@ -107,7 +111,7 @@ def token end def config - @config ||= Config.new + @config ||= Config.new(@runner) end def default_options diff --git a/lib/runner.rb b/lib/runner.rb index 0430164..7a9ce0b 100644 --- a/lib/runner.rb +++ b/lib/runner.rb @@ -5,8 +5,9 @@ module GitlabCi class Runner attr_accessor :current_build, :thread - def initialize - puts '* Gitlab CI Runner started' + def initialize(runner) + @runner = runner + puts "* Gitlab CI Runner #{@runner} started" puts '* Waiting for builds' loop do if running? @@ -34,7 +35,7 @@ def abort_if_timeout def update_build return unless @current_build.completed? - puts "#{Time.now.to_s} | Completed build #{@current_build.id}, #{@current_build.state}." + puts "#{Time.now.to_s} | Completed build #{@current_build.id}, #{@current_build.state} on runner #{@runner}." # Make sure we push latest build info submitted # before we clean build @@ -70,14 +71,14 @@ def get_build end def network - @network ||= Network.new + @network ||= Network.new(@runner) end def run(build_data) - @current_build = GitlabCi::Build.new(build_data) - puts "#{Time.now.to_s} | Starting new build #{@current_build.id}..." + @current_build = GitlabCi::Build.new(@runner, build_data) + puts "#{Time.now.to_s} | Starting new build #{@current_build.id} with runner #{@runner}..." @current_build.run - puts "#{Time.now.to_s} | Build #{@current_build.id} started." + puts "#{Time.now.to_s} | Build #{@current_build.id} on runner #{@runner} started." end def collect_trace diff --git a/lib/setup.rb b/lib/setup.rb index 910d19c..40e1073 100644 --- a/lib/setup.rb +++ b/lib/setup.rb @@ -47,7 +47,7 @@ def register_runner def write_token(token) puts "Runner token: #{token}" - Config.new.write('token', token) + Config.new(0).write('token', token) end end end diff --git a/lib/support/init.d/gitlab_ci_runner b/lib/support/init.d/gitlab_ci_runner old mode 100644 new mode 100755 index a8a2492..75ebd77 --- a/lib/support/init.d/gitlab_ci_runner +++ b/lib/support/init.d/gitlab_ci_runner @@ -2,26 +2,26 @@ ### BEGIN INIT INFO # Provides: gitlab-ci-runner -# Required-Start: $local_fs $remote_fs $network $syslog -# Required-Stop: $local_fs $remote_fs $network $syslog +# Required-Start: $local_fs $remote_fs $network $syslog gitlab_ci +# Required-Stop: $local_fs $remote_fs $network $syslog gitlab_ci # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: GitLab CI Runner init.d script # Description: Enables automatic start of runners at boot time in the background. ### END INIT INFO -APP_ROOT="/home/gitlab_ci_runner/gitlab-ci-runner" -APP_USER="gitlab_ci_runner" +APP_ROOT="/home/git/gitlab-ci-runner" +APP_USER="git" PID_PATH="$APP_ROOT/tmp/pids" PROCESS_NAME="ruby ./bin/runner" RUNNERS_PID="$PID_PATH/runners.pid" -RUNNERS_NUM=1 # number of runners to spawn +RUNNERS_NUM=10 # number of runners to spawn START_RUNNER="nohup bundle exec ./bin/runner" NAME="gitlab-ci-runner" DESC="GitLab CI runner" RUNNERS_REGISTERED=0 RUNNERS_RUNNING=0 -INIT_LOG="/var/log/gitlab_ci_runner.log" +INIT_LOG="$APP_ROOT/log/gitlab_ci_runner.log" check_pid() { # Number of registered runners in PID file @@ -60,7 +60,7 @@ start() { chown $APP_USER $INIT_LOG fi echo "Starting runner #$i" - execute "$START_RUNNER >> $INIT_LOG 2>&1 & echo \$! >> $RUNNERS_PID" + execute "$START_RUNNER $i >> $INIT_LOG 2>&1 & echo \$! >> $RUNNERS_PID" done echo "SUCCESS: Started $RUNNERS_NUM $DESC(s)." fi @@ -86,7 +86,7 @@ stop() { echo -n "Trying to stop registered runners..." if [ $RUNNERS_REGISTERED -gt 0 ]; then - execute "cat $RUNNERS_PID | xargs kill -USR2" + execute "cat $RUNNERS_PID | xargs kill -9" rm -f $RUNNERS_PID echo "OK" else @@ -96,7 +96,7 @@ stop() { if [ $KILL_GHOSTS -eq 1 ]; then echo -ne "Trying to kill ghost runners..." - ps -C "$PROCESS_NAME" -o "%p" h | xargs kill -USR2 + ps -C "$PROCESS_NAME" -o "%p" h | xargs kill -9 [ $? -eq 0 ] && echo "OK" else echo "No ghost runners have been found.This is good." From 46c330513435b5e69fdb3fe8fd3ffb2550d04ebc Mon Sep 17 00:00:00 2001 From: GitLab Date: Sat, 2 Aug 2014 23:52:24 +0200 Subject: [PATCH 2/7] Reverts changes in init.d startup script --- lib/support/init.d/gitlab_ci_runner | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) mode change 100755 => 100644 lib/support/init.d/gitlab_ci_runner diff --git a/lib/support/init.d/gitlab_ci_runner b/lib/support/init.d/gitlab_ci_runner old mode 100755 new mode 100644 index 75ebd77..a8a2492 --- a/lib/support/init.d/gitlab_ci_runner +++ b/lib/support/init.d/gitlab_ci_runner @@ -2,26 +2,26 @@ ### BEGIN INIT INFO # Provides: gitlab-ci-runner -# Required-Start: $local_fs $remote_fs $network $syslog gitlab_ci -# Required-Stop: $local_fs $remote_fs $network $syslog gitlab_ci +# Required-Start: $local_fs $remote_fs $network $syslog +# Required-Stop: $local_fs $remote_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: GitLab CI Runner init.d script # Description: Enables automatic start of runners at boot time in the background. ### END INIT INFO -APP_ROOT="/home/git/gitlab-ci-runner" -APP_USER="git" +APP_ROOT="/home/gitlab_ci_runner/gitlab-ci-runner" +APP_USER="gitlab_ci_runner" PID_PATH="$APP_ROOT/tmp/pids" PROCESS_NAME="ruby ./bin/runner" RUNNERS_PID="$PID_PATH/runners.pid" -RUNNERS_NUM=10 # number of runners to spawn +RUNNERS_NUM=1 # number of runners to spawn START_RUNNER="nohup bundle exec ./bin/runner" NAME="gitlab-ci-runner" DESC="GitLab CI runner" RUNNERS_REGISTERED=0 RUNNERS_RUNNING=0 -INIT_LOG="$APP_ROOT/log/gitlab_ci_runner.log" +INIT_LOG="/var/log/gitlab_ci_runner.log" check_pid() { # Number of registered runners in PID file @@ -60,7 +60,7 @@ start() { chown $APP_USER $INIT_LOG fi echo "Starting runner #$i" - execute "$START_RUNNER $i >> $INIT_LOG 2>&1 & echo \$! >> $RUNNERS_PID" + execute "$START_RUNNER >> $INIT_LOG 2>&1 & echo \$! >> $RUNNERS_PID" done echo "SUCCESS: Started $RUNNERS_NUM $DESC(s)." fi @@ -86,7 +86,7 @@ stop() { echo -n "Trying to stop registered runners..." if [ $RUNNERS_REGISTERED -gt 0 ]; then - execute "cat $RUNNERS_PID | xargs kill -9" + execute "cat $RUNNERS_PID | xargs kill -USR2" rm -f $RUNNERS_PID echo "OK" else @@ -96,7 +96,7 @@ stop() { if [ $KILL_GHOSTS -eq 1 ]; then echo -ne "Trying to kill ghost runners..." - ps -C "$PROCESS_NAME" -o "%p" h | xargs kill -9 + ps -C "$PROCESS_NAME" -o "%p" h | xargs kill -USR2 [ $? -eq 0 ] && echo "OK" else echo "No ghost runners have been found.This is good." From 584b8ccb4881c661ff5915969584c1498a5b82f4 Mon Sep 17 00:00:00 2001 From: GitLab Date: Sat, 2 Aug 2014 23:59:54 +0200 Subject: [PATCH 3/7] Fixes build spec --- spec/build_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/build_spec.rb b/spec/build_spec.rb index 0d4fc67..b302ada 100644 --- a/spec/build_spec.rb +++ b/spec/build_spec.rb @@ -3,7 +3,7 @@ describe 'Build' do describe :run do - let(:build) { GitlabCi::Build.new(build_data) } + let(:build) { GitlabCi::Build.new(0, build_data) } before do build.run From 106aaf4f4b8cb755bebc956488f21485181fa46e Mon Sep 17 00:00:00 2001 From: Marek Date: Sun, 3 Aug 2014 00:05:47 +0200 Subject: [PATCH 4/7] Update build_spec.rb --- spec/build_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/build_spec.rb b/spec/build_spec.rb index b302ada..f3252da 100644 --- a/spec/build_spec.rb +++ b/spec/build_spec.rb @@ -27,3 +27,4 @@ def build_data } end end + From 75a36c6290cf3e09701d61f2919d4ec548766938 Mon Sep 17 00:00:00 2001 From: GitLab Date: Sun, 3 Aug 2014 13:08:56 +0200 Subject: [PATCH 5/7] Fixes init.d start script --- lib/support/init.d/gitlab_ci_runner | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/support/init.d/gitlab_ci_runner b/lib/support/init.d/gitlab_ci_runner index a8a2492..289ff1a 100644 --- a/lib/support/init.d/gitlab_ci_runner +++ b/lib/support/init.d/gitlab_ci_runner @@ -21,7 +21,8 @@ NAME="gitlab-ci-runner" DESC="GitLab CI runner" RUNNERS_REGISTERED=0 RUNNERS_RUNNING=0 -INIT_LOG="/var/log/gitlab_ci_runner.log" +INIT_LOG='/var/log/gitlab_ci_runner.log' +#INIT_LOG='/var/log/gitlab_ci_runner_$RUNNER.log' check_pid() { # Number of registered runners in PID file @@ -54,13 +55,16 @@ start() { # Spawn runners for (( i=1; i<=$RUNNERS_NUM; i++ )) do + # eval is required for making dedicated logs for each logger. Then use $RUNNER substitution in INIT_LOG variable + RUNNER=$((i-1)) + eval "LOGFILE=\"$INIT_LOG\"" # Check log file - if [ ! -f $INIT_LOG ]; then - touch $INIT_LOG - chown $APP_USER $INIT_LOG + if [ ! -f $LOGFILE ]; then + touch $LOGFILE + chown $APP_USER $LOGFILE fi echo "Starting runner #$i" - execute "$START_RUNNER >> $INIT_LOG 2>&1 & echo \$! >> $RUNNERS_PID" + execute "$START_RUNNER $i >> $LOGFILE 2>&1 & echo \$! >> $RUNNERS_PID" done echo "SUCCESS: Started $RUNNERS_NUM $DESC(s)." fi @@ -99,7 +103,7 @@ stop() { ps -C "$PROCESS_NAME" -o "%p" h | xargs kill -USR2 [ $? -eq 0 ] && echo "OK" else - echo "No ghost runners have been found.This is good." + echo "No ghost runners have been found. This is good." fi } From 031f0983f3b10bcab528c3340037672e99b8d805 Mon Sep 17 00:00:00 2001 From: GitLab Date: Sun, 18 Jan 2015 23:02:32 +0100 Subject: [PATCH 6/7] Converts single quotes to double --- lib/config.rb | 2 +- lib/setup.rb | 2 +- spec/build_spec.rb | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index b64488b..86a66a0 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -27,7 +27,7 @@ def url end def builds_dir - @builds_path ||= File.join($root_path, 'tmp', 'builds', 'runner-' + @runner.to_s) + @builds_path ||= File.join($root_path, "tmp", "builds", "runner-" + @runner.to_s) end def write(key, value) diff --git a/lib/setup.rb b/lib/setup.rb index 40e1073..fd0b7cc 100644 --- a/lib/setup.rb +++ b/lib/setup.rb @@ -47,7 +47,7 @@ def register_runner def write_token(token) puts "Runner token: #{token}" - Config.new(0).write('token', token) + Config.new(0).write("token", token) end end end diff --git a/spec/build_spec.rb b/spec/build_spec.rb index f3252da..b302ada 100644 --- a/spec/build_spec.rb +++ b/spec/build_spec.rb @@ -27,4 +27,3 @@ def build_data } end end - From b7bc8ba1ac7983b7726d937ddb34028495f30760 Mon Sep 17 00:00:00 2001 From: GitLab Date: Sun, 18 Jan 2015 23:13:31 +0100 Subject: [PATCH 7/7] Fixes long line --- lib/config.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/config.rb b/lib/config.rb index 86a66a0..2f72f79 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -27,7 +27,8 @@ def url end def builds_dir - @builds_path ||= File.join($root_path, "tmp", "builds", "runner-" + @runner.to_s) + rid = "runner-" + @runner.to_s + @builds_path ||= File.join($root_path, "tmp", "builds", rid) end def write(key, value)