diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 74ea48ffa..62edc6672 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -34,6 +34,7 @@ Contributors * Nick Fagerlund * Nick Lewis * Nigel Kersten +* Pascal Vibet * Patrick Carlisle * Paul Berry * Peter Meier diff --git a/Gemfile b/Gemfile index 5e807422e..41157da73 100644 --- a/Gemfile +++ b/Gemfile @@ -45,3 +45,7 @@ end group :mysql do gem 'mysql2', '~> 0.3.11' end + +group :sqlite do + gem 'sqlite3' +end diff --git a/README.markdown b/README.markdown index 7c310659f..539b00de4 100644 --- a/README.markdown +++ b/README.markdown @@ -38,6 +38,12 @@ precompile assets for production using: * `RAILS_ENV=production bundle exec rake assets:precompile` +Dashboard is an front-end of puppet, so it keep all reports. If infrastructure is big, puppetdashboard database become very large (more than 50Gio). +So add periodicaly 2 tasks: + + * `RAILS_ENV=production bundle exec rake reports:prune upto=20 unit=day` + * `RAILS_ENV=production bundle exec rake db:raw:optimize` + Contributing ------------ diff --git a/ext/debian/conf/database.yml.example b/ext/debian/conf/database.yml.example new file mode 100644 index 000000000..a02a9a28c --- /dev/null +++ b/ext/debian/conf/database.yml.example @@ -0,0 +1,66 @@ +# config/database.yml +# +# IMPORTANT NOTE: Before starting Dashboard, you will need to ensure that the +# MySQL user and databases you've specified in this file exist, and that the +# user has all permissions on the relevant databases. This will have to be done +# with an external database administration tool. If using the command-line +# `mysql` client, the commands to do this will resemble the following: +# +# CREATE DATABASE dashboard_production CHARACTER SET utf8; +# CREATE USER 'dashboard'@'localhost' IDENTIFIED BY 'my_password'; +# GRANT ALL PRIVILEGES ON dashboard_production.* TO 'dashboard'@'localhost'; +# +# ----- +# +# This file should be a YAML hash with one key for each of the standard Rails +# environments: production, development, and test. +# +# - The "production" environment gives the best performance, and should be used +# most of the time by most users. **Note that Rails does not consider +# production its default environment, and you must specify it manually with +# the RAILS_ENV environment variable when running any rake tasks.** +# - The "development" environment gives worse performance, but yields better +# logging and error reporting when something goes wrong. +# - The "test" environment is only used for running Dashboard's automated tests. +# It should never be used by most users. If you are using the test +# environment, **DO NOT set its database to the same database used by the +# development or production environments,** as it will be erased and +# re-written every time the automated tests are run. +# +# The environment is set when Dashboard is started, a console is started, or a +# rake task is run. Most production-quality Rails servers (such as Passenger) +# will default to the production environment, but the included WEBrick server +# script will default to development unless run with the `-e production` option. +# +# Each environment should be a hash with keys for: +# +# - database +# - username +# - password +# - encoding +# - adapter +# +# At the moment, "adapter" can be "mysql2" or "postgresql", and "encoding" should always +# be "utf8". +# +production: + database: dashboard_production + username: dashboard + password: + encoding: utf8 + adapter: postgresql + +development: + database: dashboard_development + username: dashboard + password: + encoding: utf8 + adapter: postgresql + +test: + database: dashboard_test + username: dashboard + password: + encoding: utf8 + adapter: postgresql + diff --git a/ext/debian/conf/settings.yml.example b/ext/debian/conf/settings.yml.example new file mode 100644 index 000000000..dc928d983 --- /dev/null +++ b/ext/debian/conf/settings.yml.example @@ -0,0 +1,108 @@ +#===[ Settings ]========================================================= +# +# This file is meant for storing setting information that is never +# published or committed to a revision control system. +# +# Do not modify this "config/settings.yml.example" file directly -- you +# should copy it to "config/settings.yml" and customize it there. +# +#---[ Values ]---------------------------------------------------------- + +# Node name to use when contacting the puppet master. This is the +# CN that is used in Dashboard's certificate. +cn_name: 'dashboard' + +ca_crl_path: 'certs/dashboard.ca_crl.pem' + +ca_certificate_path: 'certs/dashboard.ca_cert.pem' + +certificate_path: 'certs/dashboard.cert.pem' + +private_key_path: 'certs/dashboard.private_key.pem' + +public_key_path: 'certs/dashboard.public_key.pem' + +# Hostname of the certificate authority. +ca_server: 'puppet' + +# Port for the certificate authority. +ca_port: 8140 + +# Key length for SSL certificates +key_length: 1024 + +# The "inventory service" allows you to connect to a puppet master to retrieve and node facts +enable_inventory_service: false + +# Hostname of the inventory server. +inventory_server: 'puppet' + +# Port for the inventory server. +inventory_port: 8140 + +# Set this to true to allow Dashboard to display diffs on files that +# are archived in the file bucket. +use_file_bucket_diffs: false + +# Hostname of the file bucket server. +file_bucket_server: 'puppet' + +# Port for the file bucket server. +file_bucket_port: 8140 + +# Amount of time in seconds since last report before a node is considered no longer reporting +no_longer_reporting_cutoff: 3600 + +# How many days of history to display on the "Daily Run Status" graph +daily_run_history_length: 30 + +use_external_node_classification: true + +# Uncomment the following line to set a local time zone. Run +# "rake time:zones:local" for the name of your local time zone. +#time_zone: 'Pacific Time (US & Canada)' + +# Look at http://ruby-doc.org/core/classes/Time.html#M000298 for the strftime formatting +datetime_format: '%Y-%m-%d %H:%M %Z' +date_format: '%A, %B %e, %Y' + +# Set this to the URL of an image. The image will be scaled to the specified dimensions. +custom_logo_url: '/images/dashboard_logo.png' +custom_logo_width: 155px +custom_logo_height: 23px +custom_logo_alt_text: 'Puppet Dashboard' + +# We will be deprecating using "http://dashboard_servername/reports" as the puppet master's reporturl. +# Set this to 'true' once you have changed all your puppet masters to send reports to +# "http://dashboard_servername/reports/upload" +disable_legacy_report_upload_url: false + +# Set this to true to prevent storing details about not-changed events +# which saves a tremendous amount of space in the resource_statuses table. +disable_report_unchanged_events: false + +# Disables the UI and controller actions for editing nodes, classes, groups and reports. Report submission is still allowed +enable_read_only_mode: false + +# Default number of items of each kind to display per page +nodes_per_page: 20 +classes_per_page: 50 +groups_per_page: 50 +reports_per_page: 20 + +# Previously Dashboard used only ID numbers as URL slugs, set to true to return to that style. +numeric_url_slugs: false + +# Adds optional custom links in the top bar of puppet dashboard. +# Exemple that will add two links: +#custom_links: +# - +# href: /link/href +# title: The title that will be displayed +# - +# href: /node_groups/1 +# title: Special group + +# Generate a new secret_token for production use! +secret_token: 'b1bbd28f6f9ebfc25f09da9bff4643f75d5ad8c8da8b60234168c81cd6d32c15e5e5421196ee99da248d37d84f73c9ecb38608fc0d8b2709872290a3f43b244e' +#===[ fin ]============================================================= diff --git a/ext/debian/control b/ext/debian/control index bfc9e95ea..6f2720e59 100644 --- a/ext/debian/control +++ b/ext/debian/control @@ -10,9 +10,9 @@ Homepage: http://github.com/puppetlabs/puppet-dashboard Package: puppet-dashboard Architecture: all Pre-Depends: debconf -Depends: ruby, ruby1.8 (>= 1.8.7), rake (>=0.8.3), dbconfig-common, libdbd-mysql-ruby, mysql-client, rubygems, libhttpclient-ruby1.8 -Suggests: mysql-server -Recommends: libapache2-mod-passenger +Depends: ruby, deconf-common, rubygems, lighttpclient-ruby1.8, bundler, g++, ruby1.9.1-dev, make, libxml2, zliblg-dev, libxml2-dev, patch, libxslt-dev, libpq-dev, libsqlite2-dev, nodejs +Suggests: mysql-server, lidbd-mysql-ruby, mysql-client, ruby-mysql2, libmysql-ruby, libmysqlclient-dev +Recommends: apache2|nginx Description: Dashboard for Puppet The Puppet Dashboard is a web application for managing the Puppet configuration management tool. diff --git a/ext/debian/dirs b/ext/debian/dirs index 16ecf34e7..bc8036d45 100644 --- a/ext/debian/dirs +++ b/ext/debian/dirs @@ -1,4 +1,7 @@ usr/share/puppet-dashboard/log usr/share/puppet-dashboard/tmp usr/share/puppet-dashboard/spool +var/log/puppet-dashboard +var/spool/puppet-dashboard +var/tmp/puppet-dashboard diff --git a/ext/debian/postinst b/ext/debian/postinst index 02bcb2974..ce1b8686a 100644 --- a/ext/debian/postinst +++ b/ext/debian/postinst @@ -14,25 +14,26 @@ mv_conffile() { case "$1" in configure) - for item in config/environment.rb public log tmp spool; do - chown -R www-data:www-data /usr/share/puppet-dashboard/${item} + for item in config/environment.rb public; do + chown -R root:www-data /usr/share/puppet-dashboard/${item} + chmod 0640 /usr/share/puppet-dashboard/${item} done - if [ ! -d /etc/puppet-dashboard ]; then - mkdir /etc/puppet-dashboard - fi + ln -sf /var/log/puppet-dashboard /usr/share/puppet-dashboard/log + ln -sf /var/cache/puppet-dashboard /usr/share/puppet-dashboard/tmp/cache + ln -s /var/lib/puppet-dashboard/* /usr/share/puppet-dashboard/tmp/ + ln -sf /var/run/puppet /usr/share/puppet-dashboard/tmp/pids - if [ -f /usr/share/puppet-dashboard/config/database.yml ] && [ ! -e /etc/puppet-dashboard/database.yml ]; then + if [ -f /usr/share/puppet-dashboard/config/database.yml ]; then mv_conffile "/usr/share/puppet-dashboard/config/database.yml" "/etc/puppet-dashboard/database.yml" fi - if [ -f /usr/share/puppet-dashboard/config/settings.yml ] && [ ! -e /etc/puppet-dashboard/settings.yml ]; then + if [ -f /usr/share/puppet-dashboard/config/settings.yml ]; then mv_conffile "/usr/share/puppet-dashboard/config/settings.yml" "/etc/puppet-dashboard/settings.yml" fi - chown -R www-data:www-data /etc/puppet-dashboard - chmod -R o-rwx /etc/puppet-dashboard - find /etc/puppet-dashboard -type f -exec chmod a-x {} \; + chown -R root:www-data /etc/puppet-dashboard/*.yml + chmod -R o-rwx /etc/puppet-dashboard/*.yml if [ ! -h /usr/share/puppet-dashboard/config/database.yml ]; then ln -s /etc/puppet-dashboard/database.yml /usr/share/puppet-dashboard/config/database.yml @@ -42,6 +43,10 @@ configure) ln -s /etc/puppet-dashboard/settings.yml /usr/share/puppet-dashboard/config/settings.yml fi + if [ ! -f /etc/nginx/sites-enabled/puppet-dashboard ]; then + cp /usr/share/puppet-dashboard/config/puppet-dashboard.nginx /etc/nginx/sites-available/puppet-dashboard + ln -s /etc/nginx/sites-available/puppet-dashboard /etc/nginx/sites-enabled/puppet-dashboard + fi esac #DEBHELPER# diff --git a/ext/debian/puppet-dashboard-workers.init b/ext/debian/puppet-dashboard-workers.init index 13826fb70..ed0f1f67d 100644 --- a/ext/debian/puppet-dashboard-workers.init +++ b/ext/debian/puppet-dashboard-workers.init @@ -2,7 +2,7 @@ ### BEGIN INIT INFO # Provides: puppet-dashboard-workers # Required-Start: $local_fs $remote_fs $network $syslog -# Required-Stop: $local_fs $remote_fs $network $syslog +# Required-Stop: $local_fs $remote_fs $network $syslog puppet-dashboard # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # X-Interactive: true @@ -28,6 +28,7 @@ DASHBOARD_RUBY=${DASHBOARD_RUBY:-/usr/bin/ruby} DASHBOARD_ENVIRONMENT=${DASHBOARD_ENVIRONMENT:-production} DASHBOARD_IFACE=${DASHBOARD_IFACE:-0.0.0.0} DASHBOARD_PORT=${DASHBOARD_PORT:-3000} +DASHBOARD_PID=${DASHBOARD_PID:-/var/run/puppet-dashboard} check_dashboard_enabled_option() { if [ ! "$START" = "yes" ]; then @@ -40,7 +41,7 @@ check_dashboard_enabled_option() { check_puppet_dashboard_worker_status() { RETVAL=1 - for pidfile in $(ls -1 "${DASHBOARD_HOME}"/tmp/pids/delayed_job*.pid 2>/dev/null) + for pidfile in $(ls -1 ${DASHBOARD_PID}/delayed_job*.pid 2>/dev/null) do status_of_proc -p $pidfile ${DASHBOARD_RUBY} "Puppet Dashboard Worker (pid $(cat $pidfile))" || return $? RETVAL=$? @@ -53,7 +54,7 @@ check_puppet_dashboard_worker_status() { stop_puppet_dashboard_workers() { RETVAL=0 - for pidfile in $(ls -1 "${DASHBOARD_HOME}"/tmp/pids/delayed_job*.pid 2>/dev/null) + for pidfile in $(ls -1 ${DASHBOARD_PID}/delayed_job*.pid 2>/dev/null) do start-stop-daemon --stop --quiet --oknodo --pidfile $pidfile --retry 10 ((RETVAL=RETVAL+$?)) diff --git a/ext/debian/default b/ext/debian/puppet-dashboard.default similarity index 100% rename from ext/debian/default rename to ext/debian/puppet-dashboard.default diff --git a/ext/debian/init b/ext/debian/puppet-dashboard.init similarity index 96% rename from ext/debian/init rename to ext/debian/puppet-dashboard.init index ae1731879..f65922b5a 100644 --- a/ext/debian/init +++ b/ext/debian/puppet-dashboard.init @@ -28,6 +28,8 @@ DASHBOARD_RUBY=${DASHBOARD_RUBY:-/usr/bin/ruby} DASHBOARD_ENVIRONMENT=${DASHBOARD_ENVIRONMENT:-production} DASHBOARD_IFACE=${DASHBOARD_IFACE:-0.0.0.0} DASHBOARD_PORT=${DASHBOARD_PORT:-3000} +DASHBOARD_PID=${DASHBOARD_PID:-/var/run/puppet-dashboard} +PID_FILE="${DASHBOARD_PID}/puppet-dashboard.pid" check_dashboard_enabled_option() { if [ ! "$START" = "yes" ]; then @@ -50,7 +52,7 @@ start_puppet_dashboard() { # Wait five seconds, then check if the pid is still valid, # since start-stop-daemon returns success if the process is executed, # but does not return an error if it exits right away. - sleep 5 + sleep 5; echo check_puppet_dashboard_status } diff --git a/ext/debian/logrotate b/ext/debian/puppet-dashboard.logrotate similarity index 76% rename from ext/debian/logrotate rename to ext/debian/puppet-dashboard.logrotate index 47020e767..86012afd5 100644 --- a/ext/debian/logrotate +++ b/ext/debian/puppet-dashboard.logrotate @@ -1,5 +1,5 @@ # Puppet-Dashboard logs: -/var/log/puppet-dashboard/*/*.log { +/var/log/puppet-dashboard/*.log { daily missingok rotate 14 diff --git a/ext/debian/puppet-dashboard_cleanup-reports.cron.weekly b/ext/debian/puppet-dashboard_cleanup-reports.cron.weekly new file mode 100644 index 000000000..5eb269a81 --- /dev/null +++ b/ext/debian/puppet-dashboard_cleanup-reports.cron.weekly @@ -0,0 +1,15 @@ +#!/bin/bash + +set -u + +PUPPET_DB_PATH=/usr/share/puppet-dashboard + +# Prune records up to one month old: +UPTO=10 +# Valid units of time are: hr,day,wk,min,yr,mon +UNIT=mon + +output=`rake -s -f ${PUPPET_DB_PATH}/Rakefile RAILS_ENV=production reports:prune upto=${UPTO} unit=${UNIT} 2>&1 1> /dev/null` +if [ $? -ne 0 ]; then + logger -i "${output}" +fi diff --git a/ext/debian/puppet-dashboard_optimize-db.cron.monthly b/ext/debian/puppet-dashboard_optimize-db.cron.monthly new file mode 100644 index 000000000..4b82c26d9 --- /dev/null +++ b/ext/debian/puppet-dashboard_optimize-db.cron.monthly @@ -0,0 +1,15 @@ +i#!/bin/bash + +set -u + +PUPPET_DB_PATH=/usr/share/puppet-dashboard + +# Prune records up to one month old: +UPTO=1 +# Valid units of time are: hr,day,wk,min,yr,mon +UNIT=mon + +output=`rake -s -f ${PUPPET_DB_PATH}/Rakefile RAILS_ENV=production db:raw:optimize 2>&1 1> /dev/null` +if [ $? -ne 0 ]; then + logger -i "${output}" +fi diff --git a/ext/debian/rules b/ext/debian/rules index c7dc10904..f9e644160 100755 --- a/ext/debian/rules +++ b/ext/debian/rules @@ -6,11 +6,15 @@ include /usr/share/cdbs/1/rules/buildcore.mk ruby_ver = 1.8 binary-install/puppet-dashboard:: - mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/etc/puppet-dashboard - mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/certs - mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/var/$(cdbs_curpkg)/tmp + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/etc/puppet-dashboard/certs + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/usr/cache/puppet-dashboard mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/var/log/puppet-dashboard + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/var/spool/puppet-dashboard + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/var/tmp/puppet-dashboard + mkdir -p $(CURDIR)/debian/$(cdbs_curpkg)/var/run/puppet-dashboard dh_installinit --name=puppet-dashboard-workers + dh_installinit --name=puppet-dashboard cp -r app \ bin \ config \ @@ -26,7 +30,6 @@ binary-install/puppet-dashboard:: $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard cp $(CURDIR)/config/database.yml.example $(CURDIR)/debian/$(cdbs_curpkg)/etc/puppet-dashboard/database.yml cp $(CURDIR)/config/settings.yml.example $(CURDIR)/debian/$(cdbs_curpkg)/etc/puppet-dashboard/settings.yml - cp -r $(CURDIR)/public $(CURDIR)/debian/$(cdbs_curpkg)/var/$(cdbs_curpkg) chmod -R a+rx $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/script rm -rf $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/ext/{redhat,debian,build_defaults.yaml,project_data.yaml} # Clean up the "extra" files @@ -48,4 +51,6 @@ binary-install/puppet-dashboard:: $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/lib \ $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/app \ $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/spec \ - $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/vendor + $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/vendor \ + $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/log \ + $(CURDIR)/debian/$(cdbs_curpkg)/usr/share/puppet-dashboard/tmp/{sessions,sockets,pids} diff --git a/lib/tasks/db_raw.rake b/lib/tasks/db_raw.rake index 3b65df55b..9daa16393 100644 --- a/lib/tasks/db_raw.rake +++ b/lib/tasks/db_raw.rake @@ -57,10 +57,16 @@ namespace :db do case adapter when 'mysql', 'mysql2' puts "Optimizing tables, this may take a while:" - for table in ActiveRecord::Base.connection.select_values('SHOW TABLES').sort + for table in ActiveRecord::Base.connection.tables.sort puts "* #{table}" ActiveRecord::Base.connection.execute("OPTIMIZE TABLE #{table}") end + when 'postgresql' + puts "Optimizing tables, this may take a while:" + for table in ActiveRecord::Base.connection.tables.sort + puts "* #{table}" + ActiveRecord::Base.connection.execute("VACUUM FULL ANALYZE #{table}") + end else raise "Don't know how to optimize for database engine: #{adapter}" end diff --git a/lib/tasks/prune_reports.rake b/lib/tasks/prune_reports.rake index b12a0c030..897825a7d 100644 --- a/lib/tasks/prune_reports.rake +++ b/lib/tasks/prune_reports.rake @@ -81,13 +81,13 @@ UNITS: namespace :prune do desc 'Delete orphaned records whose report has already been deleted' task :orphaned => :environment do - report_dependent_deletion = 'report_id not in (select id from reports)' + report_dependent_deletion = 'e LEFT JOIN reports r ON (e.reports_id = r.id) WHERE r.id IS NULL' orphaned_tables = ActiveSupport::OrderedHash[ Metric, report_dependent_deletion, ReportLog, report_dependent_deletion, ResourceStatus, report_dependent_deletion, - ResourceEvent, 'resource_status_id not in (select id from resource_statuses)' + ResourceEvent, 'e LEFT JOIN resource_statuses s ON (e.resource_status_id = s.id) WHERE resource_status_id.id IS NULL' ] puts "Going to delete orphaned records from #{orphaned_tables.keys.map(&:table_name).join(', ')}\n" @@ -95,18 +95,18 @@ UNITS: orphaned_tables.each do |model, deletion_where_clause| puts "Preparing to delete from #{model.table_name}" start_time = Time.now - deletion_count = model.count(:conditions => deletion_where_clause) + deletion_count = model.count(:joins => deletion_where_clause) puts "#{start_time.to_s(:db)}: Deleting #{deletion_count} orphaned records from #{model.table_name}" pbar = ProgressBar.new('Deleting', deletion_count, STDOUT) - # Deleting a very large group of records in MySQL can be very slow with no feedback + # Deleting a very large group of records can be very slow with no feedback # Breaking the deletion up into blocks turns out to be overall faster # and allows for progress feedback DELETION_BATCH_SIZE = 1000 while deletion_count > 0 ActiveRecord::Base.connection.execute( - "delete from #{model.table_name} where #{deletion_where_clause} limit #{DELETION_BATCH_SIZE}" + "DELETE FROM #{model.table_name} USING #{model.table_name} WHERE #{deletion_where_clause}" ) pbar.inc(DELETION_BATCH_SIZE) deletion_count -= DELETION_BATCH_SIZE