From 17077892d2b1b321d691b7d1f5e558b073ae91a5 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sat, 15 Apr 2023 07:58:58 +0100 Subject: [PATCH 1/3] add admin cron jobs helpers --- helpers/admin_jobs_helper.rb | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 helpers/admin_jobs_helper.rb diff --git a/helpers/admin_jobs_helper.rb b/helpers/admin_jobs_helper.rb new file mode 100644 index 00000000..1eb758d1 --- /dev/null +++ b/helpers/admin_jobs_helper.rb @@ -0,0 +1,107 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module AdminJobsHelper + + def redis_cron + Redis.new(host: NcboCron.settings.redis_host, port: NcboCron.settings.redis_port, timeout: 30) + end + + def cron_daemon_options + cron_settings_json = redis_cron.get "cron:daemon:options" + error 500, "Unable to get CRON daemon options from Redis" if cron_settings_json.nil? + JSON.parse(cron_settings_json, symbolize_names: true) + end + + def scheduled_jobs_map + cron_options = cron_daemon_options + + { + parse: { + title: "parse semantic resources", + enabled: cron_options[:enable_processing] || false, + scheduler_type: "every", + schedule: "5m" # this is hardcoded in the scheduler class + }, + pull: { + title: "pull remote semantic resources", + enabled: cron_options[:enable_pull] || false, + scheduler_type: "cron", + schedule: cron_options[:pull_schedule] + }, + flush: { + title: "flush classes", + enabled: cron_options[:enable_flush] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_flush] + }, + warmq: { + title: "warm up queries", + enabled: cron_options[:enable_warmq] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_warmq] + }, + mapping_counts: { + title: "mapping counts generation", + enabled: cron_options[:enable_mapping_counts] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_mapping_counts] + }, + ontology_analytics: { + title: "semantic resource analytics", + enabled: cron_options[:enable_ontology_analytics] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_ontology_analytics] + }, + ontologies_report: { + title: "semantic resources report", + enabled: cron_options[:enable_ontologies_report] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_ontologies_report] + }, + index_synchronizer: { + title: "index synchronization", + enabled: cron_options[:enable_index_synchronizer] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_index_synchronizer] + }, + spam_deletion: { + title: "spam deletion", + enabled: cron_options[:enable_spam_deletion] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_spam_deletion] + }, + obofoundry_sync: { + title: "OBO Foundry synchronization", + enabled: cron_options[:enable_obofoundry_sync] || false, + scheduler_type: "cron", + schedule: cron_options[:cron_obofoundry_sync] + } + } + end + + def stream_file(filename) + if !File.exist? filename + stream { |out| out << "" } + else + content_type "text/plain" + stream do |out| + File.open(filename, mode = "r") do |f| + loop do + chunk = f.read 4096 + if chunk.nil? + break + else + out << chunk + end + end + end + end + end + end + end + end +end + +helpers Sinatra::Helpers::AdminJobsHelper From e8b41921fafd3c4952c93bc1d2f560bd72d75ce3 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sat, 15 Apr 2023 08:00:31 +0100 Subject: [PATCH 2/3] add cron jobs administration endpoints --- controllers/admin_controller.rb | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 7ae6d800..d9e6876a 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -9,18 +9,26 @@ class AdminController < ApplicationController end } - # TODO: remove this endpoint. It's termporary to test the update check functionality - # get "/latestversion" do - # iid = params["iid"] - # ver = params["version"] - # - # latest_ver_info = { - # update_version: "2.5RC3", #"2.6RC1", - # update_available: true, - # notes: "blah blah and more" - # } - # reply MultiJson.dump latest_ver_info - # end + + get "/scheduled_jobs" do + reply MultiJson.dump scheduled_jobs_map + end + + get "/scheduled_jobs/log" do + log_path = cron_daemon_options[:log_path] + stream_file(log_path) + end + + get "/scheduled_jobs/:job/log" do + scheduled_jobs = scheduled_jobs_map + job_name = params["job"] + error 404, "You must provide a valid `job` to retrieve its log" unless scheduled_jobs.has_key?(job_name.to_sym) + + log_dir_name = File.dirname(cron_daemon_options[:log_path]) + log_file_name = "#{log_dir_name.chomp '/'}/scheduler-#{job_name.gsub '_', '-'}.log" + + stream_file(log_file_name) + end get "/update_info" do um = NcboCron::Models::UpdateManager.new From 544de924a680751c734e504716a495c09b0c749e Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sun, 30 Jul 2023 01:19:30 +0200 Subject: [PATCH 3/3] fix admin jobs helper JSON module not found --- helpers/admin_jobs_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/admin_jobs_helper.rb b/helpers/admin_jobs_helper.rb index 1eb758d1..eb3e8767 100644 --- a/helpers/admin_jobs_helper.rb +++ b/helpers/admin_jobs_helper.rb @@ -11,7 +11,7 @@ def redis_cron def cron_daemon_options cron_settings_json = redis_cron.get "cron:daemon:options" error 500, "Unable to get CRON daemon options from Redis" if cron_settings_json.nil? - JSON.parse(cron_settings_json, symbolize_names: true) + ::JSON.parse(cron_settings_json, symbolize_names: true) end def scheduled_jobs_map