From d0d1f0a3e4f126dd9902abbe1796860f9f9b22fc Mon Sep 17 00:00:00 2001 From: DevTomii Date: Tue, 27 Jun 2023 16:50:55 +0200 Subject: [PATCH] Que monitoring implemented (#586) * Que monitoring implemented * Que monitoring added to admin page * Changed Gemfile.lock * Got rid of global variable --- Gemfile | 3 +++ Gemfile.lock | 15 ++++++++++++++ app/lib/environment.rb | 35 ++++++++++++++++++++++++++++++++ app/services/stats_reporter.rb | 27 ++++++++++++++++++++++++ app/views/layouts/admin.html.erb | 4 ++++ config.ru | 8 +++++++- config/clock.rb | 1 + config/initializers/queweb.rb | 5 +++++ config/routes.rb | 6 ++++++ lib/tasks/tasks.rake | 4 ++++ 10 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 app/lib/environment.rb create mode 100644 app/services/stats_reporter.rb create mode 100644 config/initializers/queweb.rb diff --git a/Gemfile b/Gemfile index 05b43cff..aacfd44a 100644 --- a/Gemfile +++ b/Gemfile @@ -34,7 +34,9 @@ gem 'jbuilder' # Use Capistrano for deployment # gem 'capistrano-rails', group: :development +# Gems for tracking the statuses of jobs gem 'que' +gem 'que-web' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false @@ -48,6 +50,7 @@ gem 'pry-rails' gem 'aws-sdk-rails' gem "aws-sdk-s3", require: false +gem 'aws-sdk-cloudwatch' gem 'rollbar' gem 'oj' # needed by rollbar diff --git a/Gemfile.lock b/Gemfile.lock index 0bf43131..c739a434 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -66,6 +66,9 @@ GEM aws-partitions (1.734.0) aws-record (2.10.1) aws-sdk-dynamodb (~> 1.18) + aws-sdk-cloudwatch (1.73.0) + aws-sdk-core (~> 3, >= 3.165.0) + aws-sigv4 (~> 1.1) aws-sdk-core (3.171.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.651.0) @@ -229,6 +232,8 @@ GEM minitest (5.18.0) msgpack (1.6.1) multi_xml (0.6.0) + mustermann (3.0.0) + ruby2_keywords (~> 0.0.1) net-imap (0.3.4) date net-protocol @@ -286,6 +291,9 @@ GEM puma (6.1.1) nio4r (~> 2.0) que (2.2.0) + que-web (0.10.0) + que (>= 1) + sinatra racc (1.6.2) rack (2.2.6.4) rack-protection (3.0.5) @@ -382,6 +390,11 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) + sinatra (3.0.5) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.0.5) + tilt (~> 2.0) sitemap_generator (6.3.0) builder (~> 3.0) snaky_hash (2.0.1) @@ -443,6 +456,7 @@ PLATFORMS ruby DEPENDENCIES + aws-sdk-cloudwatch aws-sdk-rails aws-sdk-s3 bootsnap @@ -474,6 +488,7 @@ DEPENDENCIES pry-rails puma que + que-web rails (~> 6.1.7.3) rails-i18n recaptcha diff --git a/app/lib/environment.rb b/app/lib/environment.rb new file mode 100644 index 00000000..2c973c21 --- /dev/null +++ b/app/lib/environment.rb @@ -0,0 +1,35 @@ +module Environment + extend self + + def stats_reporter + @stats_reporter ||= StatsReporter.new( + aws_cloudwatch_client, + Rails.env + ) + end + + # INFRASTRUCTURE AWS OBJECTS + def aws_cloudwatch_client + @aws_cloudwatch_client ||= build_aws_cloudwatch_client + end + + def aws_credentials + @aws_credentials ||= Aws::Credentials.new( + ENV.fetch('AWS_ACCESS_KEY_ID'), + ENV.fetch('AWS_SECRET_ACCESS_KEY') + ) + end + + def aws_region + @aws_region ||= ENV.fetch('AWS_REGION') + end + + private + def build_aws_cloudwatch_client + Aws::CloudWatch::Client.new( + credentials: aws_credentials, + region: aws_region + ) + end + +end diff --git a/app/services/stats_reporter.rb b/app/services/stats_reporter.rb new file mode 100644 index 00000000..2b8c1ac3 --- /dev/null +++ b/app/services/stats_reporter.rb @@ -0,0 +1,27 @@ +class StatsReporter + def initialize(cloudwatch_client, environment) + @client = cloudwatch_client + @environment = environment + end + + def report_quarter_hourly + report_metric('JobsCount', Que::ActiveRecord::Model.count) + report_metric('FailedJobsCount', Que::ActiveRecord::Model.errored.count) + report_metric('StuckJobsCount', Que::ActiveRecord::Model.where('run_at < ?', 1.hour.ago).count) + end + + private + def report_metric(name, value) + @client.put_metric_data( + namespace: 'NavodyDigital', + metric_data: [ + { + metric_name: name, + dimensions: [{ name: 'Environment', value: @environment }], + timestamp: Time.current.utc, + value: value, + } + ] + ) + end +end diff --git a/app/views/layouts/admin.html.erb b/app/views/layouts/admin.html.erb index db3afe09..065f3771 100644 --- a/app/views/layouts/admin.html.erb +++ b/app/views/layouts/admin.html.erb @@ -69,6 +69,10 @@
  • <%= link_to 'Categories', admin_categories_path, class: 'govuk-header__link' %>
    • + +
  • + <%= link_to 'Que', '/admin/que', class: 'govuk-header__link' %> +
    • diff --git a/config.ru b/config.ru index f7ba0b52..65042a6f 100644 --- a/config.ru +++ b/config.ru @@ -1,5 +1,11 @@ # This file is used by Rack-based servers to start the application. -require_relative 'config/environment' +# Added que-web gem for showing job tracking +require 'que/web' + +map '/admin/que' do + run Que::Web +end +require_relative 'config/environment' run Rails.application diff --git a/config/clock.rb b/config/clock.rb index cb0e8a88..04b89eaa 100644 --- a/config/clock.rb +++ b/config/clock.rb @@ -17,4 +17,5 @@ module Clockwork every(1.day, 'navody:check_or_sr_identifiers_status', at: '9:00') every(20.minutes, 'navody:cleanup') every(1.week, 'navody:schedule_law_check_job', at: 'Monday 8:00') + every(15.minutes, 'navody:report_quarter_hourly') end diff --git a/config/initializers/queweb.rb b/config/initializers/queweb.rb new file mode 100644 index 00000000..1cbdb610 --- /dev/null +++ b/config/initializers/queweb.rb @@ -0,0 +1,5 @@ +# Authentication allowing only admin to view the que status +Que::Web.use(Rack::Auth::Basic) do |user, password| + [user, password] == [Rails.application.config_for(:app).dig(:admin, :username), + Rails.application.config_for(:app).dig(:admin, :password)] +end diff --git a/config/routes.rb b/config/routes.rb index ee722034..eaa97608 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,5 @@ Rails.application.routes.draw do + get :health, to: 'health#index' get 'robots.:format', to: 'robots#index' @@ -18,6 +19,7 @@ put :hide, on: :member post :reposition, on: :collection end + resources :apps, except: [:show] resources :current_topics, except: [:show, :destroy] resources :pages, except: [:show] @@ -39,6 +41,10 @@ put :feature, on: :member put :hide, on: :member end + + # Route for que + require 'que/web' + mount Que::Web => '/admin/que' end root to: 'pages#index' diff --git a/lib/tasks/tasks.rake b/lib/tasks/tasks.rake index 1fafc924..862d3e45 100644 --- a/lib/tasks/tasks.rake +++ b/lib/tasks/tasks.rake @@ -12,4 +12,8 @@ namespace :navody do task schedule_law_check_job: :environment do Legal::ScheduleLawCheckJob.perform_later end + + task report_quarter_hourly: :environment do + Environment.stats_reporter.report_quarter_hourly + end end