Que monitoring implemented (#586)

* Que monitoring implemented

* Que monitoring added to admin page

* Changed Gemfile.lock

* Got rid of global variable

Gemfile

@@ -34,7 +34,9 @@ gem 'jbuilder'
 # Use Capistrano for deployment
 # gem 'capistrano-rails', group: :development
 
+# Gems for tracking the statuses of jobs
 gem 'que'
+gem 'que-web'
 
 # Reduces boot times through caching; required in config/boot.rb
 gem 'bootsnap', require: false
@@ -48,6 +50,7 @@ gem 'pry-rails'
 
 gem 'aws-sdk-rails'
 gem "aws-sdk-s3", require: false
+gem 'aws-sdk-cloudwatch'
 
 gem 'rollbar'
 gem 'oj' # needed by rollbar

Gemfile.lock

@@ -66,6 +66,9 @@ GEM
     aws-partitions (1.734.0)
     aws-record (2.10.1)
       aws-sdk-dynamodb (~> 1.18)
+    aws-sdk-cloudwatch (1.73.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-core (3.171.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.651.0)
@@ -229,6 +232,8 @@ GEM
     minitest (5.18.0)
     msgpack (1.6.1)
     multi_xml (0.6.0)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
     net-imap (0.3.4)
       date
       net-protocol
@@ -286,6 +291,9 @@ GEM
     puma (6.1.1)
       nio4r (~> 2.0)
     que (2.2.0)
+    que-web (0.10.0)
+      que (>= 1)
+      sinatra
     racc (1.6.2)
     rack (2.2.6.4)
     rack-protection (3.0.5)
@@ -382,6 +390,11 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    sinatra (3.0.5)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.5)
+      tilt (~> 2.0)
     sitemap_generator (6.3.0)
       builder (~> 3.0)
     snaky_hash (2.0.1)
@@ -443,6 +456,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk-cloudwatch
   aws-sdk-rails
   aws-sdk-s3
   bootsnap
@@ -474,6 +488,7 @@ DEPENDENCIES
   pry-rails
   puma
   que
+  que-web
   rails (~> 6.1.7.3)
   rails-i18n
   recaptcha

app/lib/environment.rb

@@ -0,0 +1,35 @@
+module Environment
+  extend self
+
+  def stats_reporter
+    @stats_reporter ||= StatsReporter.new(
+      aws_cloudwatch_client,
+      Rails.env
+    )
+  end
+
+  # INFRASTRUCTURE AWS OBJECTS
+  def aws_cloudwatch_client
+    @aws_cloudwatch_client ||= build_aws_cloudwatch_client
+  end
+
+  def aws_credentials
+    @aws_credentials ||= Aws::Credentials.new(
+      ENV.fetch('AWS_ACCESS_KEY_ID'),
+      ENV.fetch('AWS_SECRET_ACCESS_KEY')
+    )
+  end
+
+  def aws_region
+    @aws_region ||= ENV.fetch('AWS_REGION')
+  end
+
+  private
+  def build_aws_cloudwatch_client
+    Aws::CloudWatch::Client.new(
+      credentials: aws_credentials,
+      region: aws_region
+    )
+  end
+
+end

app/services/stats_reporter.rb

@@ -0,0 +1,27 @@
+class StatsReporter
+  def initialize(cloudwatch_client, environment)
+    @client = cloudwatch_client
+    @environment = environment
+  end
+
+  def report_quarter_hourly
+    report_metric('JobsCount', Que::ActiveRecord::Model.count)
+    report_metric('FailedJobsCount', Que::ActiveRecord::Model.errored.count)
+    report_metric('StuckJobsCount', Que::ActiveRecord::Model.where('run_at < ?', 1.hour.ago).count)
+  end
+
+  private
+  def report_metric(name, value)
+    @client.put_metric_data(
+      namespace: 'NavodyDigital',
+      metric_data: [
+        {
+          metric_name: name,
+          dimensions: [{ name: 'Environment', value: @environment }],
+          timestamp: Time.current.utc,
+          value: value,
+        }
+      ]
+    )
+  end
+end

app/views/layouts/admin.html.erb

@@ -69,6 +69,10 @@
             <li class="govuk-header__navigation-item govuk-header__navigation-item<%= klass %>">
               <%= link_to 'Categories', admin_categories_path, class: 'govuk-header__link' %>
             </li>
+            <!--Added que link to navbar-->
+            <li class="govuk-header__navigation-item govuk-header__navigation-item">
+              <%= link_to 'Que', '/admin/que', class: 'govuk-header__link' %>
+            </li>
           </ul>
         </nav>
       </div>

config.ru

@@ -1,5 +1,11 @@
 # This file is used by Rack-based servers to start the application.
 
-require_relative 'config/environment'
+# Added que-web gem for showing job tracking
+require 'que/web'
+
+map '/admin/que' do
+  run Que::Web
+end
 
+require_relative 'config/environment'
 run Rails.application

config/clock.rb

@@ -17,4 +17,5 @@ module Clockwork
   every(1.day, 'navody:check_or_sr_identifiers_status', at: '9:00')
   every(20.minutes, 'navody:cleanup')
   every(1.week, 'navody:schedule_law_check_job', at: 'Monday 8:00')
+  every(15.minutes, 'navody:report_quarter_hourly')
 end

config/initializers/queweb.rb

@@ -0,0 +1,5 @@
+# Authentication allowing only admin to view the que status
+Que::Web.use(Rack::Auth::Basic) do |user, password|
+  [user, password] == [Rails.application.config_for(:app).dig(:admin, :username),
+                       Rails.application.config_for(:app).dig(:admin, :password)]
+end

config/routes.rb

@@ -1,4 +1,5 @@
 Rails.application.routes.draw do
+
   get :health, to: 'health#index'
   get 'robots.:format', to: 'robots#index'
 
@@ -18,6 +19,7 @@
       put :hide, on: :member
       post :reposition, on: :collection
     end
+
     resources :apps, except: [:show]
     resources :current_topics, except: [:show, :destroy]
     resources :pages, except: [:show]
@@ -39,6 +41,10 @@
       put :feature, on: :member
       put :hide, on: :member
     end
+
+    # Route for que
+    require 'que/web'
+    mount Que::Web => '/admin/que'
   end
 
   root to: 'pages#index'

lib/tasks/tasks.rake

@@ -12,4 +12,8 @@ namespace :navody do
   task schedule_law_check_job: :environment do
     Legal::ScheduleLawCheckJob.perform_later
   end
+
+  task report_quarter_hourly: :environment do
+    Environment.stats_reporter.report_quarter_hourly
+  end
 end