From dad56ecede05b010d2ff6c44c247cde631413846 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Thu, 9 Jan 2020 16:55:29 -0600 Subject: [PATCH 1/7] [Gemfile] Update travis --- Gemfile | 2 +- Gemfile.lock | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index e9649921..18e79b68 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ gem 'slim' # Services gems gem 'minigit', '~> 0.0.4' gem 'tracker_api', '~> 1.6' -gem 'travis', '~> 1.7.6' +gem 'travis', '~> 1.8.10' gem 'awesome_spawn', '>= 1.4.1' gem 'default_value_for', '>= 3.1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 4e099211..e49cb315 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -69,7 +69,6 @@ GEM timers (>= 4.1.1) celluloid-supervision (0.20.6) timers (>= 4.1.1) - coderay (1.1.2) coercible (1.0.0) descendants_tracker (~> 0.0.1) coffee-rails (4.2.2) @@ -223,10 +222,6 @@ GEM parser (2.7.1.2) ast (~> 2.4.0) pg (1.2.2) - pry (0.9.12.6) - coderay (~> 1.0) - method_source (~> 0.8) - slop (~> 3.4) pusher-client (0.6.2) json websocket (~> 1.0) @@ -340,7 +335,6 @@ GEM slim (4.0.1) temple (>= 0.7.6, < 0.9) tilt (>= 2.0.6, < 2.1) - slop (3.6.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -369,15 +363,13 @@ GEM multi_json representable virtus - travis (1.7.7) - addressable (~> 2.3) + travis (1.8.10) backports faraday (~> 0.9) faraday_middleware (~> 0.9, >= 0.9.1) gh (~> 0.13) highline (~> 1.6) launchy (~> 2.1) - pry (~> 0.9, < 0.10) pusher-client (~> 0.4) typhoeus (~> 0.6, >= 0.6.8) turbolinks (5.2.1) @@ -443,7 +435,7 @@ DEPENDENCIES thin timecop tracker_api (~> 1.6) - travis (~> 1.7.6) + travis (~> 1.8.10) turbolinks uglifier (>= 1.3.0) webmock From 23487ecbab07f69a816bc0e8d644f90145c32548 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 13 Jan 2020 18:12:37 -0600 Subject: [PATCH 2/7] [Gemfile] Add `gitter-api-ruby` gem --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index 18e79b68..8bfd12b7 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'sinatra', :require => false gem 'slim' # Services gems +gem 'gitter-api', '~> 0.1.0' gem 'minigit', '~> 0.0.4' gem 'tracker_api', '~> 1.6' gem 'travis', '~> 1.8.10' diff --git a/Gemfile.lock b/Gemfile.lock index e49cb315..267c924e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -163,6 +163,7 @@ GEM multi_json (~> 1.0) net-http-persistent (~> 2.9) net-http-pipeline + gitter-api (0.1.0) globalid (0.4.2) activesupport (>= 4.2.0) haml (5.1.2) @@ -411,6 +412,7 @@ DEPENDENCIES faraday (~> 0.9.2) faraday-http-cache (~> 2.0.0) foreman (~> 0.64.0) + gitter-api (~> 0.1.0) haml (~> 5.1) haml_lint (~> 0.35.0) influxdb (~> 0.3.13) From c4a732348c7d8f622a944029833fbbec19482e37 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 13 Jan 2020 18:08:44 -0600 Subject: [PATCH 3/7] Add TravisV3Client A small Travis API client library that is a compliment to the `travis` gem, but allows the use the v3 API endpoints for specific calls with the v2 based client library entities. Currently implements two methods: * `.repo_branch_builds`: Fetch a collection of builds from a branch * `.repo_build`: Fetch a singular build via ID For the second, the client actually still uses the v2 API, but doesn't use the default model's `.builds.first` approach (doesn't filter/fetch properly), and just fetches using the `find_one` interface. --- lib/travis_v3_client.rb | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 lib/travis_v3_client.rb diff --git a/lib/travis_v3_client.rb b/lib/travis_v3_client.rb new file mode 100644 index 00000000..2490cd38 --- /dev/null +++ b/lib/travis_v3_client.rb @@ -0,0 +1,45 @@ +require 'travis' + +# A helper class for making v3 API class to the TravisCI API, since the current +# API mostly only makes v2 calls, and does a decent amount of inificient ones +# at that... +# +# Expects for most calls that a base Travis::Entity has been set for the given +# client call. Passed in as a hash during initialization. +# +class TravisV3Client + attr_reader :connection, :repo, :user_agent + + def initialize(base_entities = {}) + @connection = Faraday.new(:url => Travis::Client::ORG_URI) + + @connection.headers['Authorization'] = "token #{::Settings.travis.access_token}" + @connection.headers['travis-api-version'] = '3' + + @repo = base_entities[:repo] + + set_user_agent + end + + def repo_branch_builds(branch = "master", params = {}) + query_params = { + "branch.name" => branch, + "build.event_type" => "push,api,cron" + }.merge(params) + + data = connection.get("/repo/#{repo.id}/builds", query_params) + repo.session.load(JSON.parse(data.body))["builds"] + end + + private + + # Use the Travis user agent for this + def set_user_agent + base_model = repo + if base_model + agent_string = repo.session.headers['User-Agent'] + @connection.headers['User-Agent'] = agent_string + agent_string + end + end +end From 945c53b2d19db25c9fac26bf2f792616abb2d2be Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 13 Jan 2020 18:13:09 -0600 Subject: [PATCH 4/7] Add BuildFailureNotifier lib This class is a service class in charge of the logistics of sending a message to a configured gitter channel for a given repo, notifying there has been a failure. --- lib/build_failure_notifier.rb | 98 ++++++++++++++++++++ spec/lib/build_failure_notifier_spec.rb | 37 ++++++++ spec/support/travis_branch_monitor_helper.rb | 33 +++++++ 3 files changed, 168 insertions(+) create mode 100644 lib/build_failure_notifier.rb create mode 100644 spec/lib/build_failure_notifier_spec.rb create mode 100644 spec/support/travis_branch_monitor_helper.rb diff --git a/lib/build_failure_notifier.rb b/lib/build_failure_notifier.rb new file mode 100644 index 00000000..6e6b6db9 --- /dev/null +++ b/lib/build_failure_notifier.rb @@ -0,0 +1,98 @@ +require 'travis' +require 'gitter/api' + +class BuildFailureNotifier + def self.gitter + @gitter ||= begin + api_url = Settings.gitter.api_url + client_settings = { + :token => Settings.gitter_credentials.token, + :api_prefix => Settings.gitter.api_prefix, + :api_uri => api_url && URI(api_url) + } + + Gitter::API::Client.new(client_settings) + end + end + + def self.repo_room_map + settings = Settings.travis_branch_monitor + repo_map = settings.included_repos.to_h.merge(settings.excluded_repos.to_h) + + repo_map.stringify_keys! + repo_map.each do |repo, room_uri| + repo_map[repo] = repo if room_uri.nil? + end + end + + attr_reader :branch, :build, :repo, :repo_path, :room + + def initialize(branch) + @branch = branch + @repo_path = branch.repo.name + @repo = Travis::Repository.find(repo_path) + @room = repo_room_map[repo_path] + @build = repo.session.find_one(Travis::Client::Build, branch.travis_build_failure_id) + end + + def post_failure + notification_msg = <<~MSG + > ### :red_circle: Build Failure in #{repo_branches_markdown_url}! + > + > **Travis Build**: #{travis_build_url} + MSG + notification_msg << "> **Failure PR**: #{offending_pr}\n" if offending_pr + notification_msg << "> **Commit**: #{commit_url}\n" if commit_url + notification_msg << "> **Compare**: #{compare_url}\n" if compare_url + + gitter_room.send_message(notification_msg) + end + + def report_passing + notification_msg = <<~MSG + > ### :green_heart: #{repo_branches_markdown_url} now passing! + MSG + + gitter_room.send_message(notification_msg) + end + + private + + def gitter + self.class.gitter + end + + def repo_room_map + self.class.repo_room_map + end + + # join room if needed, otherwise returns room + def gitter_room + @gitter_room ||= gitter.join_room(room) + end + + def travis_build_url + "https://travis-ci.org/#{repo_path}/builds/#{build.id}" + end + + # find the PR that caused this mess... + def offending_pr + if build.commit && build.commit.message =~ /^Merge pull request #(\d+)/ + "https://github.com/#{repo_path}/issues/#{$1}" + end + end + + def commit_url + if build.commit + "https://github.com/#{repo_path}/commit/#{build.commit.sha[0, 8]}" + end + end + + def compare_url + build.commit.compare_url if build.commit && build.commit.compare_url + end + + def repo_branches_markdown_url + "[`#{repo_path}`](https://travis-ci.org/#{repo_path}/branches)" + end +end diff --git a/spec/lib/build_failure_notifier_spec.rb b/spec/lib/build_failure_notifier_spec.rb new file mode 100644 index 00000000..ea35df33 --- /dev/null +++ b/spec/lib/build_failure_notifier_spec.rb @@ -0,0 +1,37 @@ +describe BuildFailureNotifier do + include IncludedReposConfigMethods + + describe "#repo_room_map (private)" do + it "builds a map from a hash of only keys" do + stub_settings included_repos_keys_only + expected_map = { + "ManageIQ/manageiq" => "ManageIQ/manageiq", + "ManageIQ/miq_bot" => "ManageIQ/miq_bot" + } + + expect(described_class.repo_room_map).to eq(expected_map) + end + + it "builds a map from a hash of keys with values" do + stub_settings included_repos_keys_and_values + expected_map = { + "ManageIQ/manageiq-ui-classic" => "ManageIQ/ui", + "ManageIQ/manageiq-gems-pending" => "ManageIQ/core" + } + + expect(described_class.repo_room_map).to eq(expected_map) + end + + it "builds a map from a mixed hash with keys and some values" do + stub_settings included_repos_mixed_keys_with_some_values + expected_map = { + "ManageIQ/manageiq-ui-classic" => "ManageIQ/ui", + "ManageIQ/manageiq-gems-pending" => "ManageIQ/core", + "ManageIQ/manageiq" => "ManageIQ/manageiq", + "ManageIQ/miq_bot" => "ManageIQ/miq_bot" + } + + expect(described_class.repo_room_map).to eq(expected_map) + end + end +end diff --git a/spec/support/travis_branch_monitor_helper.rb b/spec/support/travis_branch_monitor_helper.rb new file mode 100644 index 00000000..a806beca --- /dev/null +++ b/spec/support/travis_branch_monitor_helper.rb @@ -0,0 +1,33 @@ +require 'yaml' + +module IncludedReposConfigMethods + def included_repos_keys_only + YAML.safe_load <<~YAML + --- + travis_branch_monitor: + included_repos: + ManageIQ/manageiq: + ManageIQ/miq_bot: + YAML + end + + def included_repos_keys_and_values + YAML.safe_load <<~YAML + travis_branch_monitor: + included_repos: + ManageIQ/manageiq-ui-classic: ManageIQ/ui + ManageIQ/manageiq-gems-pending: ManageIQ/core + YAML + end + + def included_repos_mixed_keys_with_some_values + YAML.safe_load <<~YAML + travis_branch_monitor: + included_repos: + ManageIQ/manageiq-ui-classic: ManageIQ/ui + ManageIQ/manageiq-gems-pending: ManageIQ/core + ManageIQ/manageiq: + ManageIQ/miq_bot: + YAML + end +end From c550b0ac91045d3c263f16daac9094e07b202d00 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 13 Jan 2020 18:14:40 -0600 Subject: [PATCH 5/7] [Branch] Add build failure concept Use the Branch record to keep track of build failures that we find when polling the Travis API. The main purpose is to avoid spamming gitter with extra notifications, but also can be used to identify that a branch has previously been broken and now has been fixed. Makes sense to also notify on the passing cases so others aren't taking the time to investigate a failure when this has already been investigated/fixed by another. --- app/models/branch.rb | 33 +++++++++++++++++++ ...0200109223351_add_branch_build_failures.rb | 6 ++++ db/schema.rb | 4 ++- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20200109223351_add_branch_build_failures.rb diff --git a/app/models/branch.rb b/app/models/branch.rb index 976ab852..577e6b26 100644 --- a/app/models/branch.rb +++ b/app/models/branch.rb @@ -99,4 +99,37 @@ def fq_branch_name def git_service GitService::Branch.new(self) end + + # Branch Failure + + def notify_of_failure + if passing? + BuildFailureNotifier.new(self).report_passing + update(:last_build_failure_notified_at => nil, :travis_build_failure_id => nil) + elsif should_notify_of_failure? + update(:last_build_failure_notified_at => Time.zone.now) + + BuildFailureNotifier.new(self).post_failure + end + end + + def previously_failing? + !!travis_build_failure_id + end + + def should_notify_of_failure? + last_build_failure_notified_at.nil? || last_build_failure_notified_at < 1.day.ago + end + + # If we have reported a failure before and the branch is now green. + # + # The other advantage of checking `last_build_failure_notified_at.nil?` here + # is that we save a Travis API call, since we shouldn't be creating + # BuildFailure records without having found a build failure elsewhere (e.g. + # TravisBranchMonitor). + # + # New records will short circut before hitting `Travis::Repository.find`. + def passing? + !last_build_failure_notified_at.nil? && Travis::Repository.find(repo.name).branch(name).green? + end end diff --git a/db/migrate/20200109223351_add_branch_build_failures.rb b/db/migrate/20200109223351_add_branch_build_failures.rb new file mode 100644 index 00000000..1d1d5927 --- /dev/null +++ b/db/migrate/20200109223351_add_branch_build_failures.rb @@ -0,0 +1,6 @@ +class AddBranchBuildFailures < ActiveRecord::Migration[5.2] + def change + add_column :branches, :travis_build_failure_id, :integer + add_column :branches, :last_build_failure_notified_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index a0b6193a..b73a2c36 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2017_10_06_050814) do +ActiveRecord::Schema.define(version: 2020_01_09_223351) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -44,6 +44,8 @@ t.string "merge_target" t.string "pr_title" t.integer "linter_offense_count" + t.integer "travis_build_failure_id" + t.datetime "last_build_failure_notified_at" end create_table "repos", id: :serial, force: :cascade do |t| From b7440e635e04bdfe291b91f3432f1016d6bb5cf4 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 13 Jan 2020 18:18:03 -0600 Subject: [PATCH 6/7] Add TravisBranchMonitor Worker that is in charge of monitoring Travis for build_failures, and will create BuildFailure records and send messages to gitter as needed. --- app/workers/travis_branch_monitor.rb | 94 ++++++++++++++++++++++ spec/workers/travis_branch_monitor_spec.rb | 69 ++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 app/workers/travis_branch_monitor.rb create mode 100644 spec/workers/travis_branch_monitor_spec.rb diff --git a/app/workers/travis_branch_monitor.rb b/app/workers/travis_branch_monitor.rb new file mode 100644 index 00000000..8a0f3a98 --- /dev/null +++ b/app/workers/travis_branch_monitor.rb @@ -0,0 +1,94 @@ +require 'travis' + +class TravisBranchMonitor + include Sidekiq::Worker + sidekiq_options :queue => :miq_bot, :retry => false + + include Sidetiq::Schedulable + recurrence { hourly.minute_of_hour(0, 15, 30, 45) } + + include SidekiqWorkerMixin + + class << self + private + + # For this class, sometimes the repo needs to be mapped to a specific + # gitter room, so a hash is required. + # + # This override allows for doing this in the config + # + # travis_branch_monitor: + # included_repos: + # ManageIQ/manageiq-ui-classic: ManageIQ/ui + # ManageIQ/manageiq-gems-pending: ManageIQ/core + # ManageIQ/manageiq: + # ManageIQ/miq_bot: + # + # Which you are allowed to leave the value empty, and the key will be used + # where appropriate (not used in this class). + # + # The result from the above for this method will then be: + # + # [ + # [ + # "ManageIQ/manageiq-ui-classic", + # "ManageIQ/manageiq-gems-pending", + # "ManageIQ/manageiq", + # "ManageIQ/miq_bot" + # ], + # [] + # ] + # + def included_and_excluded_repos + super # just used for error handling... + + [ + settings.included_repos.try(:to_h).try(:stringify_keys).try(:keys), + settings.excluded_repos.try(:to_h).try(:stringify_keys).try(:keys) + ] + end + end + + def perform + if !first_unique_worker? + logger.info("#{self.class} is already running, skipping") + else + process_repos + end + end + + def process_repos + enabled_repos.each do |repo| + process_repo(repo) + end + end + + def process_repo(repo) + repo.regular_branch_names.each do |branch_record| + process_branch(repo, branch_record) + end + end + + def process_branch(repo, branch_record) + # If we already have a failure record, call notify with that record + return branch_record.notify_of_failure if branch_record.previously_failing? + + # otherwise, check if any builds exist with a failures, and if so, update + # the branch_record to add the `travis_build_failure_id`. + v3_client = TravisV3Client.new(:repo => Travis::Repository.find(repo.name)) + branch_builds = v3_client.repo_branch_builds(branch_record.name) + + if branch_builds.first.failed? + first_failure = find_first_recent_failure(branch_builds) + branch_record.update(:travis_build_failure_id => first_failure.id) + + branch_record.notify_of_failure + end + end + + private + + def find_first_recent_failure(builds) + builds.take_while(&:failed?).last + end +end diff --git a/spec/workers/travis_branch_monitor_spec.rb b/spec/workers/travis_branch_monitor_spec.rb new file mode 100644 index 00000000..bf41b985 --- /dev/null +++ b/spec/workers/travis_branch_monitor_spec.rb @@ -0,0 +1,69 @@ +describe TravisBranchMonitor do + include IncludedReposConfigMethods + + describe ".included_and_excluded_repos (private)" do + it "builds the list from a hash of only keys" do + stub_settings included_repos_keys_only + expected = %w[ManageIQ/manageiq ManageIQ/miq_bot] + + expect(described_class.send(:included_and_excluded_repos)).to eq([expected, nil]) + end + + it "builds the list from a hash of keys with values" do + stub_settings included_repos_keys_and_values + expected = %w[ManageIQ/manageiq-ui-classic ManageIQ/manageiq-gems-pending] + + expect(described_class.send(:included_and_excluded_repos)).to eq([expected, nil]) + end + + it "builds the list from a mixed hash with keys and some values" do + stub_settings included_repos_mixed_keys_with_some_values + expected = %w[ + ManageIQ/manageiq-ui-classic + ManageIQ/manageiq-gems-pending + ManageIQ/manageiq + ManageIQ/miq_bot + ] + + expect(described_class.send(:included_and_excluded_repos)).to eq([expected, nil]) + end + end + + describe "#find_first_recent_failure (private)" do + def passed(build_id) + build = Travis::Client::Build.new(nil, build_id) + allow(build).to receive(:inspect_info).and_return("Foo/foo##{build_id}") + build.tap { |b| b.update_attributes(:state => "passed") } + end + + def failed(build_id) + build = Travis::Client::Build.new(nil, build_id) + allow(build).to receive(:inspect_info).and_return("Foo/foo##{build_id}") + build.tap { |b| b.update_attributes(:state => "failed") } + end + + it "returns earliest failure" do + earliest_failure = failed(2) + builds = [ + failed(4), + failed(3), + earliest_failure, + passed(1) + ] + + expect(subject.send(:find_first_recent_failure, builds)).to eq(earliest_failure) + end + + it "returns nil if the first build has passed" do + builds = [ + passed(5), + failed(4), + failed(3), + failed(2), + passed(1) + ] + + expect(subject.send(:find_first_recent_failure, builds)).to eq(nil) + end + end +end From d68b6a4e7da5ba88d60b54d373484268082ee037 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Tue, 14 Jan 2020 19:03:35 -0600 Subject: [PATCH 7/7] [rake] Add travis_branch_monitor:poll_single A task for triggering a single run of the TravisBranchMonitor without needing sidekiq running. --- lib/tasks/travis_branch_monitor.rake | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 lib/tasks/travis_branch_monitor.rake diff --git a/lib/tasks/travis_branch_monitor.rake b/lib/tasks/travis_branch_monitor.rake new file mode 100644 index 00000000..8ec6e298 --- /dev/null +++ b/lib/tasks/travis_branch_monitor.rake @@ -0,0 +1,5 @@ +namespace :travis_branch_monitor do + task :poll_single => :environment do + TravisBranchMonitor.new.perform + end +end