Output HTML artifact containing all failures:

Buildkite annotations, which this plugin creates, are limited to 100KB, and this plugin has a
sophisticated algorithm for truncating the annotation to only the number of failures that would generate a
Markdown-formatted annotation of less than that size. With stacktraces, that number can be as low
as 15 failures: a small fraction of what a change to a large monorepo can cause.

This change causes the plugin to also upload an artifact containing the HTML-ified non-truncated Markdown
output, and link to that artifact from the annotation if the annotation has been truncated.

This requires a refactor: Truncater now knows nothing about Formatter or Markdown; it only handles
the truncation algorithm on the strings returned by a block. Much of the logic to wire Truncater and
Formatters together has been moved to a new Processor class (this name was too similar to "Runner", so I
renamed that to "Main"). This makes each class a bit more cohesive: Truncater only handles string truncation,
Processor hands off the result of Formatter to Truncater, and Main writes the result of Processor to
annotations and artifacts.

Also:
- I repurposed the `test_layout` template to also wrap the artifact, making the appearance similar to the
  annotation within Buildkite's UI.
- There's now a mixin for classes that want to reraise-or-log, depending on the result of a `fail_on_error`
  method, and any error can now be accompanied by a "diagnostics" JSON object.
- We now have a runtime dependency on a Markdown-to-HTML converter, so I switched to Kramdown, which
  is pure Ruby. The current minimal Dockerfile doesn't have the tools to compile C extensions.

Gemfile

@@ -6,6 +6,7 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gem 'bundler', '~> 1.16'
 gem 'haml'
+gem 'kramdown'
 
 group :development, :test do
   gem 'rake', '~> 12.3'
@@ -14,10 +15,6 @@ group :development, :test do
   gem 'rubocop'
 end
 
-group :development do
-  gem 'commonmarker'
-end
-
 group :test do
   gem 'simplecov', require: false
 end

Gemfile.lock

@@ -2,23 +2,21 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    commonmarker (0.17.9)
-      ruby-enum (~> 0.5)
-    concurrent-ruby (1.0.5)
     diff-lcs (1.3)
     docile (1.3.0)
     haml (5.0.4)
       temple (>= 0.8.0)
       tilt
-    i18n (1.0.1)
-      concurrent-ruby (~> 1.0)
     json (2.1.0)
+    kramdown (2.3.1)
+      rexml
     parallel (1.12.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
     powerpack (0.1.1)
     rainbow (3.0.0)
     rake (12.3.3)
+    rexml (3.2.4)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
@@ -41,8 +39,6 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-enum (0.7.2)
-      i18n
     ruby-progressbar (1.9.0)
     simplecov (0.16.1)
       docile (~> 1.1)
@@ -58,8 +54,8 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.16)
-  commonmarker
   haml
+  kramdown
   rake (~> 12.3)
   rspec (~> 3.0)
   rspec_junit_formatter

bin/run-dev

@@ -5,7 +5,6 @@ require 'bundler/setup'
 require 'pathname'
 $LOAD_PATH.unshift(Pathname.new(__FILE__).dirname.dirname.join('lib').to_s)
 
-require 'commonmarker'
 require 'yaml'
 require 'test_summary_buildkite_plugin'
 
@@ -20,8 +19,7 @@ module TestSummaryBuildkitePlugin
       log(args, stdin: stdin)
       if args.first == 'annotate'
         context = args[2]
-        content = CommonMarker.render_html(stdin)
-        html = HamlRender.render('test_layout', content: content)
+        html = Utils.standalone_markdown(stdin)
         FileUtils.mkdir_p('tmp')
         out = Pathname.new('tmp').join(context + '.html')
         out.open('w') { |file| file.write(html) }

lib/test_summary_buildkite_plugin.rb

@@ -3,11 +3,13 @@
 require 'json'
 
 require 'test_summary_buildkite_plugin/agent'
+require 'test_summary_buildkite_plugin/error_handler'
 require 'test_summary_buildkite_plugin/failure'
 require 'test_summary_buildkite_plugin/formatter'
 require 'test_summary_buildkite_plugin/haml_render'
 require 'test_summary_buildkite_plugin/input'
-require 'test_summary_buildkite_plugin/runner'
+require 'test_summary_buildkite_plugin/main'
+require 'test_summary_buildkite_plugin/processor'
 require 'test_summary_buildkite_plugin/tap'
 require 'test_summary_buildkite_plugin/truncater'
 require 'test_summary_buildkite_plugin/utils'
@@ -18,6 +20,6 @@ def self.run
     plugins = JSON.parse(ENV.fetch('BUILDKITE_PLUGINS'), symbolize_names: true)
     # plugins is an array of hashes, keyed by <github-url>#<version>
     options = plugins.find { |k, _| k.to_s.include?('test-summary') }.values.first
-    Runner.new(options).run
+    Main.new(options).run
   end
 end

lib/test_summary_buildkite_plugin/error_handler.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module TestSummaryBuildkitePlugin
+  module ErrorHandler
+    def handle_error(err, diagnostics = nil)
+      if fail_on_error
+        raise err
+      else
+        Utils.log_error(err, diagnostics)
+      end
+    end
+  end
+end

lib/test_summary_buildkite_plugin/formatter.rb

@@ -4,51 +4,55 @@
 
 module TestSummaryBuildkitePlugin
   class Formatter
-    def self.create(**options)
+    def self.create(input:, output_path:, options:)
       options[:type] ||= 'details'
       type = options[:type].to_sym
       raise "Unknown type: #{type}" unless TYPES.key?(type)
-      TYPES[type].new(options)
+      TYPES[type].new(input: input, output_path: output_path, options: options)
     end
 
     class Base
-      attr_reader :options
+      attr_reader :input, :output_path, :options
 
-      def initialize(options = {})
+      def initialize(input:, output_path:, options:)
+        @input = input
+        @output_path = output_path
         @options = options || {}
       end
 
-      def markdown(input)
+      def markdown(truncate)
         return nil if input.failures.count.zero?
-        [heading(input), input_markdown(input), footer(input)].compact.join("\n\n")
+        [heading(truncate), input_markdown(truncate), footer].compact.join("\n\n")
       end
 
       protected
 
-      def input_markdown(input)
-        if show_first.negative? || show_first >= include_failures(input).count
-          failures_markdown(include_failures(input))
+      def input_markdown(truncate)
+        failures = include_failures(truncate)
+
+        if show_first.negative? || show_first >= failures.count
+          failures_markdown(failures, truncate)
         elsif show_first.zero?
-          details('Show failures', failures_markdown(include_failures(input)))
+          details('Show failures', failures_markdown(failures, truncate))
         else
-          failures_markdown(include_failures(input)[0...show_first]) +
-            details('Show additional failures', failures_markdown(include_failures(input)[show_first..-1]))
+          failures_markdown(failures[0...show_first], false) +
+            details('Show additional failures', failures_markdown(failures[show_first..-1], truncate))
         end
       end
 
-      def failures_markdown(failures)
-        render_template('failures', failures: failures)
+      def failures_markdown(failures, truncate)
+        render_template('failures', failures: failures, output_path: truncate ? output_path : nil)
       end
 
-      def heading(input)
+      def heading(truncate)
         count = input.failures.count
-        show_count = include_failures(input).count
+        show_count = include_failures(truncate).count
         s = "##### #{input.label}: #{count} failure#{'s' unless count == 1}"
         s += "\n\n_Including first #{show_count} failures_" if show_count < count
         s
       end
 
-      def footer(input)
+      def footer
         job_ids = input.failures.map(&:job_id).uniq.reject(&:nil?)
         render_template('footer', job_ids: job_ids)
       end
@@ -65,11 +69,7 @@ def type
         options[:type] || 'details'
       end
 
-      def truncate
-        options[:truncate]
-      end
-
-      def include_failures(input)
+      def include_failures(truncate)
         if truncate
           input.failures[0...truncate]
         else

lib/test_summary_buildkite_plugin/input.rb

@@ -16,6 +16,8 @@ def self.create(type:, **options)
     end
 
     class Base
+      include ErrorHandler
+
       attr_reader :label, :artifact_path, :options
 
       def initialize(label:, artifact_path:, **options)
@@ -77,14 +79,6 @@ def job_id_regex
           DEFAULT_JOB_ID_REGEX
         end
       end
-
-      def handle_error(err)
-        if fail_on_error
-          raise err
-        else
-          Utils.log_error(err)
-        end
-      end
     end
 
     class OneLine < Base

lib/test_summary_buildkite_plugin/runner.rb → lib/test_summary_buildkite_plugin/main.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 module TestSummaryBuildkitePlugin
-  class Runner
+  class Main
     MAX_MARKDOWN_SIZE = 100_000
+    OUTPUT_PATH = 'test-summary.html'
 
     attr_reader :options
 
@@ -11,23 +12,37 @@ def initialize(options)
     end
 
     def run
-      markdown = Truncater.new(
+      processor = Processor.new(
+        formatter_options: formatter,
         max_size: MAX_MARKDOWN_SIZE,
+        output_path: OUTPUT_PATH,
         inputs: inputs,
-        formatter_opts: options[:formatter],
         fail_on_error: fail_on_error
-      ).markdown
-      if markdown.nil? || markdown.empty?
+      )
+
+      if processor.truncated_markdown.nil? || processor.truncated_markdown.empty?
         puts('No errors found! 🎉')
       else
-        annotate(markdown)
+        upload_artifact(processor.inputs_markdown)
+        annotate(processor.truncated_markdown)
       end
     end
 
+    private
+
+    def upload_artifact(markdown)
+      File.write(OUTPUT_PATH, Utils.standalone_markdown(markdown))
+      Agent.run('artifact', 'upload', OUTPUT_PATH)
+    end
+
     def annotate(markdown)
       Agent.run('annotate', '--context', context, '--style', style, stdin: markdown)
     end
 
+    def formatter
+      options[:formatter] || {}
+    end
+
     def inputs
       @inputs ||= options[:inputs].map { |opts| Input.create(opts.merge(fail_on_error: fail_on_error)) }
     end

lib/test_summary_buildkite_plugin/processor.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module TestSummaryBuildkitePlugin
+  class Processor
+    include ErrorHandler
+
+    attr_reader :formatter_options, :max_size, :output_path, :inputs, :fail_on_error
+
+    def initialize(formatter_options:, max_size:, output_path:, inputs:, fail_on_error:)
+      @formatter_options = formatter_options
+      @max_size = max_size
+      @output_path = output_path
+      @inputs = inputs
+      @fail_on_error = fail_on_error
+      @_formatters = {}
+    end
+
+    def truncated_markdown
+      @truncated_markdown ||= begin
+        truncater = Truncater.new(
+          max_size: max_size,
+          max_truncate: inputs.map(&:failures).map(&:count).max
+        ) do |truncate|
+          inputs_markdown(truncate)
+        end
+
+        truncater.markdown
+      rescue StandardError => e
+        handle_error(e, diagnostics)
+        HamlRender.render('truncater_exception', {})
+      end
+    end
+
+    def inputs_markdown(truncate = nil)
+      inputs.map { |input| input_markdown(input, truncate) }.compact.join("\n\n")
+    end
+
+    private
+
+    def input_markdown(input, truncate)
+      formatter(input).markdown(truncate)
+    rescue StandardError => e
+      handle_error(e)
+    end
+
+    def formatter(input)
+      @_formatters[input] ||= Formatter.create(input: input, output_path: output_path, options: formatter_options)
+    end
+
+    def diagnostics
+      {
+        formatter: formatter_options,
+        inputs: inputs.map do |input|
+          {
+            type: input.class,
+            failure_count: input.failures.count,
+            markdown_bytesize: input_markdown(input, nil)&.bytesize
+          }
+        end
+      }
+    end
+  end
+end