Fix randomly failing tests (#4064)

* Fix randomly failing tests

When testing that some function was not called when mocked, other tests
from the same test suite might interfere, if the call a function with
the same arguments. Fix by making each function use different from-to
arguments.

* Add more info when the test formatter fails to match a failed test

* Bring back formatter checking in Dockerfile

* Update Sanbase.FailedTestFormatter

* Use :warning instead of :warn in logger config

Dockerfile

@@ -70,7 +70,7 @@ COPY src src
 COPY assets assets
 
 # check that the code is formatted
-# RUN mix format --check-formatted
+RUN mix format --check-formatted
 
 # compile assets
 RUN cd assets && npm install

config/test.exs

@@ -26,7 +26,7 @@ config :sanbase, Sanbase.PresignedS3Url.S3,
 # Print only warnings and errors during test. Do not log JSON in tests.
 config :logger, :console,
   format: "$time $metadata[$level] $message\n",
-  level: :warn
+  level: :warning
 
 config :sanbase, Sanbase.RepoReader, projects_data_endpoint_secret: "no_secret"
 

lib/mix/failure_test_formatter.ex

@@ -7,29 +7,30 @@ defmodule Sanbase.FailedTestFormatter do
   def init(_opts) do
     {:ok,
      %{
-       failed: [],
-       failure_counter: 0
+       :error => %{list: [], counter: 0},
+       :invalid => %{list: [], counter: 0}
      }}
   end
 
   def handle_cast({:test_finished, %ExUnit.Test{state: {:failed, _}} = test}, config) do
     config =
       case test do
-        %ExUnit.Test{state: {:failed, [{:error, _error, failures}]}} = test
-        when is_list(failures) ->
+        %ExUnit.Test{state: {:failed, [{kind, _error, _stacktrace}]}} = test
+        when kind in [:error, :invalid] ->
           # Add a leading dot so the file:line string can be copy-pasted in the
           # terminal to directly execute it
           file = String.replace_leading(test.tags.file, File.cwd!() <> "/", "")
           line = test.tags.line
+          test_identifier = "#{file}:#{line}"
 
-          %{
-            config
-            | failed: ["#{file}:#{line}" | config.failed],
-              failure_counter: config.failure_counter + 1
-          }
+          config
+          |> Map.update(kind, %{counter: 1, list: [test_identifier]}, fn map ->
+            map |> Map.update!(:counter, &(&1 + 1)) |> Map.update!(:list, &[test_identifier | &1])
+          end)
 
-        _ ->
-          IO.warn("Unexpected failed test format.")
+        # TODO: Support ExUnit.MultiError
+        test ->
+          IO.warn("Unexpected failed test format. Got: #{inspect(test)}")
           config
       end
 
@@ -46,13 +47,19 @@ defmodule Sanbase.FailedTestFormatter do
   end
 
   defp print_suite(config) do
-    if config.failure_counter > 0 do
-      message = config.failed |> Enum.map(&(" " <> &1)) |> Enum.join("\n")
+    for kind <- Map.keys(config) do
+      if (get_in(config, [kind, :counter]) || 0) > 0 do
+        # All tests that failed an assert will have `kind = :error`
+        error_tests_message =
+          get_in(config, [kind, :list]) |> Enum.map(&(" " <> &1)) |> Enum.join("\n")
 
-      formatted_message =
-        IO.ANSI.red() <> "\n\nFailed tests:\n" <> message <> "\n" <> IO.ANSI.reset()
+        formatted_message =
+          IO.ANSI.red() <>
+            "\n\n#{String.capitalize(to_string(kind))} tests:\n" <>
+            error_tests_message <> "\n" <> IO.ANSI.reset()
 
-      IO.puts(formatted_message)
+        IO.puts(formatted_message)
+      end
     end
   end
 end

test/sanbase_web/graphql/metric/api_metric_social_metrics_timeframe_restriction_test.exs

@@ -49,7 +49,7 @@ defmodule Sanbase.Graphql.ApiMetricSocialMetricsTimeframeRestrictionTest do
     end
 
     test "cannot access historical data for social metrics", context do
-      {from, to} = from_to(5 * 365, 2 * 365)
+      {from, to} = from_to(5 * 365 - 1, 2 * 365)
       slug = context.project.slug
       metric = Enum.random(context.metrics)
       interval = "1d"
@@ -70,7 +70,7 @@ defmodule Sanbase.Graphql.ApiMetricSocialMetricsTimeframeRestrictionTest do
     end
 
     test "can access realtime data for social metrics", context do
-      {from, to} = from_to(2, 0)
+      {from, to} = from_to(3, 0)
       slug = context.project.slug
       metric = Enum.random(context.metrics)
       interval = "5m"
@@ -82,7 +82,7 @@ defmodule Sanbase.Graphql.ApiMetricSocialMetricsTimeframeRestrictionTest do
     end
 
     test "can access historical data for social metrics", context do
-      {from, to} = from_to(5 * 365, 2 * 365)
+      {from, to} = from_to(5 * 365 - 2, 2 * 365)
       slug = context.project.slug
       metric = Enum.random(context.metrics)
       interval = "1d"