Enhance testing experience

.travis.yml

@@ -13,9 +13,5 @@ before_script:
   - mix local.hex --force
   - mix do deps.get, deps.compile, compile --warnings-as-errors
 script:
-  - mix test --cover --trace --exclude will_fail:true --exclude unstable_test:true --exclude not_implemented:true
-  - ./.travis-scripts/verify_storing_counterexamples.sh
-  - ./.travis-scripts/verify-verbose.sh
-  - ./.travis-scripts/verify-detect-exceptions.sh
-  - ./.travis-scripts/verify-verbose-in-elixir-syntax.sh
+  - PROPCHECK_DEBUG=1 PROPCHECK_VERBOSE=1 PROPCHECK_NUMTESTS=200 PROPCHECK_SEARCH_STEPS=1000 mix tests --cover --trace
   - mix credo --strict

README.md

@@ -130,6 +130,13 @@ Please use the GitHub issue tracker for
 * bug reports and for
 * submitting pull requests
 
+Before submitting a pull request, please use Credo to ensure code consistency
+and run `mix tests` to check PropCheck itself. `mix tests` is a Mix alias that
+runs regular tests (via `mix test`) and some external tests (via another Mix
+alias `mix test_ext`) which test PropCheck's side effects, like storing
+counterexamples or proper output format, that can't be easily tested using
+regular tests.
+
 ## License
 
 PropCheck is provided under the GPL 3 License due to its intimate use of PropEr

mix.exs

@@ -18,6 +18,7 @@ defmodule PropCheck.Mixfile do
      description: description(),
      propcheck: [counter_examples: "_build/propcheck.ctx"],
      aliases: aliases(),
+     preferred_cli_env: [tests: :test, test_ext: :test],
      deps: deps()]
   end
 
@@ -50,7 +51,11 @@ defmodule PropCheck.Mixfile do
   defp elixirc_paths(_),     do: ["lib"]
 
   def aliases do
-    [clean: ["clean", "propcheck.clean"]]
+    [
+      clean: ["clean", "propcheck.clean"],
+      test_ext: &external_tests/1,
+      tests: ["test_ext", "test"],
+    ]
   end
 
   # Dependencies can be Hex packages:
@@ -72,4 +77,17 @@ defmodule PropCheck.Mixfile do
       {:credo, "~> 1.0.0", only: [:dev, :test], runtime: false}
     ]
   end
+
+  defp external_tests(_args) do
+    run = fn arg ->
+      r = Mix.shell().cmd(arg)
+      r > 0 && System.at_exit(fn _ -> exit({:shutdown, r}) end)
+      r
+    end
+
+    run.("./test/verify_storing_counterexamples.sh")
+    run.("./test/verify-verbose.sh")
+    run.("./test/verify-detect-exceptions.sh")
+    run.("./test/verify-verbose-in-elixir-syntax.sh")
+  end
 end

test/basic_types_test.exs

@@ -1,6 +1,7 @@
 defmodule PropCheck.Test.BasicTypes do
-  use ExUnit.Case
-  use PropCheck
+  use ExUnit.Case, async: true
+  use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
+  import PropCheck.TestHelpers, except: [config: 0]
 
   doctest(PropCheck.BasicTypes)
 

test/cache_dsl_test.exs

@@ -3,16 +3,17 @@ defmodule PropCheck.Test.Cache.DSL do
   Testing of the cache based on the EQC api and PropEr as backend.
   """
 
-  use ExUnit.Case
-  use PropCheck
+  use ExUnit.Case, async: true
+  use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
   use PropCheck.StateM.DSL
+  import PropCheck.TestHelpers, except: [config: 0]
 
   alias PropCheck.Test.Cache
   # require Logger
 
   @cache_size 10
 
-  property "run the sequential cache", [:verbose] do
+  property "run the sequential cache" do
     forall cmds <- commands(__MODULE__) do
       # Logger.debug "Commands to run: #{inspect cmds}"
       Cache.start_link(@cache_size)
@@ -28,16 +29,16 @@ defmodule PropCheck.Test.Cache.DSL do
           State: #{inspect events.state, pretty: true}
           Result: #{inspect events.result, pretty: true}
           """)
-      |> aggregate(command_names cmds)
-      |> collect(events
-        |> history_of_states()
-        |> Enum.map(fn model -> model.count end)
-        |> Enum.max())
+      # |> aggregate(command_names cmds)
+      # |> collect(events
+      #   |> history_of_states()
+      #   |> Enum.map(fn model -> model.count end)
+      #   |> Enum.max())
     end
   end
 
   @tag will_fail: true
-  property "run the misconfigured sequential cache", [:verbose] do
+  property "run the misconfigured sequential cache" do
     forall cmds <- commands(__MODULE__) do
       # Logger.debug "Commands to run: #{inspect cmds}"
       # Cache size is half as big expected, so we will find some cache misses,
@@ -55,11 +56,11 @@ defmodule PropCheck.Test.Cache.DSL do
           State: #{inspect events.state, pretty: true}
           Result: #{inspect events.result, pretty: true}
           """)
-      |> aggregate(command_names cmds)
-      |> collect(events
-        |> history_of_states()
-        |> Enum.map(fn model -> model.count end)
-        |> Enum.max())
+      # |> aggregate(command_names cmds)
+      # |> collect(events
+      #   |> history_of_states()
+      #   |> Enum.map(fn model -> model.count end)
+      #   |> Enum.max())
     end
     # |> fails
   end

test/counter_dsl_test.exs

@@ -12,9 +12,10 @@ defmodule PropCheck.Test.CounterDSL do
   These states are reflected in the commands and for determining the
   weights of the commands.
   """
-  use PropCheck
+  use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
   use PropCheck.StateM.DSL
-  use ExUnit.Case
+  use ExUnit.Case, async: true
+  import PropCheck.TestHelpers, except: [config: 0]
   import ExUnit.CaptureIO
   require Logger
 
@@ -26,7 +27,7 @@ defmodule PropCheck.Test.CounterDSL do
   ### The properties
   #########################################################################
 
-  property "infinity counter works fine", [:verbose] do
+  property "infinity counter works fine" do
     forall cmds <- commands(__MODULE__) do
       trap_exit do
         {:ok, _pid} = Counter.start_link()
@@ -41,14 +42,14 @@ defmodule PropCheck.Test.CounterDSL do
             Env: #{inspect events.env, pretty: true}
             Result: #{inspect events.result, pretty: true}
             """)
-        |> aggregate(command_names cmds)
-        |> measure("length of commands", length(cmds))
+        # |> aggregate(command_names cmds)
+        # |> measure("length of commands", length(cmds))
       end
     end
   end
 
   @tag will_fail: true
-  property "modulo counter does not increment infinite times", [:verbose] do
+  property "modulo counter does not increment infinite times" do
     Logger.error "This property will fail in order to show the shrinking results"
     Logger.error "It is excluded from the set of test for the travis build"
     forall cmds <- commands(__MODULE__) do
@@ -65,8 +66,8 @@ defmodule PropCheck.Test.CounterDSL do
             Env: #{inspect events.env, pretty: true}
             Result: #{inspect events.result, pretty: true}
             """)
-        |> aggregate(command_names cmds)
-        |> measure("length of commands", length(cmds))
+        # |> aggregate(command_names cmds)
+        # |> measure("length of commands", length(cmds))
       end
     end
   end

test/counterstrike_test.exs

@@ -1,13 +1,14 @@
 defmodule PropCheck.Test.CounterStrikeTest do
 
-  use ExUnit.Case
-  use PropCheck
+  use ExUnit.Case, async: true
+  use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
+  import PropCheck.TestHelpers, except: [config: 0]
   require Logger
 
   alias PropCheck.CounterStrike
 
   setup do
-    IO.puts "This is a setup callback for #{inspect self()}"
+    debugln "This is a setup callback for #{inspect self()}"
     filename = "counterstrike_test.#{System.unique_integer([:positive, :monotonic])}.dets"
     path = Path.join(Mix.Project.build_path(), filename)
     File.rm(path)
@@ -49,26 +50,31 @@ defmodule PropCheck.Test.CounterStrikeTest do
     end
   end
 
+  #
+  # helper for "PropEr error is not stored" test, can't be inside the tests
+  # because it triggers strange error in ExUnit when `async` is true and
+  # `--trace` options is enabled. Bug is already fixed in Elixir master
+  defmodule Helper do
+    use ExUnit.Case
+    use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
+    import PropCheck.TestHelpers, except: [config: 0]
+
+    @tag will_fail: true # must be run manually
+    property "cant_generate" do
+      # Check that no counterexample is stored if PropEr reported an error
+      gen = such_that b <- false, when: b
+      forall b <- gen do
+        b
+      end
+    end
+  end
+
   test "PropEr error is not stored" do
     # Check that an invalid counterexample is not stored. PropEr returns
     # {:error, _} on internal errors such as inability to generate a
     # value instance. Such errors cannot be used as counterexamples in
     # subsequent runs.
 
-    defmodule Helper do
-      use ExUnit.Case
-      use PropCheck
-
-      @tag will_fail: true # must be run manually
-      property "cant_generate" do
-        # Check that no counterexample is stored if PropEr reported an error
-        gen = such_that b <- false, when: b
-        forall b <- gen do
-          b
-        end
-      end
-    end
-
     assert_raise ExUnit.AssertionError, ~r/cant_generate/, fn ->
       apply(Helper, :"property cant_generate", [[]])
     end

test/let_test.exs

@@ -1,9 +1,10 @@
 defmodule PropCheck.Test.LetAndShrinks do
-  use ExUnit.Case
-  use PropCheck
+  use ExUnit.Case, async: true
+  use PropCheck, default_opts: &PropCheck.TestHelpers.config/0
+  import PropCheck.TestHelpers, except: [config: 0]
 
   @tag will_fail: true
-  property "a simple forall shrinks", [:verbose] do
+  property "a simple forall shrinks" do
     forall n <- integer(100, 1000) do
       n != 180
     end
@@ -16,7 +17,7 @@ defmodule PropCheck.Test.LetAndShrinks do
   end
 
   @tag will_fail: true
-  property "a simple let shrinks", [:verbose, numtests: 1_000] do
+  property "a simple let shrinks", [scale_numtests(10)] do
     pred = fn k -> k < 700 end
     forall n <- kilo_numbers() do
       pred.(n)
@@ -38,7 +39,7 @@ defmodule PropCheck.Test.LetAndShrinks do
     end
 
     @tag will_fail: true
-    property "a simple integer shrinks in SM DSL", [numtests: 1000] do
+    property "a simple integer shrinks in SM DSL", [scale_numtests(10)] do
       forall cmds <- commands(__MODULE__) do
           events = run_commands(cmds)
           (events.result == :ok)
@@ -49,7 +50,7 @@ defmodule PropCheck.Test.LetAndShrinks do
               Env: #{inspect events.env, pretty: true}
               Result: #{inspect events.result, pretty: true}
               """)
-          |> aggregate(command_names cmds)
+          # |> aggregate(command_names cmds)
       end
     end
 
@@ -75,7 +76,7 @@ defmodule PropCheck.Test.LetAndShrinks do
     end
 
     @tag will_fail: true
-    property "a simple let will shrink in SM DSL", [numtests: 1000] do
+    property "a simple let will shrink in SM DSL", [scale_numtests(10)] do
       forall cmds <- commands(__MODULE__) do
           events = run_commands(cmds)
           (events.result == :ok)
@@ -86,7 +87,7 @@ defmodule PropCheck.Test.LetAndShrinks do
               Env: #{inspect events.env, pretty: true}
               Result: #{inspect events.result, pretty: true}
               """)
-          |> aggregate(command_names cmds)
+          # |> aggregate(command_names cmds)
       end
     end
 
@@ -115,7 +116,7 @@ defmodule PropCheck.Test.LetAndShrinks do
     def precondition(_state, _call), do: true
 
     @tag will_fail: true
-    property "let shrinks in PropEr's native SM", [numtests: 1000] do
+    property "let shrinks in PropEr's native SM", [scale_numtests(10)] do
       forall cmds <- commands(__MODULE__) do
           {history, state, result} = run_commands(__MODULE__, cmds)
           (result == :ok)
@@ -125,7 +126,7 @@ defmodule PropCheck.Test.LetAndShrinks do
               State: #{inspect state, pretty: true}
               Result: #{inspect result, pretty: true}
               """)
-          |> aggregate(command_names cmds)
+          # |> aggregate(command_names cmds)
       end
     end
 
@@ -151,7 +152,7 @@ defmodule PropCheck.Test.LetAndShrinks do
     end
 
     @tag will_fail: true
-    property "a let with a list shrinks in SM DSL", [numtests: 1000] do
+    property "a let with a list shrinks in SM DSL", [scale_numtests(10)] do
       forall cmds <- commands(__MODULE__) do
           events = run_commands(cmds)
           (events.result == :ok)
@@ -162,7 +163,7 @@ defmodule PropCheck.Test.LetAndShrinks do
               Env: #{inspect events.env, pretty: true}
               Result: #{inspect events.result, pretty: true}
               """)
-          |> aggregate(command_names cmds)
+          # |> aggregate(command_names cmds)
       end
     end