Bug related to Random.int

bam.opam

@@ -14,6 +14,7 @@ depends: [
   "ocaml"
   "dune" {>= "3.7"}
   "pringo"
+  "zarith"
   "odoc" {with-doc}
   "tezt" {with-test}
 ]

dune-project

@@ -19,7 +19,7 @@
  (name bam)
  (synopsis "A property-based testing library with internal shrinking")
  (description "A property-based testing allowing to define generators with internal shrinking easily")
- (depends ocaml dune pringo (odoc :with-doc) (tezt :with-test))
+ (depends ocaml dune pringo zarith (odoc :with-doc) (tezt :with-test))
  (tags
   (test pbt shrinking internal)))
 

example/1-simple/README.md

@@ -0,0 +1,27 @@
+# 1-simple
+
+A simple example on how to use the library.
+
+```ocaml
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property _x = Ok () in
+  Pbt.register ~__FILE__ ~title:"Simple example of bam" ~tags:["bam"; "simple"]
+    ~gen ~property ()	
+```
+
+You can run the example with:
+
+```bash
+$ dune exec example/main.exe -- simple
+[14:05:37.035] [SUCCESS] (1/1) Simple example of bam
+```
+
+The test run successfully!
+
+**Next steps:**
+- The next example [2-simple-failure]() will investigate how to debug
+  a test when the property fails.
+

example/1-simple/dune

@@ -0,0 +1,3 @@
+(library
+  (name simple)
+  (libraries tezt-bam))

example/1-simple/simple.ml

@@ -0,0 +1,7 @@
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property _x = Ok () in
+  Pbt.register ~__FILE__ ~title:"Simple example of bam" ~tags:["bam"; "simple"]
+    ~gen ~property ()

example/2-simple-failure/README.md

@@ -0,0 +1,64 @@
+# 2-simple-failure
+
+Let's try *BAM* when a failure arises.
+
+```ocaml
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property x =
+    if x < 100 then Ok () else Error (`Fail "integer is not smallar than 100")
+  in
+  Pbt.register ~pp:Format.pp_print_int ~__FILE__
+    ~title:"Simple failure example with bam" ~tags:["bam"; "simple_failure"]
+    ~gen ~property ()
+```
+
+Do note this time we have provided an optional parameter `pp` to enable `Tezt` to print the counter-example found.
+
+You can run the example as follows:
+
+```bash
+dune exec example/main.exe -- simple_failure
+[15:30:55.229] [pbt] \  | /
+[15:30:55.229] [pbt] - BAM -
+[15:30:55.229] [pbt]  / | \ 
+[15:30:55.229] [pbt] Please run the test again with option --shrink to get a smaller counter-example
+[15:30:55.229] [pbt] Counter example found:
+[15:30:55.229] [pbt] 323253551143260932
+[15:30:55.229] [error] Test failed with error:
+[15:30:55.229] [error] integer is not smallar than 100
+[15:30:55.229] [FAILURE] (1/1, 1 failed) Simple failure example with bam
+[15:30:55.229] Try again with: _build/default/example/main.exe --verbose --file example/2-simple-failure/simple_failure.ml --title 'Simple failure example with bam' --seed 555586205
+```
+
+This time `Tezt` is more verbose. On a failure, the counter-example
+found is printed. By default, shrinking is not enabled, hence the
+counter-example found can be rather big.
+
+Moreover, the error raised associated to the counter-example is
+printed.
+
+If you want to run this example again using the shrinking of bam, you can use the option `--shrink`. To be sue the very same initial counter-example will be printed, you can use the seed given by Tezt:
+
+```ocaml
+$ dune exec example/main.exe -- simple_failure --shrink --seed 555586205
+[15:36:00.803] [pbt] \  | /         
+[15:36:00.803] [pbt] - BAM -
+[15:36:00.803] [pbt]  / | \ 
+[15:36:00.803] [pbt] Counter example found:
+[15:36:00.803] [pbt] 100
+[15:36:00.803] [error] Test failed with error:
+[15:36:00.803] [error] integer is not smallar than 100
+[15:36:00.803] [FAILURE] (1/1, 1 failed) Simple failure example with bam
+[15:36:00.803] Try again with: _build/default/example/main.exe --verbose --file example/2-simple-failure/simple_failure.ml --title 'Simple failure example with bam' --seed 555586205
+```
+
+At this stage, we see the shrinking heuristic allowed us to find a
+smaller-counter example which is `100`. Such a counter-example can be
+a good candidate for starting debugging the test.
+
+**Next steps:**
+- The next example [3-debugging]() will investigate how bam can help
+  for debugging

example/2-simple-failure/dune

@@ -0,0 +1,3 @@
+(library
+  (name simple_failure)
+  (libraries tezt-bam))

example/2-simple-failure/simple_failure.ml

@@ -0,0 +1,10 @@
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property x =
+    if x < 100 then Ok () else Error (`Fail "integer is not smallar than 100")
+  in
+  Pbt.register ~pp:Format.pp_print_int ~__FILE__
+    ~title:"Simple failure example with bam" ~tags:["bam"; "simple_failure"]
+    ~gen ~property ()

example/3-debugging/README.md

@@ -0,0 +1,74 @@
+# 3-debugging
+
+Let's take a simple example and let's add a `printf`:
+
+```ocaml
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property x =
+    Format.eprintf "HEY: %d@." x ;
+    if x < 100 && x <> 16 then Ok ()
+    else Error (`Fail "integer is not smallar than 100")
+  in
+  Pbt.register ~pp:Format.pp_print_int ~__FILE__ ~title:"Debugging with bam"
+    ~tags:["bam"; "debugging"] ~gen ~property ()
+```
+
+And now run it:
+
+```bash
+$ dune exec example/main.exe -- debugging --shrink
+HEY: 1531078954999384395            
+[15:46:01.805] [pbt] \  | /
+[15:46:01.805] [pbt] - BAM -
+[15:46:01.805] [pbt]  / | \ 
+[15:46:01.805] [pbt] Counter example found:
+[15:46:01.805] [pbt] 100
+[15:46:01.805] [error] Test failed with error:
+[15:46:01.805] [error] integer is not smallar than 100
+[15:46:01.805] [FAILURE] (1/1, 1 failed) Debugging with bam
+[15:46:01.805] Try again with: _build/default/example/main.exe --verbose --file example/3-debugging/debugging.ml --title 'Debugging with bam' --seed 830377664
+```
+
+Since we have used the shrinking, the property function run multiple
+times. However, only one line appear on stdout. The reason is that by
+default, **BAM** captures stdout/stderr and run one last time the test
+on the smallest counter-example found. This is to ease debugging so
+that what is seen is only relevant output on the counter-example
+found. Such behaviour can be relaxed by specifying the option `--no-capture`:
+
+```bash
+saroupille@saroupille-debian:~/Git/bam$ dune exec example/main.exe -- debugging --shrink --no-capture
+HEY: 3365002382978735690
+HEY: 0
+HEY: 1
+HEY: 2
+HEY: 4
+HEY: 8
+HEY: 16
+HEY: 9
+HEY: 10
+HEY: 11
+HEY: 13
+HEY: 15
+HEY: 16
+[16:08:35.821] [pbt] \  | /
+[16:08:35.821] [pbt] - BAM -
+[16:08:35.821] [pbt]  / | \ 
+[16:08:35.821] [pbt] Counter example found:
+[16:08:35.821] [pbt] 16
+[16:08:35.821] [error] Test failed with error:
+[16:08:35.821] [error] integer is not smallar than 100
+[16:08:35.821] [FAILURE] (1/1, 1 failed) Debugging with bam
+[16:08:35.821] Try again with: _build/default/example/main.exe --verbose --file example/3-debugging/debugging.ml --title 'Debugging with bam' --seed 353027053
+```
+
+We see that during the shrinking, a lot of values are tested. Since
+[bam] runs twice the property on the counter-example, this explains
+why we see twice `16`. But do note that the first time `16` is
+executed other attempts are made to find a smaller counter-example.
+
+**Next steps:**
+- The next example [4-writing-generators]() will investigate how bam can help

example/3-debugging/debugging.ml

@@ -0,0 +1,11 @@
+open Tezt_bam
+
+let register () =
+  let gen = Std.int () in
+  let property x =
+    Format.eprintf "HEY: %d@." x ;
+    if x < 100 && x <> 16 then Ok ()
+    else Error (`Fail "integer is not smallar than 100")
+  in
+  Pbt.register ~pp:Format.pp_print_int ~__FILE__ ~title:"Debugging with bam"
+    ~tags:["bam"; "debugging"] ~gen ~property ()

example/3-debugging/dune

@@ -0,0 +1,3 @@
+(library
+  (name debugging)
+  (libraries tezt-bam))

example/4-writing-generators/README.md

@@ -0,0 +1,61 @@
+# 4-writing-generators
+
+One can take advantage of the monadic interface of *bam* to define
+generators.
+
+```ocaml
+open Tezt_bam
+
+type t = A of {x: int; b: bool} | B of string
+
+let pp fmt = function
+  | A {x; b} ->
+      Format.fprintf fmt "A {x=%d;%b}" x b
+  | B str ->
+      Format.fprintf fmt "B %s" str
+
+let register () =
+  let gen =
+    let open Std.Syntax in
+    let a =
+      let* x = Std.int () in
+      let* b = Std.bool () in
+      return (A {x; b})
+    in
+    let b =
+      let* str = Std.string ~size:(Std.int ~min:1 ~max:10 ()) () in
+      return (B str)
+    in
+    Std.oneof [(1, a); (1, b)]
+  in
+  let property = function
+    | B str ->
+        if String.get str 0 = 'c' then Ok ()
+        else Error (`Fail "string did not start with 'c'")
+    | A _ ->
+        Ok ()
+  in
+  Pbt.register ~pp ~__FILE__ ~title:"Writing generators"
+    ~tags:["bam"; "writing_generators"]
+    ~gen ~property ()
+```
+
+And now run it:
+
+```bash
+$ dune exec example/main.exe -- writing_generators --shrink
+[16:27:21.716] [pbt] \  | /         
+[16:27:21.716] [pbt] - BAM -
+[16:27:21.716] [pbt]  / | \ 
+[16:27:21.716] [pbt] Counter example found:
+[16:27:21.716] [pbt] B a
+[16:27:21.716] [error] Test failed with error:
+[16:27:21.716] [error] string did not start with 'c'
+[16:27:21.716] [FAILURE] (1/1, 1 failed) Writing generators
+```
+
+With *bam* it is encouraged to extensively use and abuse of the
+monadic interface. The library ensures it will behave well with
+respect to shrinking.
+
+

example/4-writing-generators/dune

@@ -0,0 +1,3 @@
+(library
+  (name writing_generators)
+  (libraries tezt-bam))

example/4-writing-generators/writing_generators.ml

@@ -0,0 +1,34 @@
+open Tezt_bam
+
+type t = A of {x: int; b: bool} | B of string
+
+let pp fmt = function
+  | A {x; b} ->
+      Format.fprintf fmt "A {x=%d;%b}" x b
+  | B str ->
+      Format.fprintf fmt "B %s" str
+
+let register () =
+  let gen =
+    let open Std.Syntax in
+    let a =
+      let* x = Std.int () in
+      let* b = Std.bool () in
+      return (A {x; b})
+    in
+    let b =
+      let* str = Std.string ~size:(Std.int ~min:1 ~max:10 ()) () in
+      return (B str)
+    in
+    Std.oneof [(1, a); (1, b)]
+  in
+  let property = function
+    | B str ->
+        if String.get str 0 = 'c' then Ok ()
+        else Error (`Fail "string did not start with 'c'")
+    | A _ ->
+        Ok ()
+  in
+  Pbt.register ~pp ~__FILE__ ~title:"Writing generators"
+    ~tags:["bam"; "writing_generators"]
+    ~gen ~property ()

example/dune

@@ -0,0 +1,3 @@
+(executable
+ (name main)
+ (libraries tezt-bam simple simple_failure writing_generators debugging))

example/main.ml

@@ -0,0 +1,8 @@
+open Tezt_bam
+
+let () =
+  Simple.register () ;
+  Simple_failure.register () ;
+  Debugging.register () ;
+  Writing_generators.register () ;
+  Test.run ()

lib_bam/dune

@@ -1,7 +1,7 @@
 (library
  (name bam)
  (public_name bam)
- (libraries pringo))
+ (libraries pringo zarith))
 
 (documentation
  (package bam))