From 67296ea206074cc5bd364b9abdb33101cb6bbf34 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 09:44:54 +0100 Subject: [PATCH 01/10] docs: include test dependency and random test information --- src/content/docs/testing/testing.mdx | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 4f8e1af..0f0edd8 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -418,3 +418,86 @@ abstract class TestTag { ``` ::: + +## Do not share state between tests + +Tests should not share state between them to ensure they remain independent, reliable, and predictable. + +When tests share state—such as variables, test's outcome and order can influence another's, leading to inconsistent results and making it difficult to identify the root cause of failures. + +Bad ❗️ + +```dart +class _Counter { + int value = 0; + void increment() => value++; + void decrement() => value--; +} + +void main() { + group(_Counter, () { + final _Counter counter = _Counter(); + + test('increment', () { + counter.increment(); + expect(counter.value, 1); + }); + + test('decrement', () { + counter.decrement(); + // The expectation only succeeds when the previous test executes first. + expect(counter.value, 0); + }); + }); +} + +``` + + +Good ✅ + +```dart +class _Counter { + int value = 0; + void increment() => value++; + void decrement() => value--; +} + +void main() { + group(_Counter, () { + late _Counter counter; + + setUp(() => counter = _Counter()); + + test('increment', () { + counter.increment(); + expect(counter.value, 1); + }); + + test('decrement', () { + counter.decrement(); + expect(counter.value, -1); + }); + }); +} + +``` + +## Use random test ordering + +Running tests in an arbitrary (random) order is a crucial practice to identify and eliminate flaky tests, specially during continuous integration. + +Flaky tests are those that pass or fail inconsistently without changes to the codebase, often due to unintended dependencies between tests. + +By running tests in random order, these hidden dependencies are more likely to be exposed, as any reliance on the order of test execution becomes clear when tests fail unexpectedly. + +This practice ensures that tests do not share state or rely on the side effects of previous tests, leading to a more robust and reliable test suite. Overall, the tests become easier to trust and reduce debugging time caused by intermittent test failures. + + +Good ✅ + +```sh +# Randomize test ordering using the --test-randomize-ordering-seed flag +flutter test --test-randomize-ordering-seed random +dart test --test-randomize-ordering-seed random +``` From 260966b5dfc58befbbf3508fe3134fbaf4d60dfc Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 09:52:02 +0100 Subject: [PATCH 02/10] chore: format --- src/content/docs/testing/testing.mdx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 0f0edd8..e5b897e 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -421,7 +421,7 @@ abstract class TestTag { ## Do not share state between tests -Tests should not share state between them to ensure they remain independent, reliable, and predictable. +Tests should not share state between them to ensure they remain independent, reliable, and predictable. When tests share state—such as variables, test's outcome and order can influence another's, leading to inconsistent results and making it difficult to identify the root cause of failures. @@ -453,7 +453,6 @@ void main() { ``` - Good ✅ ```dart @@ -489,11 +488,10 @@ Running tests in an arbitrary (random) order is a crucial practice to identify a Flaky tests are those that pass or fail inconsistently without changes to the codebase, often due to unintended dependencies between tests. -By running tests in random order, these hidden dependencies are more likely to be exposed, as any reliance on the order of test execution becomes clear when tests fail unexpectedly. +By running tests in random order, these hidden dependencies are more likely to be exposed, as any reliance on the order of test execution becomes clear when tests fail unexpectedly. This practice ensures that tests do not share state or rely on the side effects of previous tests, leading to a more robust and reliable test suite. Overall, the tests become easier to trust and reduce debugging time caused by intermittent test failures. - Good ✅ ```sh From 462071bc978d45d0fe7154f697789f3b9b19adca Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 09:55:07 +0100 Subject: [PATCH 03/10] chore: use option over flag --- src/content/docs/testing/testing.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index e5b897e..5e1b4c1 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -495,7 +495,7 @@ This practice ensures that tests do not share state or rely on the side effects Good ✅ ```sh -# Randomize test ordering using the --test-randomize-ordering-seed flag +# Randomize test ordering using the --test-randomize-ordering-seed option flutter test --test-randomize-ordering-seed random dart test --test-randomize-ordering-seed random ``` From e7cf2c587415e74877c3ad7707d0ca2797930f70 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 10:30:58 +0100 Subject: [PATCH 04/10] chore: add very_good ad an example --- src/content/docs/testing/testing.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 5e1b4c1..7db83be 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -498,4 +498,5 @@ Good ✅ # Randomize test ordering using the --test-randomize-ordering-seed option flutter test --test-randomize-ordering-seed random dart test --test-randomize-ordering-seed random +very_good test --test-randomize-ordering-seed ``` From b025f2fe97bc23771336ec31c3d72b1f72a2e8f7 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 10:34:33 +0100 Subject: [PATCH 05/10] refactor: use tab view --- src/content/docs/testing/testing.mdx | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 7db83be..67b3f3c 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -425,8 +425,9 @@ Tests should not share state between them to ensure they remain independent, rel When tests share state—such as variables, test's outcome and order can influence another's, leading to inconsistent results and making it difficult to identify the root cause of failures. -Bad ❗️ + + ```dart class _Counter { int value = 0; @@ -436,7 +437,9 @@ class _Counter { void main() { group(_Counter, () { - final _Counter counter = _Counter(); + late _Counter counter; + + setUp(() => counter = _Counter()); test('increment', () { counter.increment(); @@ -445,16 +448,14 @@ void main() { test('decrement', () { counter.decrement(); - // The expectation only succeeds when the previous test executes first. - expect(counter.value, 0); + expect(counter.value, -1); }); }); } ``` - -Good ✅ - + + ```dart class _Counter { int value = 0; @@ -464,9 +465,7 @@ class _Counter { void main() { group(_Counter, () { - late _Counter counter; - - setUp(() => counter = _Counter()); + final _Counter counter = _Counter(); test('increment', () { counter.increment(); @@ -475,12 +474,15 @@ void main() { test('decrement', () { counter.decrement(); - expect(counter.value, -1); + // The expectation only succeeds when the previous test executes first. + expect(counter.value, 0); }); }); } ``` + + ## Use random test ordering @@ -492,7 +494,9 @@ By running tests in random order, these hidden dependencies are more likely to b This practice ensures that tests do not share state or rely on the side effects of previous tests, leading to a more robust and reliable test suite. Overall, the tests become easier to trust and reduce debugging time caused by intermittent test failures. -Good ✅ + + + ```sh # Randomize test ordering using the --test-randomize-ordering-seed option @@ -500,3 +504,5 @@ flutter test --test-randomize-ordering-seed random dart test --test-randomize-ordering-seed random very_good test --test-randomize-ordering-seed ``` + + \ No newline at end of file From a391af5a4aa1c4498a335e1c35eb3ba0de45ecfd Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 10:34:44 +0100 Subject: [PATCH 06/10] chore: format tab view --- src/content/docs/testing/testing.mdx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 67b3f3c..2b726e7 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -425,7 +425,6 @@ Tests should not share state between them to ensure they remain independent, rel When tests share state—such as variables, test's outcome and order can influence another's, leading to inconsistent results and making it difficult to identify the root cause of failures. - ```dart @@ -436,8 +435,8 @@ class _Counter { } void main() { - group(_Counter, () { - late _Counter counter; +group(\_Counter, () { +late \_Counter counter; setUp(() => counter = _Counter()); @@ -450,10 +449,11 @@ void main() { counter.decrement(); expect(counter.value, -1); }); - }); + +}); } -``` +```` ```dart @@ -480,7 +480,8 @@ void main() { }); } -``` +```` + @@ -494,7 +495,6 @@ By running tests in random order, these hidden dependencies are more likely to b This practice ensures that tests do not share state or rely on the side effects of previous tests, leading to a more robust and reliable test suite. Overall, the tests become easier to trust and reduce debugging time caused by intermittent test failures. - @@ -502,7 +502,8 @@ This practice ensures that tests do not share state or rely on the side effects # Randomize test ordering using the --test-randomize-ordering-seed option flutter test --test-randomize-ordering-seed random dart test --test-randomize-ordering-seed random -very_good test --test-randomize-ordering-seed +very_good test --test-randomize-ordering-seed ``` + - \ No newline at end of file + From f4e0cd573d91b3e986abed73b065f6c26f230479 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 10:41:14 +0100 Subject: [PATCH 07/10] chore: ignore formatting changes --- src/content/docs/testing/testing.mdx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 2b726e7..f1ea12a 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -435,8 +435,8 @@ class _Counter { } void main() { -group(\_Counter, () { -late \_Counter counter; + group(_Counter, () { + late _Counter counter; setUp(() => counter = _Counter()); @@ -449,8 +449,7 @@ late \_Counter counter; counter.decrement(); expect(counter.value, -1); }); - -}); + }); } ```` From 265243c19a9e5f30edd82eaebcc1407b321378f8 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 10:50:03 +0100 Subject: [PATCH 08/10] chore: re-format --- src/content/docs/testing/testing.mdx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index f1ea12a..f56a1a3 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -427,6 +427,7 @@ When tests share state—such as variables, test's outcome and order can influen + ```dart class _Counter { int value = 0; @@ -451,10 +452,11 @@ void main() { }); }); } +``` -```` + ```dart class _Counter { int value = 0; @@ -478,8 +480,7 @@ void main() { }); }); } - -```` +``` From 5714f261087a18a13cdc792e784e60da6551771d Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Tue, 3 Sep 2024 15:27:56 +0100 Subject: [PATCH 09/10] Update src/content/docs/testing/testing.mdx --- src/content/docs/testing/testing.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index f56a1a3..3cfbe66 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -502,7 +502,7 @@ This practice ensures that tests do not share state or rely on the side effects # Randomize test ordering using the --test-randomize-ordering-seed option flutter test --test-randomize-ordering-seed random dart test --test-randomize-ordering-seed random -very_good test --test-randomize-ordering-seed +very_good test --test-randomize-ordering-seed random ``` From d557b370211d66123923d55a655e48fe5ff43089 Mon Sep 17 00:00:00 2001 From: Alejandro Santiago Date: Thu, 5 Sep 2024 08:02:18 +0100 Subject: [PATCH 10/10] Update src/content/docs/testing/testing.mdx Co-authored-by: Joanna May --- src/content/docs/testing/testing.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/testing/testing.mdx b/src/content/docs/testing/testing.mdx index 3cfbe66..283516f 100644 --- a/src/content/docs/testing/testing.mdx +++ b/src/content/docs/testing/testing.mdx @@ -423,7 +423,7 @@ abstract class TestTag { Tests should not share state between them to ensure they remain independent, reliable, and predictable. -When tests share state—such as variables, test's outcome and order can influence another's, leading to inconsistent results and making it difficult to identify the root cause of failures. +When tests share state (such as relying on static members), the order that tests are executed in can cause inconsistent results. Implicitly sharing state between tests means that tests no longer exist in isolation and are influenced by each other. As a result, it can be difficult to identify the root cause of test failures.