Introduce commentCharacter attribute in @Csv{File}Source (#5031)

The `@CsvSource` and `@CsvFileSource` annotations now allow specifying a
custom comment character using the new `commentCharacter` attribute to
resolve conflicts with using `#` as delimiter.

Resolves #5028.

Author: vdmitrienko

documentation/src/docs/asciidoc/release-notes/release-notes-6.0.1.adoc

@@ -35,7 +35,12 @@ repository on GitHub.
 [[release-notes-6.0.1-junit-jupiter-bug-fixes]]
 ==== Bug Fixes
 
-* ❓
+* A regression introduced in version 6.0.0 caused an exception when using `@CsvSource` or
+  `@CsvFileSource` if the `delimiter` or `delimiterString` attribute was set to `+++#+++`.
+  This occurred because `+++#+++` was used as the default comment character without an
+  option to change it. To resolve this, a new `commentCharacter` attribute has been added
+  to both annotations. Its default value remains `+++#+++`, but it can now be customized
+  to avoid conflicts with other control characters.
 
 [[release-notes-6.0.1-junit-jupiter-deprecations-and-breaking-changes]]
 ==== Deprecations and Breaking Changes
@@ -45,7 +50,8 @@ repository on GitHub.
 [[release-notes-6.0.1-junit-jupiter-new-features-and-improvements]]
 ==== New Features and Improvements
 
-* ❓
+* The `@CsvSource` and `@CsvFileSource` annotations now allow specifying
+  a custom comment character using the new `commentCharacter` attribute.
 
 
 [[release-notes-6.0.1-junit-vintage]]

documentation/src/docs/asciidoc/user-guide/writing-tests.adoc

@@ -2270,12 +2270,16 @@ The generated display names for the previous example include the CSV header name
 ----
 
 In contrast to CSV records supplied via the `value` attribute, a text block can contain
-comments. Any line beginning with a `+++#+++` symbol will be treated as a comment and
-ignored. Note, however, that the `+++#+++` symbol must be the first character on the line
-without any leading whitespace. It is therefore recommended that the closing text block
-delimiter (`"""`) be placed either at the end of the last line of input or on the
-following line, left aligned with the rest of the input (as can be seen in the example
-below which demonstrates formatting similar to a table).
+comments. Any line beginning with the value of the `commentCharacter` attribute (`+++#+++`
+by default) will be treated as a comment and ignored. Note that there is one exception
+to this rule: if the comment character appears within a quoted field, it loses
+its special meaning.
+
+The comment character must be the first character on the line without any leading
+whitespace. It is therefore recommended that the closing text block delimiter (`"""`)
+be placed either at the end of the last line of input or on the following line,
+left aligned with the rest of the input (as can be seen in the example below which
+demonstrates formatting similar to a table).
 
 [source,java,indent=0]
 ----
@@ -2325,8 +2329,8 @@ The default delimiter is a comma (`,`), but you can use another character by set
 cannot be set simultaneously.
 
 .Comments in CSV files
-NOTE: Any line beginning with a `+++#+++` symbol will be interpreted as a comment and will
-be ignored.
+NOTE: Any line beginning with the value of the `commentCharacter` attribute (`+++#+++`
+by default) will be interpreted as a comment and will be ignored.
 
 [source,java,indent=0]
 ----

gradle/config/japicmp/accepted-breaking-changes.txt

@@ -0,0 +1,2 @@
+org.junit.jupiter.params.provider.CsvFileSource#commentCharacter
+org.junit.jupiter.params.provider.CsvSource#commentCharacter

junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvFileSource.java

@@ -10,6 +10,7 @@
 
 package org.junit.jupiter.params.provider;
 
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.apiguardian.api.API.Status.STABLE;
 
 import java.lang.annotation.Documented;
@@ -35,8 +36,8 @@
  * that the first record may optionally be used to supply CSV headers (see
  * {@link #useHeadersInDisplayName}).
  *
- * <p>Any line beginning with a {@code #} symbol will be interpreted as a comment
- * and will be ignored.
+ * <p>Any line beginning with a {@link #commentCharacter}
+ * will be interpreted as a comment and will be ignored.
  *
  * <p>The column delimiter (which defaults to a comma ({@code ,})) can be customized
  * via either {@link #delimiter} or {@link #delimiterString}.
@@ -63,6 +64,10 @@
  * column is trimmed by default. This behavior can be changed by setting the
  * {@link #ignoreLeadingAndTrailingWhitespace} attribute to {@code true}.
  *
+ * <p>Note that {@link #delimiter} (or {@link #delimiterString}),
+ * {@link #quoteCharacter}, and {@link #commentCharacter} are treated as
+ * <em>control characters</em> and must all be distinct.
+ *
  * <h2>Inheritance</h2>
  *
  * <p>This annotation is inherited to subclasses.
@@ -235,4 +240,22 @@
 	@API(status = STABLE, since = "5.10")
 	boolean ignoreLeadingAndTrailingWhitespace() default true;
 
+	/**
+	 * The character used to denote comments when reading the CSV files.
+	 *
+	 * <p>Any line that begins with this character will be treated as a comment
+	 * and ignored during parsing. Note that there is one exception to this rule:
+	 * if the comment character appears within a quoted field, it loses its
+	 * special meaning.
+	 *
+	 * <p>The comment character must be the first character on the line without
+	 * any leading whitespace.
+	 *
+	 * <p>Defaults to {@code '#'}.
+	 *
+	 * @since 6.0.1
+	 */
+	@API(status = EXPERIMENTAL, since = "6.0.1")
+	char commentCharacter() default '#';
+
 }

junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvReaderFactory.java

@@ -18,7 +18,9 @@
 import java.nio.charset.Charset;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Stream;
 
+import de.siegmar.fastcsv.reader.CommentStrategy;
 import de.siegmar.fastcsv.reader.CsvCallbackHandler;
 import de.siegmar.fastcsv.reader.CsvReader;
 import de.siegmar.fastcsv.reader.CsvRecord;
@@ -65,15 +67,20 @@ private static void validateDelimiter(char delimiter, String delimiterString, An
 
 	static CsvReader<? extends CsvRecord> createReaderFor(CsvSource csvSource, String data) {
 		String delimiter = selectDelimiter(csvSource.delimiter(), csvSource.delimiterString());
+		var commentStrategy = csvSource.textBlock().isEmpty() ? NONE : SKIP;
 		// @formatter:off
+		validateControlCharactersDiffer(
+				delimiter, csvSource.quoteCharacter(), csvSource.commentCharacter(), commentStrategy);
+
 		var builder = CsvReader.builder()
 				.skipEmptyLines(SKIP_EMPTY_LINES)
 				.trimWhitespacesAroundQuotes(TRIM_WHITESPACES_AROUND_QUOTES)
 				.allowExtraFields(ALLOW_EXTRA_FIELDS)
 				.allowMissingFields(ALLOW_MISSING_FIELDS)
 				.fieldSeparator(delimiter)
 				.quoteCharacter(csvSource.quoteCharacter())
-				.commentStrategy(csvSource.textBlock().isEmpty() ? NONE : SKIP);
+				.commentStrategy(commentStrategy)
+				.commentCharacter(csvSource.commentCharacter());
 
 		var callbackHandler = createCallbackHandler(
 				csvSource.emptyValue(),
@@ -90,15 +97,20 @@ static CsvReader<? extends CsvRecord> createReaderFor(CsvFileSource csvFileSourc
 			Charset charset) {
 
 		String delimiter = selectDelimiter(csvFileSource.delimiter(), csvFileSource.delimiterString());
+		var commentStrategy = SKIP;
 		// @formatter:off
+		validateControlCharactersDiffer(
+				delimiter, csvFileSource.quoteCharacter(), csvFileSource.commentCharacter(), commentStrategy);
+
 		var builder = CsvReader.builder()
 				.skipEmptyLines(SKIP_EMPTY_LINES)
 				.trimWhitespacesAroundQuotes(TRIM_WHITESPACES_AROUND_QUOTES)
 				.allowExtraFields(ALLOW_EXTRA_FIELDS)
 				.allowMissingFields(ALLOW_MISSING_FIELDS)
 				.fieldSeparator(delimiter)
 				.quoteCharacter(csvFileSource.quoteCharacter())
-				.commentStrategy(SKIP);
+				.commentStrategy(commentStrategy)
+				.commentCharacter(csvFileSource.commentCharacter());
 
 		var callbackHandler = createCallbackHandler(
 				csvFileSource.emptyValue(),
@@ -121,6 +133,26 @@ private static String selectDelimiter(char delimiter, String delimiterString) {
 		return DEFAULT_DELIMITER;
 	}
 
+	private static void validateControlCharactersDiffer(String delimiter, char quoteCharacter, char commentCharacter,
+			CommentStrategy commentStrategy) {
+
+		if (commentStrategy == NONE) {
+			Preconditions.condition(stringValuesUnique(delimiter, quoteCharacter),
+				() -> ("delimiter or delimiterString: '%s' and quoteCharacter: '%s' " + //
+						"must differ").formatted(delimiter, quoteCharacter));
+		}
+		else {
+			Preconditions.condition(stringValuesUnique(delimiter, quoteCharacter, commentCharacter),
+				() -> ("delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' " + //
+						"must all differ").formatted(delimiter, quoteCharacter, commentCharacter));
+		}
+	}
+
+	private static boolean stringValuesUnique(Object... values) {
+		long uniqueCount = Stream.of(values).map(String::valueOf).distinct().count();
+		return uniqueCount == values.length;
+	}
+
 	private static CsvCallbackHandler<? extends CsvRecord> createCallbackHandler(String emptyValue,
 			Set<String> nullValues, boolean ignoreLeadingAndTrailingWhitespaces, int maxCharsPerColumn,
 			boolean useHeadersInDisplayName) {

junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvSource.java

@@ -10,6 +10,7 @@
 
 package org.junit.jupiter.params.provider;
 
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.apiguardian.api.API.Status.STABLE;
 
 import java.lang.annotation.Documented;
@@ -62,6 +63,16 @@
  * physical line within the text block. Thus, if a CSV column wraps across a
  * new line in a text block, the column must be a quoted string.
  *
+ * <p>Note that {@link #delimiter} (or {@link #delimiterString}),
+ * {@link #quoteCharacter}, and {@link #commentCharacter} (when
+ * {@link #textBlock} is used) are treated as <em>control characters</em>.
+ *
+ * <ul>
+ *   <li>{@link #delimiter} and {@link #quoteCharacter} must always be distinct.</li>
+ *   <li>{@link #commentCharacter} must be distinct from the others only when
+ *   {@link #textBlock} is used.</li>
+ * </ul>
+ *
  * <h2>Inheritance</h2>
  *
  * <p>This annotation is inherited to subclasses.
@@ -132,17 +143,20 @@
 	 * {@link #useHeadersInDisplayName}).
 	 *
 	 * <p>In contrast to CSV records supplied via {@link #value}, a text block
-	 * can contain comments. Any line beginning with a hash tag ({@code #}) will
-	 * be treated as a comment and ignored. Note, however, that the {@code #}
-	 * symbol must be the first character on the line without any leading
-	 * whitespace. It is therefore recommended that the closing text block
+	 * can contain comments. Any line beginning with a {@link #commentCharacter}
+	 * will be treated as a comment and ignored. Note that there is one exception
+	 * to this rule: if the comment character appears within a quoted field,
+	 * it loses its special meaning.
+	 *
+	 * <p>The comment character must be the first character on the line without
+	 * any leading whitespace. It is therefore recommended that the closing text block
 	 * delimiter {@code """} be placed either at the end of the last line of
 	 * input or on the following line, vertically aligned with the rest of the
 	 * input (as can be seen in the example below).
 	 *
 	 * <p>Java's <a href="https://docs.oracle.com/en/java/javase/15/text-blocks/index.html">text block</a>
 	 * feature automatically removes <em>incidental whitespace</em> when the code
-	 * is compiled. However other JVM languages such as Groovy and Kotlin do not.
+	 * is compiled. However, other JVM languages such as Groovy and Kotlin do not.
 	 * Thus, if you are using a programming language other than Java and your text
 	 * block contains comments or new lines within quoted strings, you will need
 	 * to ensure that there is no leading whitespace within your text block.
@@ -296,4 +310,22 @@
 	@API(status = STABLE, since = "5.10")
 	boolean ignoreLeadingAndTrailingWhitespace() default true;
 
+	/**
+	 * The character used to denote comments in a {@linkplain #textBlock text block}.
+	 *
+	 * <p>Any line that begins with this character will be treated as a comment
+	 * and ignored during parsing. Note that there is one exception to this rule:
+	 * if the comment character appears within a quoted field, it loses its
+	 * special meaning.
+	 *
+	 * <p>The comment character must be the first character on the line without
+	 * any leading whitespace.
+	 *
+	 * <p>Defaults to {@code '#'}.
+	 *
+	 * @since 6.0.1
+	 */
+	@API(status = EXPERIMENTAL, since = "6.0.1")
+	char commentCharacter() default '#';
+
 }

jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvArgumentsProviderTests.java

@@ -400,6 +400,83 @@ void honorsCommentCharacterWhenUsingTextBlockAttribute() {
 		assertThat(arguments).containsExactly(array("bar", "#baz"), array("#bar", "baz"));
 	}
 
+	@Test
+	void honorsCustomCommentCharacter() {
+		var annotation = csvSource().textBlock("""
+				*foo
+				bar, *baz
+				'*bar', baz
+				""").commentCharacter('*').build();
+
+		var arguments = provideArguments(annotation);
+
+		assertThat(arguments).containsExactly(array("bar", "*baz"), array("*bar", "baz"));
+	}
+
+	@Test
+	void doesNotThrowExceptionWhenDelimiterAndCommentCharacterTheSameWhenUsingValueAttribute() {
+		var annotation = csvSource().lines("foo#bar").delimiter('#').commentCharacter('#').build();
+
+		var arguments = provideArguments(annotation);
+
+		assertThat(arguments).containsExactly(array("foo", "bar"));
+	}
+
+	@ParameterizedTest
+	@MethodSource("invalidDelimiterAndQuoteCharacterCombinations")
+	void doesNotThrowExceptionWhenDelimiterAndCommentCharacterAreTheSameWhenUsingValueAttribute(Object delimiter,
+			char quoteCharacter) {
+
+		var builder = csvSource().lines("foo").quoteCharacter(quoteCharacter);
+
+		var annotation = delimiter instanceof Character c //
+				? builder.delimiter(c).build() //
+				: builder.delimiterString(delimiter.toString()).build();
+
+		var message = "delimiter or delimiterString: '%s' and quoteCharacter: '%s' must differ";
+		assertPreconditionViolationFor(() -> provideArguments(annotation).findAny()) //
+				.withMessage(message.formatted(delimiter, quoteCharacter));
+	}
+
+	static Stream<Arguments> invalidDelimiterAndQuoteCharacterCombinations() {
+		return Stream.of(
+			// delimiter
+			Arguments.of('*', '*'), //
+			// delimiterString
+			Arguments.of("*", '*'));
+	}
+
+	@ParameterizedTest
+	@MethodSource("invalidDelimiterQuoteCharacterAndCommentCharacterCombinations")
+	void throwsExceptionWhenControlCharactersAreTheSameWhenUsingTextBlockAttribute(Object delimiter,
+			char quoteCharacter, char commentCharacter) {
+
+		var builder = csvSource().textBlock("""
+				foo""").quoteCharacter(quoteCharacter).commentCharacter(commentCharacter);
+
+		var annotation = delimiter instanceof Character c //
+				? builder.delimiter(c).build() //
+				: builder.delimiterString(delimiter.toString()).build();
+
+		var message = "delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' " + //
+				"must all differ";
+		assertPreconditionViolationFor(() -> provideArguments(annotation).findAny()) //
+				.withMessage(message.formatted(delimiter, quoteCharacter, commentCharacter));
+	}
+
+	static Stream<Arguments> invalidDelimiterQuoteCharacterAndCommentCharacterCombinations() {
+		return Stream.of(
+			// delimiter
+			Arguments.of('#', '#', '#'), //
+			Arguments.of('#', '#', '*'), //
+			Arguments.of('*', '#', '#'), //
+			Arguments.of('#', '*', '#'), //
+			// delimiterString
+			Arguments.of("#", '#', '*'), //
+			Arguments.of("#", '*', '#') //
+		);
+	}
+
 	@Test
 	void supportsCsvHeadersWhenUsingTextBlockAttribute() {
 		var annotation = csvSource().useHeadersInDisplayName(true).textBlock("""

jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvFileArgumentsProviderTests.java

@@ -134,6 +134,36 @@ void ignoresCommentedOutEntries() {
 		assertThat(arguments).containsExactly(array("foo", "bar"));
 	}
 
+	@Test
+	void honorsCustomCommentCharacter() {
+		var annotation = csvFileSource()//
+				.resources("test.csv")//
+				.commentCharacter(';')//
+				.delimiter(',')//
+				.build();
+
+		var arguments = provideArguments(annotation, "foo, bar \n;baz, qux");
+
+		assertThat(arguments).containsExactly(array("foo", "bar"));
+	}
+
+	@ParameterizedTest
+	@MethodSource("org.junit.jupiter.params.provider.CsvArgumentsProviderTests#"
+			+ "invalidDelimiterQuoteCharacterAndCommentCharacterCombinations")
+	void throwsExceptionWhenControlCharactersNotDiffer(Object delimiter, char quoteCharacter, char commentCharacter) {
+		var builder = csvFileSource().resources("test.csv") //
+				.quoteCharacter(quoteCharacter).commentCharacter(commentCharacter);
+
+		var annotation = delimiter instanceof Character c //
+				? builder.delimiter(c).build() //
+				: builder.delimiterString(delimiter.toString()).build();
+
+		var message = "delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' "
+				+ "must all differ";
+		assertPreconditionViolationFor(() -> provideArguments(annotation, "foo").findAny()) //
+				.withMessage(message.formatted(delimiter, quoteCharacter, commentCharacter));
+	}
+
 	@Test
 	void closesInputStreamForClasspathResource() {
 		var closed = new AtomicBoolean(false);