From e5dad4cea1db2016728a2993b6168defbd33a440 Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Tue, 8 Nov 2022 15:55:02 +0100 Subject: [PATCH 01/18] python: Enable markdown acceptance tests --- python/Makefile | 4 ++-- testdata/good/datatables.feature.md.tokens | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 testdata/good/datatables.feature.md.tokens diff --git a/python/Makefile b/python/Makefile index 76db3bf94..2e2316f88 100644 --- a/python/Makefile +++ b/python/Makefile @@ -8,8 +8,8 @@ SOURCE_FILES = $(shell find . -name "*.py" | grep -v $(GHERKIN_PARSER)) GHERKIN = bin/gherkin GHERKIN_GENERATE_TOKENS = bin/gherkin-generate-tokens -GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature") -BAD_FEATURE_FILES = $(shell find ../testdata/bad -name "*.feature") +GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature" -o -name "*.feature.md") +BAD_FEATURE_FILES = $(shell find ../testdata/bad -name "*.feature" -o -name "*.feature.md") TOKENS = $(patsubst ../testdata/%,acceptance/testdata/%.tokens,$(GOOD_FEATURE_FILES)) ASTS = $(patsubst ../testdata/%,acceptance/testdata/%.ast.ndjson,$(GOOD_FEATURE_FILES)) diff --git a/testdata/good/datatables.feature.md.tokens b/testdata/good/datatables.feature.md.tokens new file mode 100644 index 000000000..e69de29bb From 208577333fa26a3d715112ade993911b4d7d5ec1 Mon Sep 17 00:00:00 2001 From: temyers Date: Fri, 25 Nov 2022 22:13:23 +0800 Subject: [PATCH 02/18] fix: markdown - match empty lines --- python/bin/gherkin_generate_tokens.py | 8 +++++++- python/gherkin/token_matcher_markdown.py | 2 +- python/test/gherkin_in_markdown_token_matcher_test.py | 11 ++++++++++- testdata/good/minimal.feature.md | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/python/bin/gherkin_generate_tokens.py b/python/bin/gherkin_generate_tokens.py index f512273f8..1d5164a52 100644 --- a/python/bin/gherkin_generate_tokens.py +++ b/python/bin/gherkin_generate_tokens.py @@ -7,12 +7,18 @@ from gherkin.token_scanner import TokenScanner from gherkin.token_formatter_builder import TokenFormatterBuilder from gherkin.parser import Parser +from gherkin.token_matcher_markdown import GherkinInMarkdownTokenMatcher files = sys.argv[1:] if sys.version_info < (3, 0) and os.name != 'nt': # for Python2 unless on Windows native UTF8Writer = codecs.getwriter('utf8') sys.stdout = UTF8Writer(sys.stdout) + parser = Parser(TokenFormatterBuilder()) for file in files: scanner = TokenScanner(file) - print(parser.parse(scanner)) + + if(file.endswith('.md')): + print(parser.parse(scanner), GherkinInMarkdownTokenMatcher()) + else: + print(parser.parse(scanner)) \ No newline at end of file diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index 317200180..d17e6464e 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -83,7 +83,7 @@ def match_Comment(self, token): table_cells = token.line.table_cells if(self._is_gfm_table_separator(table_cells)): return True - return self._set_token_matched(token,None,False) + return self._set_token_matched(token,None) def match_Empty(self, token): diff --git a/python/test/gherkin_in_markdown_token_matcher_test.py b/python/test/gherkin_in_markdown_token_matcher_test.py index 82a28215b..fbebc884f 100644 --- a/python/test/gherkin_in_markdown_token_matcher_test.py +++ b/python/test/gherkin_in_markdown_token_matcher_test.py @@ -229,4 +229,13 @@ def test_it_matches_ExamplesLine(): assert tm.match_ExamplesLine(token) assert token.matched_type == 'ExamplesLine' assert token.matched_keyword == 'Examples' - assert token.matched_text == '' \ No newline at end of file + assert token.matched_text == '' + +def test_it_matches_Empty(): + tm = GherkinInMarkdownTokenMatcher('en') + line = GherkinLine('''''',location['line']) + token = Token(gherkin_line=line, location=location) + assert tm.match_Empty(token) + assert token.matched_type == 'Empty' + assert token.matched_keyword == None + assert token.matched_text == None \ No newline at end of file diff --git a/testdata/good/minimal.feature.md b/testdata/good/minimal.feature.md index 887f6c721..f0bb63033 100644 --- a/testdata/good/minimal.feature.md +++ b/testdata/good/minimal.feature.md @@ -2,4 +2,4 @@ ## Scenario: minimalistic - * Given the minimalism +* Given the minimalism From 709372c29fe6b0f230187e5b8a03746cade391d1 Mon Sep 17 00:00:00 2001 From: temyers Date: Fri, 25 Nov 2022 22:44:21 +0800 Subject: [PATCH 03/18] fix: fix typo in markdown token generation --- python/bin/gherkin_generate_tokens.py | 2 +- python/test/gherkin_in_markdown_token_matcher_test.py | 11 ++++++++++- testdata/good/minimal.feature.md | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/bin/gherkin_generate_tokens.py b/python/bin/gherkin_generate_tokens.py index 1d5164a52..82e5c2250 100644 --- a/python/bin/gherkin_generate_tokens.py +++ b/python/bin/gherkin_generate_tokens.py @@ -19,6 +19,6 @@ scanner = TokenScanner(file) if(file.endswith('.md')): - print(parser.parse(scanner), GherkinInMarkdownTokenMatcher()) + print(parser.parse(scanner, GherkinInMarkdownTokenMatcher()) ) else: print(parser.parse(scanner)) \ No newline at end of file diff --git a/python/test/gherkin_in_markdown_token_matcher_test.py b/python/test/gherkin_in_markdown_token_matcher_test.py index fbebc884f..f2d98236f 100644 --- a/python/test/gherkin_in_markdown_token_matcher_test.py +++ b/python/test/gherkin_in_markdown_token_matcher_test.py @@ -6,7 +6,16 @@ from gherkin.gherkin_line import GherkinLine location = { 'line': 1, 'column': 1 } -def test_it_matches_FeatureLine(): +def test_it_matches_FeatureLineH1(): + tm = GherkinInMarkdownTokenMatcher('en') + line = GherkinLine('''# Feature: hello''',location['line']) + token = Token(gherkin_line=line, location=location) + assert tm.match_FeatureLine(token) + assert token.matched_type == 'FeatureLine' + assert token.matched_keyword == 'Feature' + assert token.matched_text == 'hello' + +def test_it_matches_FeatureLineH2(): tm = GherkinInMarkdownTokenMatcher('en') line = GherkinLine('''## Feature: hello''',location['line']) token = Token(gherkin_line=line, location=location) diff --git a/testdata/good/minimal.feature.md b/testdata/good/minimal.feature.md index f0bb63033..78ed9f3b6 100644 --- a/testdata/good/minimal.feature.md +++ b/testdata/good/minimal.feature.md @@ -2,4 +2,4 @@ ## Scenario: minimalistic -* Given the minimalism +* Given the minimalism From b8ce724eff58359a48eb3046c50349af9fac9b0b Mon Sep 17 00:00:00 2001 From: temyers Date: Mon, 28 Nov 2022 13:44:51 +0800 Subject: [PATCH 04/18] fix: ensure markdown table headers are matched correctly. --- python/gherkin/token_matcher_markdown.py | 1 + python/test/gherkin_in_markdown_token_matcher_test.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index d17e6464e..cfb1d9e35 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -82,6 +82,7 @@ def match_Comment(self, token): if(token.line.startswith('|')): table_cells = token.line.table_cells if(self._is_gfm_table_separator(table_cells)): + self._set_token_matched(token,"Comment") return True return self._set_token_matched(token,None) diff --git a/python/test/gherkin_in_markdown_token_matcher_test.py b/python/test/gherkin_in_markdown_token_matcher_test.py index f2d98236f..78cdf025d 100644 --- a/python/test/gherkin_in_markdown_token_matcher_test.py +++ b/python/test/gherkin_in_markdown_token_matcher_test.py @@ -160,11 +160,13 @@ def test_it_matches_table_separator_row_as_comment(): l1 = GherkinLine(' | h1 | h2 |',location['line']) t1 = Token(l1,location) assert tm.match_TableRow(t1) + assert t1.location['column'] == 3 l2 = GherkinLine(' | --- | --- |',location['line']) t2 = Token(l2,location) assert not tm.match_TableRow(t2) assert tm.match_Comment(t2) + assert t2.location['column'] == 3 def test_it_matches_indented_tags(): tm = GherkinInMarkdownTokenMatcher('en') From c9c88a193b6738da1b24b84936f96784430c4539 Mon Sep 17 00:00:00 2001 From: temyers Date: Mon, 28 Nov 2022 13:45:32 +0800 Subject: [PATCH 05/18] WIP - add scenario parsing manually - Markdown files make feature header optional --- python/gherkin/parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/gherkin/parser.py b/python/gherkin/parser.py index 1c0baad9b..ffcb450bc 100644 --- a/python/gherkin/parser.py +++ b/python/gherkin/parser.py @@ -250,6 +250,11 @@ def match_token_at_0(self, token, context): if self.match_Empty(context, token): self.build(context, token) return 0 + # FIXME - move into gherkin-python.razor + if self.match_ScenarioLine(context,token): + self.build(context, token) + return 12 + state_comment = "State: 0 - Start" token.detach From 0ef264e4c9d71d04389fed7d8de17c55c72a2ab7 Mon Sep 17 00:00:00 2001 From: temyers Date: Mon, 28 Nov 2022 13:59:11 +0800 Subject: [PATCH 06/18] Add token test data - approval testing --- testdata/good/datatables.feature.md.tokens | 9 +++++ testdata/good/docstrings.feature.md.tokens | 9 +++++ testdata/good/minimal.feature.md.tokens | 6 +++ testdata/good/misc.feature.md.tokens | 26 +++++++++++++ testdata/good/tags.feature.md.tokens | 43 ++++++++++++++++++++++ 5 files changed, 93 insertions(+) create mode 100644 testdata/good/docstrings.feature.md.tokens create mode 100644 testdata/good/minimal.feature.md.tokens create mode 100644 testdata/good/misc.feature.md.tokens create mode 100644 testdata/good/tags.feature.md.tokens diff --git a/testdata/good/datatables.feature.md.tokens b/testdata/good/datatables.feature.md.tokens index e69de29bb..704c8c859 100644 --- a/testdata/good/datatables.feature.md.tokens +++ b/testdata/good/datatables.feature.md.tokens @@ -0,0 +1,9 @@ +(1:4)FeatureLine:()Feature/DataTables/ +(2:1)Empty:// +(3:5)ScenarioLine:()Scenario/minimalistic/ +(4:1)Empty:// +(5:3)StepLine:()Given /a simple data table/ +(6:3)TableRow:()|//5:foo,11:bar +(7:3)Comment:// +(8:3)TableRow:()|//5:boz,11:boo +EOF diff --git a/testdata/good/docstrings.feature.md.tokens b/testdata/good/docstrings.feature.md.tokens new file mode 100644 index 000000000..990c0ead2 --- /dev/null +++ b/testdata/good/docstrings.feature.md.tokens @@ -0,0 +1,9 @@ +(1:4)FeatureLine:()Feature/DocString variations/ +(2:1)Empty:// +(3:5)ScenarioLine:()Scenario/minimalistic/ +(4:1)Empty:// +(5:3)StepLine:()And /a DocString with an implicitly escaped separator inside/ +(6:1)DocStringSeparator:()````// +(7:1)Other:/```/ +(8:1)DocStringSeparator:()````// +EOF diff --git a/testdata/good/minimal.feature.md.tokens b/testdata/good/minimal.feature.md.tokens new file mode 100644 index 000000000..ad9a7d8bc --- /dev/null +++ b/testdata/good/minimal.feature.md.tokens @@ -0,0 +1,6 @@ +(1:3)FeatureLine:()Feature/Minimal/ +(2:1)Empty:// +(3:4)ScenarioLine:()Scenario/minimalistic/ +(4:1)Empty:// +(5:3)StepLine:()Given /the minimalism/ +EOF diff --git a/testdata/good/misc.feature.md.tokens b/testdata/good/misc.feature.md.tokens new file mode 100644 index 000000000..04516005b --- /dev/null +++ b/testdata/good/misc.feature.md.tokens @@ -0,0 +1,26 @@ +(1:1)Empty:// +(2:1)Empty:// +(3:1)Empty:// +(4:1)Empty:// +(5:1)Empty:// +(6:1)Empty:// +(7:1)Empty:// +(8:1)Empty:// +(9:1)Empty:// +(10:1)Empty:// +(11:1)Empty:// +(12:3)ScenarioLine:()Scenario/Something about math/ +(13:3)StepLine:()Given /step one/ +(14:3)StepLine:()When /step two/ +(15:3)StepLine:()Then /step three/ +(16:1)Empty:// +(17:3)ScenarioLine:()Scenario/Something about gravity/ +(18:4)StepLine:()Given /step one/ +(19:4)StepLine:()When /step two/ +(20:4)StepLine:()Then /step three/ +(21:1)Empty:// +(22:1)Empty:// +(23:1)Empty:// +(24:1)Empty:// +(25:1)Empty:// +EOF diff --git a/testdata/good/tags.feature.md.tokens b/testdata/good/tags.feature.md.tokens new file mode 100644 index 000000000..f5d09a7bd --- /dev/null +++ b/testdata/good/tags.feature.md.tokens @@ -0,0 +1,43 @@ +(1:1)TagLine://2:@feature_tag1,18:@feature_tag2 +(2:3)TagLine://4:@feature_tag3 +(3:3)FeatureLine:()Feature/Minimal Scenario Outline/ +(4:1)Empty:// +(5:1)TagLine://2:@scenario_tag1,19:@scenario_tag2 +(6:3)TagLine://4:@scenario_tag3 +(7:4)ScenarioLine:()Scenario/minimalistic/ +(8:3)StepLine:()Given /the minimalism/ +(9:1)Empty:// +(10:1)TagLine://2:@so_tag1,14:@so_tag2 +(11:3)TagLine://4:@so_tag3 +(12:4)ScenarioLine:()Scenario Outline/minimalistic outline/ +(13:3)StepLine:()Given /the / +(14:1)Empty:// +(15:1)TagLine://2:@ex_tag1,13:@ex_tag2 +(16:3)TagLine://4:@ex_tag3 +(17:5)ExamplesLine:()Examples// +(18:3)TableRow:()|//5:what +(19:3)Comment:// +(20:3)TableRow:()|//5:minimalism +(21:1)Empty:// +(22:1)TagLine://2:@ex_tag4,13:@ex_tag5 +(23:3)TagLine://4:@ex_tag6 +(24:5)ExamplesLine:()Examples// +(25:3)TableRow:()|//5:what +(26:3)Comment:// +(27:3)TableRow:()|//5:more minimalism +(28:1)Empty:// +(29:1)TagLine://2:@comment_tag1 +(30:4)ScenarioLine:()Scenario/comments/ +(31:1)Empty:// +(32:1)Empty:// +(33:1)TagLine://2:@comment_tag#2 +(34:4)ScenarioLine:()Scenario/hash in tags/ +(35:1)Empty:// +(36:1)Empty:// +(37:1)TagLine://2:@rule_tag +(38:4)RuleLine:()Rule// +(39:1)Empty:// +(40:1)TagLine://2:@joined_tag3,16:@joined_tag4 +(41:5)ScenarioLine:()Scenario/joined tags/ +(42:1)Empty:// +EOF From a6979162891f972eb71e81b9138cac5e484ce62c Mon Sep 17 00:00:00 2001 From: temyers Date: Wed, 30 Nov 2022 07:39:18 +0800 Subject: [PATCH 07/18] Revert change to testdata/good/minimal.feature.md --- testdata/good/minimal.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/good/minimal.feature.md b/testdata/good/minimal.feature.md index 78ed9f3b6..887f6c721 100644 --- a/testdata/good/minimal.feature.md +++ b/testdata/good/minimal.feature.md @@ -2,4 +2,4 @@ ## Scenario: minimalistic -* Given the minimalism + * Given the minimalism From 6387c51bb99e71225345c6da851b9bcdca86a160 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 13 Dec 2022 12:32:04 +0800 Subject: [PATCH 08/18] fix: generate AST for markdown features. Fix AST acceptance test for ../testdata/good/minimal.feature.md --- python/gherkin/stream/gherkin_events.py | 6 +++++- python/gherkin/token_matcher_markdown.py | 9 +++++++-- testdata/good/minimal.feature.md.tokens | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/gherkin/stream/gherkin_events.py b/python/gherkin/stream/gherkin_events.py index 23dcb15d1..9a6102c52 100644 --- a/python/gherkin/stream/gherkin_events.py +++ b/python/gherkin/stream/gherkin_events.py @@ -3,6 +3,7 @@ from gherkin.pickles.compiler import Compiler from gherkin.errors import ParserError, CompositeParserException from gherkin.stream.id_generator import IdGenerator +from gherkin.token_matcher_markdown import GherkinInMarkdownTokenMatcher def create_errors(errors, uri): for error in errors: @@ -28,7 +29,10 @@ def enum(self, source_event): source = source_event['source']['data'] try: - gherkin_document = self.parser.parse(source) + matcher=None + if(uri.endswith('.md')): + matcher=GherkinInMarkdownTokenMatcher() + gherkin_document = self.parser.parse(source, matcher) gherkin_document['uri'] = uri if (self.options.print_source): diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index cfb1d9e35..d1816ab59 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -181,11 +181,16 @@ def _match_title_line(self, prefix, keywords, keywordSuffix, token, token_type): match = re.search(u'{}({}){}(.*)'.format(prefix, keywords_or_list, keywordSuffix), token.line.get_line_text()) indent = token.line.indent result = False - + matchedKeywordType=None if(match): matchedKeyword = match.group(2) indent += len(match.group(1)) - self._set_token_matched(token, token_type, match.group(3).strip(), matchedKeyword, indent=indent) + + # only set the keyword type if this is a step keyword + if( matchedKeyword in self.dialect.given_keywords ): + matchedKeywordType = self.keyword_types[matchedKeyword][0] + + self._set_token_matched(token, token_type, match.group(3).strip(), matchedKeyword, keyword_type=matchedKeywordType, indent=indent) return True return False diff --git a/testdata/good/minimal.feature.md.tokens b/testdata/good/minimal.feature.md.tokens index ad9a7d8bc..4d34320ee 100644 --- a/testdata/good/minimal.feature.md.tokens +++ b/testdata/good/minimal.feature.md.tokens @@ -2,5 +2,5 @@ (2:1)Empty:// (3:4)ScenarioLine:()Scenario/minimalistic/ (4:1)Empty:// -(5:3)StepLine:()Given /the minimalism/ +(5:6)StepLine:(Context)Given /the minimalism/ EOF From bbe4d210ac57c08c1b140e4927386d264dd25361 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 15:39:11 +0800 Subject: [PATCH 09/18] test: Add test coverage to ensure consistency between javascript and python implementations of GherkinInMarkdownMatcher --- .../test/GherkinInMarkdownTokenMatcherTest.ts | 26 ++++++ python/gherkin/parser.py | 6 +- python/gherkin/token_matcher_markdown.py | 2 + .../gherkin_in_markdown_token_matcher_test.py | 30 ++++++- python/test/gherkin_test.py | 84 +++++++++++++++++++ 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/javascript/test/GherkinInMarkdownTokenMatcherTest.ts b/javascript/test/GherkinInMarkdownTokenMatcherTest.ts index 7eb8f9324..b4e62ae26 100644 --- a/javascript/test/GherkinInMarkdownTokenMatcherTest.ts +++ b/javascript/test/GherkinInMarkdownTokenMatcherTest.ts @@ -34,6 +34,15 @@ describe('GherkinInMarkdownTokenMatcher', function () { assert.strictEqual(token.matchedText, 'hello') }) + it('matches FeatureLine without the Feature: keyword', () => { + const line = new GherkinLine('# hello', location.line) + const token = new Token(line, location) + assert(tm.match_FeatureLine(token)) + assert.strictEqual(token.matchedType, TokenType.FeatureLine) + assert.strictEqual(token.matchedKeyword, undefined) + assert.strictEqual(token.matchedText, '# hello') + }) + it('matches bullet Step', () => { const line = new GherkinLine(' * Given I have 3 cukes', location.line) const token = new Token(line, location) @@ -186,4 +195,21 @@ describe('GherkinInMarkdownTokenMatcher', function () { ] assert.deepStrictEqual(t.matchedItems, expectedItems) }) + + it('matches arbitrary text as Empty after the FeatureLine has already been matched', () => { + // White Box testing - implementation detail... + // Given the FeatureLine has already been matched + const tFeatureLine = new Token(new GherkinLine('# something arbitrary', location.line), location); + assert(tm.match_FeatureLine(tFeatureLine)) + + + const t = new Token(new GherkinLine('arbitrary text', location.line), location); + // (tm as any).matchedFeatureLine = true + assert(tm.match_Empty(t)) + assert.strictEqual(t.matchedType, TokenType.Empty) + const expectedItems: Item[] =undefined + assert.deepStrictEqual(t.matchedItems, expectedItems) + assert.strictEqual(t.matchedKeyword, undefined) + assert.strictEqual(t.matchedText, undefined) + } ) }) diff --git a/python/gherkin/parser.py b/python/gherkin/parser.py index ffcb450bc..617330bc6 100644 --- a/python/gherkin/parser.py +++ b/python/gherkin/parser.py @@ -251,9 +251,9 @@ def match_token_at_0(self, token, context): self.build(context, token) return 0 # FIXME - move into gherkin-python.razor - if self.match_ScenarioLine(context,token): - self.build(context, token) - return 12 + # if self.match_ScenarioLine(context,token): + # self.build(context, token) + # return 12 state_comment = "State: 0 - Start" diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index d1816ab59..f96fb38b8 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -25,6 +25,7 @@ def match_FeatureLine(self, token): if(self.matched_feature_line): self._set_token_matched(token,None) + return False # We first try to match "# Feature: blah" result = self._match_title_line(KEYWORD_PREFIX_HEADER, self.dialect.feature_keywords, ':', token, 'FeatureLine') @@ -34,6 +35,7 @@ def match_FeatureLine(self, token): if not result: self._set_token_matched(token,'FeatureLine',token.line.get_line_text()) + result=True self.matched_feature_line=result return result diff --git a/python/test/gherkin_in_markdown_token_matcher_test.py b/python/test/gherkin_in_markdown_token_matcher_test.py index 78cdf025d..fecd7d14f 100644 --- a/python/test/gherkin_in_markdown_token_matcher_test.py +++ b/python/test/gherkin_in_markdown_token_matcher_test.py @@ -33,6 +33,15 @@ def test_it_matches_FeatureLine_in_French(): assert token.matched_keyword == u'Fonctionnalité' assert token.matched_text == 'hello' +def test_it_matches_FeatureLine_without_the_Feature_keyword(): + tm = GherkinInMarkdownTokenMatcher('en') + line = GherkinLine('''# hello''',location['line']) + token = Token(gherkin_line=line, location=location) + assert tm.match_FeatureLine(token) + assert token.matched_type == 'FeatureLine' + assert token.matched_keyword == None + assert token.matched_text == '# hello' + def test_it_matches_bullet_Step(): tm = GherkinInMarkdownTokenMatcher('en') line = GherkinLine(''' * Given I have 3 cukes''',location['line']) @@ -249,4 +258,23 @@ def test_it_matches_Empty(): assert tm.match_Empty(token) assert token.matched_type == 'Empty' assert token.matched_keyword == None - assert token.matched_text == None \ No newline at end of file + assert token.matched_text == None + +def test_it_matches_arbitrary_text_as_Empty_after_the_FeatureLine_has_already_been_matched(): + # White Box testing - implementation detail... + # Given the FeatureLine has already been matched + tm = GherkinInMarkdownTokenMatcher('en') + + line = GherkinLine('''# something arbitrary''',location['line']) + token = Token(gherkin_line=line, location=location) + assert(tm.match_FeatureLine(token)) + + line = GherkinLine('''arbitrary text''',location['line']) + token=Token(gherkin_line=line, location=location) + + assert(tm.match_Empty(token)) + assert token.matched_type == 'Empty' + assert token.matched_items == [] + assert token.matched_keyword == None + assert token.matched_text == None + pass \ No newline at end of file diff --git a/python/test/gherkin_test.py b/python/test/gherkin_test.py index 826f72c07..eea3cf506 100644 --- a/python/test/gherkin_test.py +++ b/python/test/gherkin_test.py @@ -5,6 +5,8 @@ from gherkin.errors import ParserError import pytest +from gherkin.token_matcher_markdown import GherkinInMarkdownTokenMatcher + def test_parser(): parser = Parser() @@ -89,3 +91,85 @@ def test_change_the_default_language(): } assert expected == feature_file + +def test_parsing_markdown_does_not_parse_a_feature_description(): + parser = Parser() + matcher = GherkinInMarkdownTokenMatcher() + + feature_file = """# Feature: hello +This is the +description +""" + ast = parser.parse(TokenScanner(feature_file), matcher) + expected = { + 'feature': { + 'tags': [], + 'description': '', + 'location': { 'line': 1, 'column': 3 }, + 'language': 'en', + 'keyword': 'Feature', + 'name': 'hello', + 'children': [], + }, + 'comments': [], + } + assert ast == expected + +def test_parsing_markdown_parses_a_feature_without_a_hash_Feature_header(): + parser = Parser() + matcher = GherkinInMarkdownTokenMatcher() + feature_file = """# Hello +This is the +description + +## Scenario: hello ++ Given a step + +## Some other header +""" + ast = parser.parse(TokenScanner(feature_file), matcher) + expected = { + 'feature': { + 'tags': [], + 'location': { + 'line': 1, + 'column': 1, + }, + 'language': 'en', + # 'keyword': None, + 'name': '# Hello', + 'description': '', + 'children': [ + { + 'scenario': { + 'id': '1', + 'tags': [], + 'location': { + 'line': 5, + 'column': 4, + }, + 'keyword': 'Scenario', + 'name': 'hello', + 'description': '', + 'steps': [ + { + 'id': '0', + 'location': { + 'line': 6, + 'column': 3, + }, + 'keyword': 'Given ', + 'keywordType': "Context", + 'text': 'a step', + # 'dataTable': None, + # 'docString': None, + }, + ], + 'examples': [], + }, + }, + ], + }, + 'comments': [], + } + assert ast == expected \ No newline at end of file From 718d07c126671c879b26e590cfe452eb913fe448 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 15:57:34 +0800 Subject: [PATCH 10/18] test: update test data using python gherkin parser --- testdata/good/datatables.feature.md.ast.ndjson | 2 +- testdata/good/datatables.feature.md.tokens | 2 +- testdata/good/docstrings.feature.md.ast.ndjson | 2 +- testdata/good/misc.feature.md.ast.ndjson | 2 +- testdata/good/misc.feature.md.tokens | 6 +++--- testdata/good/tags.feature.md.ast.ndjson | 2 +- testdata/good/tags.feature.md.tokens | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testdata/good/datatables.feature.md.ast.ndjson b/testdata/good/datatables.feature.md.ast.ndjson index ff3747d20..a5409d1f0 100644 --- a/testdata/good/datatables.feature.md.ast.ndjson +++ b/testdata/good/datatables.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"dataTable":{"location":{"column":3,"line":6},"rows":[{"cells":[{"location":{"column":5,"line":6},"value":"foo"},{"location":{"column":11,"line":6},"value":"bar"}],"id":"0","location":{"column":3,"line":6}},{"cells":[{"location":{"column":5,"line":8},"value":"boz"},{"location":{"column":11,"line":8},"value":"boo"}],"id":"1","location":{"column":3,"line":8}}]},"id":"2","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":5},"text":"a simple data table"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DataTables","tags":[]},"uri":"../testdata/good/datatables.feature.md"}} +{"gherkinDocument":{"comments":[{"location":{"column":3,"line":7},"text":null}],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"dataTable":{"location":{"column":3,"line":6},"rows":[{"cells":[{"location":{"column":5,"line":6},"value":"foo"},{"location":{"column":11,"line":6},"value":"bar"}],"id":"0","location":{"column":3,"line":6}},{"cells":[{"location":{"column":5,"line":8},"value":"boz"},{"location":{"column":11,"line":8},"value":"boo"}],"id":"1","location":{"column":3,"line":8}}]},"id":"2","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":5},"text":"a simple data table"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DataTables","tags":[]},"uri":"../testdata/good/datatables.feature.md"}} diff --git a/testdata/good/datatables.feature.md.tokens b/testdata/good/datatables.feature.md.tokens index 704c8c859..0ecbe9558 100644 --- a/testdata/good/datatables.feature.md.tokens +++ b/testdata/good/datatables.feature.md.tokens @@ -2,7 +2,7 @@ (2:1)Empty:// (3:5)ScenarioLine:()Scenario/minimalistic/ (4:1)Empty:// -(5:3)StepLine:()Given /a simple data table/ +(5:3)StepLine:(Context)Given /a simple data table/ (6:3)TableRow:()|//5:foo,11:bar (7:3)Comment:// (8:3)TableRow:()|//5:boz,11:boo diff --git a/testdata/good/docstrings.feature.md.ast.ndjson b/testdata/good/docstrings.feature.md.ast.ndjson index 7139bcd46..f1fdb4a4d 100644 --- a/testdata/good/docstrings.feature.md.ast.ndjson +++ b/testdata/good/docstrings.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"docString":{"content":"```","delimiter":"````","location":{"column":1,"line":6}},"id":"0","keyword":"And ","keywordType":"Conjunction","location":{"column":3,"line":5},"text":"a DocString with an implicitly escaped separator inside"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DocString variations","tags":[]},"uri":"../testdata/good/docstrings.feature.md"}} +{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"docString":{"content":"```","delimiter":"````","location":{"column":1,"line":6}},"id":"0","keyword":"And ","location":{"column":3,"line":5},"text":"a DocString with an implicitly escaped separator inside"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DocString variations","tags":[]},"uri":"../testdata/good/docstrings.feature.md"}} diff --git a/testdata/good/misc.feature.md.ast.ndjson b/testdata/good/misc.feature.md.ast.ndjson index f04397ef2..10254af10 100644 --- a/testdata/good/misc.feature.md.ast.ndjson +++ b/testdata/good/misc.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":3,"line":12},"name":"Something about math","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"step one"},{"id":"1","keyword":"When ","keywordType":"Action","location":{"column":3,"line":14},"text":"step two"},{"id":"2","keyword":"Then ","keywordType":"Outcome","location":{"column":3,"line":15},"text":"step three"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":17},"name":"Something about gravity","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":4,"line":18},"text":"step one"},{"id":"5","keyword":"When ","keywordType":"Action","location":{"column":4,"line":19},"text":"step two"},{"id":"6","keyword":"Then ","keywordType":"Outcome","location":{"column":4,"line":20},"text":"step three"}],"tags":[]}}],"description":"","language":"en","location":{"column":1,"line":1},"name":"Markdown document without \"# Feature:\" header","tags":[]},"uri":"../testdata/good/misc.feature.md"}} +{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":3,"line":12},"name":"Something about math","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"step one"},{"id":"1","keyword":"When ","location":{"column":3,"line":14},"text":"step two"},{"id":"2","keyword":"Then ","location":{"column":3,"line":15},"text":"step three"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":17},"name":"Something about gravity","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":4,"line":18},"text":"step one"},{"id":"5","keyword":"When ","location":{"column":4,"line":19},"text":"step two"},{"id":"6","keyword":"Then ","location":{"column":4,"line":20},"text":"step three"}],"tags":[]}}],"description":"","language":"en","location":{"column":1,"line":1},"name":"Markdown document without \"# Feature:\" header","tags":[]},"uri":"../testdata/good/misc.feature.md"}} diff --git a/testdata/good/misc.feature.md.tokens b/testdata/good/misc.feature.md.tokens index 04516005b..dd8458784 100644 --- a/testdata/good/misc.feature.md.tokens +++ b/testdata/good/misc.feature.md.tokens @@ -1,4 +1,4 @@ -(1:1)Empty:// +(1:1)FeatureLine:/Markdown document without "# Feature:" header/ (2:1)Empty:// (3:1)Empty:// (4:1)Empty:// @@ -10,12 +10,12 @@ (10:1)Empty:// (11:1)Empty:// (12:3)ScenarioLine:()Scenario/Something about math/ -(13:3)StepLine:()Given /step one/ +(13:3)StepLine:(Context)Given /step one/ (14:3)StepLine:()When /step two/ (15:3)StepLine:()Then /step three/ (16:1)Empty:// (17:3)ScenarioLine:()Scenario/Something about gravity/ -(18:4)StepLine:()Given /step one/ +(18:4)StepLine:(Context)Given /step one/ (19:4)StepLine:()When /step two/ (20:4)StepLine:()Then /step three/ (21:1)Empty:// diff --git a/testdata/good/tags.feature.md.ast.ndjson b/testdata/good/tags.feature.md.ast.ndjson index 26df9f842..75837dc9e 100644 --- a/testdata/good/tags.feature.md.ast.ndjson +++ b/testdata/good/tags.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"4","keyword":"Scenario","location":{"column":4,"line":7},"name":"minimalistic","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":8},"text":"the minimalism"}],"tags":[{"id":"1","location":{"column":2,"line":5},"name":"@scenario_tag1"},{"id":"2","location":{"column":19,"line":5},"name":"@scenario_tag2"},{"id":"3","location":{"column":4,"line":6},"name":"@scenario_tag3"}]}},{"scenario":{"description":"","examples":[{"description":"","id":"11","keyword":"Examples","location":{"column":5,"line":17},"name":"","tableBody":[{"cells":[{"location":{"column":5,"line":20},"value":"minimalism"}],"id":"7","location":{"column":3,"line":20}}],"tableHeader":{"cells":[{"location":{"column":5,"line":18},"value":"what"}],"id":"6","location":{"column":3,"line":18}},"tags":[{"id":"8","location":{"column":2,"line":15},"name":"@ex_tag1"},{"id":"9","location":{"column":13,"line":15},"name":"@ex_tag2"},{"id":"10","location":{"column":4,"line":16},"name":"@ex_tag3"}]},{"description":"","id":"17","keyword":"Examples","location":{"column":5,"line":24},"name":"","tableBody":[{"cells":[{"location":{"column":5,"line":27},"value":"more minimalism"}],"id":"13","location":{"column":3,"line":27}}],"tableHeader":{"cells":[{"location":{"column":5,"line":25},"value":"what"}],"id":"12","location":{"column":3,"line":25}},"tags":[{"id":"14","location":{"column":2,"line":22},"name":"@ex_tag4"},{"id":"15","location":{"column":13,"line":22},"name":"@ex_tag5"},{"id":"16","location":{"column":4,"line":23},"name":"@ex_tag6"}]}],"id":"21","keyword":"Scenario Outline","location":{"column":4,"line":12},"name":"minimalistic outline","steps":[{"id":"5","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"the "}],"tags":[{"id":"18","location":{"column":2,"line":10},"name":"@so_tag1"},{"id":"19","location":{"column":14,"line":10},"name":"@so_tag2"},{"id":"20","location":{"column":4,"line":11},"name":"@so_tag3"}]}},{"scenario":{"description":"","examples":[],"id":"23","keyword":"Scenario","location":{"column":4,"line":30},"name":"comments","steps":[],"tags":[{"id":"22","location":{"column":2,"line":29},"name":"@comment_tag1"}]}},{"scenario":{"description":"","examples":[],"id":"25","keyword":"Scenario","location":{"column":4,"line":34},"name":"hash in tags","steps":[],"tags":[{"id":"24","location":{"column":2,"line":33},"name":"@comment_tag#2"}]}},{"rule":{"children":[{"scenario":{"description":"","examples":[],"id":"28","keyword":"Scenario","location":{"column":5,"line":41},"name":"joined tags","steps":[],"tags":[{"id":"26","location":{"column":2,"line":40},"name":"@joined_tag3"},{"id":"27","location":{"column":16,"line":40},"name":"@joined_tag4"}]}}],"description":"","id":"30","keyword":"Rule","location":{"column":4,"line":38},"name":"","tags":[{"id":"29","location":{"column":2,"line":37},"name":"@rule_tag"}]}}],"description":"","keyword":"Feature","language":"en","location":{"column":3,"line":3},"name":"Minimal Scenario Outline","tags":[{"id":"31","location":{"column":2,"line":1},"name":"@feature_tag1"},{"id":"32","location":{"column":18,"line":1},"name":"@feature_tag2"},{"id":"33","location":{"column":4,"line":2},"name":"@feature_tag3"}]},"uri":"../testdata/good/tags.feature.md"}} +{"gherkinDocument":{"comments":[{"location":{"column":3,"line":19},"text":null},{"location":{"column":3,"line":26},"text":null}],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"4","keyword":"Scenario","location":{"column":4,"line":7},"name":"minimalistic","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":8},"text":"the minimalism"}],"tags":[{"id":"1","location":{"column":2,"line":5},"name":"@scenario_tag1"},{"id":"2","location":{"column":19,"line":5},"name":"@scenario_tag2"},{"id":"3","location":{"column":4,"line":6},"name":"@scenario_tag3"}]}},{"scenario":{"description":"","examples":[{"description":"","id":"11","keyword":"Examples","location":{"column":5,"line":17},"name":"","tableBody":[{"cells":[{"location":{"column":5,"line":20},"value":"minimalism"}],"id":"7","location":{"column":3,"line":20}}],"tableHeader":{"cells":[{"location":{"column":5,"line":18},"value":"what"}],"id":"6","location":{"column":3,"line":18}},"tags":[{"id":"8","location":{"column":2,"line":15},"name":"@ex_tag1"},{"id":"9","location":{"column":13,"line":15},"name":"@ex_tag2"},{"id":"10","location":{"column":4,"line":16},"name":"@ex_tag3"}]},{"description":"","id":"17","keyword":"Examples","location":{"column":5,"line":24},"name":"","tableBody":[{"cells":[{"location":{"column":5,"line":27},"value":"more minimalism"}],"id":"13","location":{"column":3,"line":27}}],"tableHeader":{"cells":[{"location":{"column":5,"line":25},"value":"what"}],"id":"12","location":{"column":3,"line":25}},"tags":[{"id":"14","location":{"column":2,"line":22},"name":"@ex_tag4"},{"id":"15","location":{"column":13,"line":22},"name":"@ex_tag5"},{"id":"16","location":{"column":4,"line":23},"name":"@ex_tag6"}]}],"id":"21","keyword":"Scenario Outline","location":{"column":4,"line":12},"name":"minimalistic outline","steps":[{"id":"5","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"the "}],"tags":[{"id":"18","location":{"column":2,"line":10},"name":"@so_tag1"},{"id":"19","location":{"column":14,"line":10},"name":"@so_tag2"},{"id":"20","location":{"column":4,"line":11},"name":"@so_tag3"}]}},{"scenario":{"description":"","examples":[],"id":"23","keyword":"Scenario","location":{"column":4,"line":30},"name":"comments","steps":[],"tags":[{"id":"22","location":{"column":2,"line":29},"name":"@comment_tag1"}]}},{"scenario":{"description":"","examples":[],"id":"25","keyword":"Scenario","location":{"column":4,"line":34},"name":"hash in tags","steps":[],"tags":[{"id":"24","location":{"column":2,"line":33},"name":"@comment_tag#2"}]}},{"rule":{"children":[{"scenario":{"description":"","examples":[],"id":"28","keyword":"Scenario","location":{"column":5,"line":41},"name":"joined tags","steps":[],"tags":[{"id":"26","location":{"column":2,"line":40},"name":"@joined_tag3"},{"id":"27","location":{"column":16,"line":40},"name":"@joined_tag4"}]}}],"description":"","id":"30","keyword":"Rule","location":{"column":4,"line":38},"name":"","tags":[{"id":"29","location":{"column":2,"line":37},"name":"@rule_tag"}]}}],"description":"","keyword":"Feature","language":"en","location":{"column":3,"line":3},"name":"Minimal Scenario Outline","tags":[{"id":"31","location":{"column":2,"line":1},"name":"@feature_tag1"},{"id":"32","location":{"column":18,"line":1},"name":"@feature_tag2"},{"id":"33","location":{"column":4,"line":2},"name":"@feature_tag3"}]},"uri":"../testdata/good/tags.feature.md"}} diff --git a/testdata/good/tags.feature.md.tokens b/testdata/good/tags.feature.md.tokens index f5d09a7bd..60bd55c4b 100644 --- a/testdata/good/tags.feature.md.tokens +++ b/testdata/good/tags.feature.md.tokens @@ -5,12 +5,12 @@ (5:1)TagLine://2:@scenario_tag1,19:@scenario_tag2 (6:3)TagLine://4:@scenario_tag3 (7:4)ScenarioLine:()Scenario/minimalistic/ -(8:3)StepLine:()Given /the minimalism/ +(8:3)StepLine:(Context)Given /the minimalism/ (9:1)Empty:// (10:1)TagLine://2:@so_tag1,14:@so_tag2 (11:3)TagLine://4:@so_tag3 (12:4)ScenarioLine:()Scenario Outline/minimalistic outline/ -(13:3)StepLine:()Given /the / +(13:3)StepLine:(Context)Given /the / (14:1)Empty:// (15:1)TagLine://2:@ex_tag1,13:@ex_tag2 (16:3)TagLine://4:@ex_tag3 From e6e126909df169536977e2208b460cbd48168078 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 16:02:03 +0800 Subject: [PATCH 11/18] test: update markdown pickels using javascript implementation --- testdata/good/datatables.feature.md.pickles.ndjson | 2 +- testdata/good/docstrings.feature.md.pickles.ndjson | 2 +- testdata/good/misc.feature.md.pickles.ndjson | 4 ++-- testdata/good/tags.feature.md.pickles.ndjson | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testdata/good/datatables.feature.md.pickles.ndjson b/testdata/good/datatables.feature.md.pickles.ndjson index e7bafe766..75eeb8b7b 100644 --- a/testdata/good/datatables.feature.md.pickles.ndjson +++ b/testdata/good/datatables.feature.md.pickles.ndjson @@ -1 +1 @@ -{"pickle":{"astNodeIds":["3"],"id":"5","language":"en","name":"minimalistic","steps":[{"argument":{"dataTable":{"rows":[{"cells":[{"value":"foo"},{"value":"bar"}]},{"cells":[{"value":"boz"},{"value":"boo"}]}]}},"astNodeIds":["2"],"id":"4","text":"a simple data table","type":"Context"}],"tags":[],"uri":"../testdata/good/datatables.feature.md"}} +{"pickle":{"id":"d5ba0306-6c4b-4a27-8032-581b16fcfb94","uri":"../testdata/good/datatables.feature.md","astNodeIds":["86ac77b8-9928-4327-ac49-ee5bc7b92727"],"tags":[],"name":"minimalistic","language":"en","steps":[{"id":"c30710bb-7578-4a65-8538-3ca3a00364c2","text":"a simple data table","type":"Context","argument":{"dataTable":{"rows":[{"cells":[{"value":"foo"},{"value":"bar"}]},{"cells":[{"value":"boz"},{"value":"boo"}]}]}},"astNodeIds":["5191c118-2880-4a3b-a0b9-dad8040e2b46"]}]}} diff --git a/testdata/good/docstrings.feature.md.pickles.ndjson b/testdata/good/docstrings.feature.md.pickles.ndjson index 80cac1a47..3915c77f8 100644 --- a/testdata/good/docstrings.feature.md.pickles.ndjson +++ b/testdata/good/docstrings.feature.md.pickles.ndjson @@ -1 +1 @@ -{"pickle":{"astNodeIds":["1"],"id":"3","language":"en","name":"minimalistic","steps":[{"argument":{"docString":{"content":"```"}},"astNodeIds":["0"],"id":"2","text":"a DocString with an implicitly escaped separator inside","type":"Unknown"}],"tags":[],"uri":"../testdata/good/docstrings.feature.md"}} +{"pickle":{"id":"24839cef-9f43-4ffa-9475-6ae2e533d8c9","uri":"../testdata/good/docstrings.feature.md","astNodeIds":["c11dca0c-4ca3-4c15-9036-6a01e080aa30"],"tags":[],"name":"minimalistic","language":"en","steps":[{"id":"70b74b30-a01b-423e-ad70-5ddb01f9235f","text":"a DocString with an implicitly escaped separator inside","type":"Unknown","argument":{"docString":{"content":"```"}},"astNodeIds":["52940dab-b378-4d20-970b-3967c427fc9b"]}]}} diff --git a/testdata/good/misc.feature.md.pickles.ndjson b/testdata/good/misc.feature.md.pickles.ndjson index cca046e5a..6080c88de 100644 --- a/testdata/good/misc.feature.md.pickles.ndjson +++ b/testdata/good/misc.feature.md.pickles.ndjson @@ -1,2 +1,2 @@ -{"pickle":{"astNodeIds":["3"],"id":"11","language":"en","name":"Something about math","steps":[{"astNodeIds":["0"],"id":"8","text":"step one","type":"Context"},{"astNodeIds":["1"],"id":"9","text":"step two","type":"Action"},{"astNodeIds":["2"],"id":"10","text":"step three","type":"Outcome"}],"tags":[],"uri":"../testdata/good/misc.feature.md"}} -{"pickle":{"astNodeIds":["7"],"id":"15","language":"en","name":"Something about gravity","steps":[{"astNodeIds":["4"],"id":"12","text":"step one","type":"Context"},{"astNodeIds":["5"],"id":"13","text":"step two","type":"Action"},{"astNodeIds":["6"],"id":"14","text":"step three","type":"Outcome"}],"tags":[],"uri":"../testdata/good/misc.feature.md"}} +{"pickle":{"id":"64494de7-bbeb-440a-bafd-811fa58b0e14","uri":"../testdata/good/misc.feature.md","astNodeIds":["6bc6758d-5013-4dbe-9234-0980b2044d1e"],"tags":[],"name":"Something about math","language":"en","steps":[{"id":"acbbba34-bed3-4082-8dc4-f20abf273674","text":"step one","type":"Context","astNodeIds":["6cadcc23-9981-41de-9e33-9e8ade163b5f"]},{"id":"e85a565a-3936-4e76-8949-34db37567e3e","text":"step two","type":"Action","astNodeIds":["f9e5934e-8d46-4220-8ab3-1ed48a59a7e8"]},{"id":"3752db1e-c2fa-4db1-aec4-442e862ff678","text":"step three","type":"Outcome","astNodeIds":["3d44f4f1-9d04-47cf-aec7-71bc08c23a51"]}]}} +{"pickle":{"id":"cdf3e41c-5f12-4b42-a8c5-99e5a30391de","uri":"../testdata/good/misc.feature.md","astNodeIds":["c9e5cfc4-7e16-49bc-a266-729c654e78e7"],"tags":[],"name":"Something about gravity","language":"en","steps":[{"id":"be702179-8d6f-4bb9-ad71-f04d2e10dd96","text":"step one","type":"Context","astNodeIds":["4f146301-cfcd-47a8-b6f1-bc3bfd224e0c"]},{"id":"003e2077-0187-4626-b10d-d3fd4bef15d0","text":"step two","type":"Action","astNodeIds":["84ded463-4b37-4817-98e0-26e2b8ea16ae"]},{"id":"5fa0b049-fbf4-47c8-9c22-2e85621aeb7f","text":"step three","type":"Outcome","astNodeIds":["4a429963-a224-460a-a001-33ba6e70948f"]}]}} diff --git a/testdata/good/tags.feature.md.pickles.ndjson b/testdata/good/tags.feature.md.pickles.ndjson index a82bd44b9..8d4d90a5d 100644 --- a/testdata/good/tags.feature.md.pickles.ndjson +++ b/testdata/good/tags.feature.md.pickles.ndjson @@ -1,6 +1,6 @@ -{"pickle":{"astNodeIds":["4"],"id":"35","language":"en","name":"minimalistic","steps":[{"astNodeIds":["0"],"id":"34","text":"the minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"1","name":"@scenario_tag1"},{"astNodeId":"2","name":"@scenario_tag2"},{"astNodeId":"3","name":"@scenario_tag3"}],"uri":"../testdata/good/tags.feature.md"}} -{"pickle":{"astNodeIds":["21","7"],"id":"37","language":"en","name":"minimalistic outline","steps":[{"astNodeIds":["5","7"],"id":"36","text":"the minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"18","name":"@so_tag1"},{"astNodeId":"19","name":"@so_tag2"},{"astNodeId":"20","name":"@so_tag3"},{"astNodeId":"8","name":"@ex_tag1"},{"astNodeId":"9","name":"@ex_tag2"},{"astNodeId":"10","name":"@ex_tag3"}],"uri":"../testdata/good/tags.feature.md"}} -{"pickle":{"astNodeIds":["21","13"],"id":"39","language":"en","name":"minimalistic outline","steps":[{"astNodeIds":["5","13"],"id":"38","text":"the more minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"18","name":"@so_tag1"},{"astNodeId":"19","name":"@so_tag2"},{"astNodeId":"20","name":"@so_tag3"},{"astNodeId":"14","name":"@ex_tag4"},{"astNodeId":"15","name":"@ex_tag5"},{"astNodeId":"16","name":"@ex_tag6"}],"uri":"../testdata/good/tags.feature.md"}} -{"pickle":{"astNodeIds":["23"],"id":"40","language":"en","name":"comments","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"22","name":"@comment_tag1"}],"uri":"../testdata/good/tags.feature.md"}} -{"pickle":{"astNodeIds":["25"],"id":"41","language":"en","name":"hash in tags","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"24","name":"@comment_tag#2"}],"uri":"../testdata/good/tags.feature.md"}} -{"pickle":{"astNodeIds":["28"],"id":"42","language":"en","name":"joined tags","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"29","name":"@rule_tag"},{"astNodeId":"26","name":"@joined_tag3"},{"astNodeId":"27","name":"@joined_tag4"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"id":"3f92a2c2-1ad2-4575-931e-a99bf47dfb21","uri":"../testdata/good/tags.feature.md","astNodeIds":["2277e04c-1300-4100-abd6-063eabad0c38"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@scenario_tag1","astNodeId":"46212da8-98c6-4a5d-b2f1-d4f3ca93e97f"},{"name":"@scenario_tag2","astNodeId":"0b45ea80-2441-455b-9afc-78eacfa3898f"},{"name":"@scenario_tag3","astNodeId":"fceb7155-c9b2-4898-8cb2-2df415d880c8"}],"name":"minimalistic","language":"en","steps":[{"id":"4ffa994f-63cc-4fa7-a73a-477f55f458db","text":"the minimalism","type":"Context","astNodeIds":["a1830df8-62b7-4db2-ad62-d3cca34b7e0f"]}]}} +{"pickle":{"id":"94bcef16-e73c-4840-882b-3dfc5b2f9739","uri":"../testdata/good/tags.feature.md","astNodeIds":["3d3adf52-351d-4a70-ae8d-a65a6f812eee","51b3f85c-e844-41e8-a981-398e3d30ad17"],"name":"minimalistic outline","language":"en","steps":[{"id":"f93c39df-2a8a-4f62-904d-8144df5c7bb7","text":"the minimalism","type":"Context","astNodeIds":["45e8a2fa-c328-4287-81c5-e15c7b6cf976","51b3f85c-e844-41e8-a981-398e3d30ad17"]}],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@so_tag1","astNodeId":"fe85917e-4674-4fc3-8cdc-eefe1d481bb3"},{"name":"@so_tag2","astNodeId":"34e2c4ca-7580-4e92-b1d5-fb4c75df0118"},{"name":"@so_tag3","astNodeId":"2f7ab5be-0728-4ea8-b9db-33562c11eb2d"},{"name":"@ex_tag1","astNodeId":"5f8272ca-c756-416f-8217-5802a3ef96e4"},{"name":"@ex_tag2","astNodeId":"da57b49d-edac-449e-8313-264bee4e89e9"},{"name":"@ex_tag3","astNodeId":"41b605cb-6494-40d8-ac28-457cd4b33a96"}]}} +{"pickle":{"id":"81abe3b6-9f02-47b7-8f42-1f6a2be46e78","uri":"../testdata/good/tags.feature.md","astNodeIds":["3d3adf52-351d-4a70-ae8d-a65a6f812eee","b1495c13-2bf0-438c-ae09-ec41dfcdf22d"],"name":"minimalistic outline","language":"en","steps":[{"id":"85fc687c-7b39-4d06-8afc-7b692aab407e","text":"the more minimalism","type":"Context","astNodeIds":["45e8a2fa-c328-4287-81c5-e15c7b6cf976","b1495c13-2bf0-438c-ae09-ec41dfcdf22d"]}],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@so_tag1","astNodeId":"fe85917e-4674-4fc3-8cdc-eefe1d481bb3"},{"name":"@so_tag2","astNodeId":"34e2c4ca-7580-4e92-b1d5-fb4c75df0118"},{"name":"@so_tag3","astNodeId":"2f7ab5be-0728-4ea8-b9db-33562c11eb2d"},{"name":"@ex_tag4","astNodeId":"fd0630c2-43b4-462f-800d-436404154c34"},{"name":"@ex_tag5","astNodeId":"d7752694-7c4d-4d78-b34b-7a5be2cfd0d4"},{"name":"@ex_tag6","astNodeId":"c7c520ed-49e0-488c-94f9-1e519e2f08f6"}]}} +{"pickle":{"id":"522516c3-040d-4f2f-9263-d88ba42fb0e3","uri":"../testdata/good/tags.feature.md","astNodeIds":["ba2ec921-8506-4415-93ec-7c29fa724785"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@comment_tag1","astNodeId":"a4c1a4f3-3fd1-4e18-a013-3257d2ec5f89"}],"name":"comments","language":"en","steps":[]}} +{"pickle":{"id":"664b327b-bb2e-41e6-8990-38a10a14c84c","uri":"../testdata/good/tags.feature.md","astNodeIds":["92be8ee0-135d-44fe-9814-0a2bb8ff7150"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@comment_tag#2","astNodeId":"ccf9263d-2032-4eb5-87a0-05940efb31c8"}],"name":"hash in tags","language":"en","steps":[]}} +{"pickle":{"id":"925937d9-cc4b-4c0e-b8f2-25062f3be01a","uri":"../testdata/good/tags.feature.md","astNodeIds":["a5de7f97-8b0f-4235-80ac-4ce0057c87a3"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@rule_tag","astNodeId":"b8caf880-2cd8-43ec-9b94-4158c112e992"},{"name":"@joined_tag3","astNodeId":"680d9c7e-95c3-486e-adf0-818ddcf6141f"},{"name":"@joined_tag4","astNodeId":"540d3559-ae0d-4f81-9ef0-bd8746e467f4"}],"name":"joined tags","language":"en","steps":[]}} From 4fb59304eb2b43f84834217dc644567b580d57b3 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:01:52 +0800 Subject: [PATCH 12/18] fix: Fix python pickle generation failure caused by missing matched_keyword_type --- .../test/GherkinInMarkdownTokenMatcherTest.ts | 18 ++++++++++++++++-- python/gherkin/token_matcher_markdown.py | 2 +- .../gherkin_in_markdown_token_matcher_test.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/javascript/test/GherkinInMarkdownTokenMatcherTest.ts b/javascript/test/GherkinInMarkdownTokenMatcherTest.ts index b4e62ae26..4e1ac205a 100644 --- a/javascript/test/GherkinInMarkdownTokenMatcherTest.ts +++ b/javascript/test/GherkinInMarkdownTokenMatcherTest.ts @@ -49,29 +49,43 @@ describe('GherkinInMarkdownTokenMatcher', function () { assert(tm.match_StepLine(token)) assert.strictEqual(token.matchedType, TokenType.StepLine) assert.strictEqual(token.matchedKeyword, 'Given ') + assert.strictEqual(token.matchedKeywordType, 'Context') assert.strictEqual(token.matchedText, 'I have 3 cukes') assert.strictEqual(token.location.column, 6) }) - + it('matches plus Step', () => { const line = new GherkinLine(' + Given I have 3 cukes', location.line) const token = new Token(line, location) assert(tm.match_StepLine(token)) assert.strictEqual(token.matchedType, TokenType.StepLine) assert.strictEqual(token.matchedKeyword, 'Given ') + assert.strictEqual(token.matchedKeywordType, 'Context') assert.strictEqual(token.matchedText, 'I have 3 cukes') assert.strictEqual(token.location.column, 6) }) - + it('matches hyphen Step', () => { const line = new GherkinLine(' - Given I have 3 cukes', location.line) const token = new Token(line, location) assert(tm.match_StepLine(token)) assert.strictEqual(token.matchedType, TokenType.StepLine) assert.strictEqual(token.matchedKeyword, 'Given ') + assert.strictEqual(token.matchedKeywordType, 'Context') assert.strictEqual(token.matchedText, 'I have 3 cukes') assert.strictEqual(token.location.column, 6) }) + + it('matches a when Step', () => { + const line = new GherkinLine(' - When I do something', location.line) + const token = new Token(line, location) + assert(tm.match_StepLine(token)) + assert.strictEqual(token.matchedType, TokenType.StepLine) + assert.strictEqual(token.matchedKeyword, 'When ') + assert.strictEqual(token.matchedKeywordType, 'Action') + assert.strictEqual(token.matchedText, 'I do something') + assert.strictEqual(token.location.column, 6) + }) it('matches arbitrary text as Other', () => { const line = new GherkinLine('Whatever', location.line) diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index f96fb38b8..02c8e930e 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -189,7 +189,7 @@ def _match_title_line(self, prefix, keywords, keywordSuffix, token, token_type): indent += len(match.group(1)) # only set the keyword type if this is a step keyword - if( matchedKeyword in self.dialect.given_keywords ): + if( matchedKeyword in self.keyword_types ): matchedKeywordType = self.keyword_types[matchedKeyword][0] self._set_token_matched(token, token_type, match.group(3).strip(), matchedKeyword, keyword_type=matchedKeywordType, indent=indent) diff --git a/python/test/gherkin_in_markdown_token_matcher_test.py b/python/test/gherkin_in_markdown_token_matcher_test.py index fecd7d14f..309f1d45a 100644 --- a/python/test/gherkin_in_markdown_token_matcher_test.py +++ b/python/test/gherkin_in_markdown_token_matcher_test.py @@ -59,6 +59,7 @@ def test_it_matches_plus_Step(): assert tm.match_StepLine(token) assert token.matched_type == 'StepLine' assert token.matched_keyword == 'Given ' + assert token.matched_keyword_type == 'Context' assert token.matched_text == 'I have 3 cukes' assert token.location['column'] == 6 @@ -69,9 +70,22 @@ def test_it_matches_hyphen_Step(): assert tm.match_StepLine(token) assert token.matched_type == 'StepLine' assert token.matched_keyword == 'Given ' + assert token.matched_keyword_type == 'Context' assert token.matched_text == 'I have 3 cukes' assert token.location['column'] == 6 +def test_it_matches_a_when_Step(): + tm = GherkinInMarkdownTokenMatcher('en') + line = GherkinLine(''' - When I do something''',location['line']) + token = Token(gherkin_line=line, location=location) + assert tm.match_StepLine(token) + assert token.matched_type == 'StepLine' + assert token.matched_keyword == 'When ' + assert token.matched_keyword_type == 'Action' + assert token.matched_text == 'I do something' + assert token.location['column'] == 6 + + def test_it_matches_arbitrary_text_as_Other(): tm = GherkinInMarkdownTokenMatcher('en') line = GherkinLine('''Whatever''',location['line']) From 12a3a9333bdea03779b30b6fdc4f98de10f6e439 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:02:49 +0800 Subject: [PATCH 13/18] test: generate markdown pickles using python implementation to remove use of uuid --- testdata/good/datatables.feature.md.pickles.ndjson | 2 +- testdata/good/docstrings.feature.md.pickles.ndjson | 2 +- testdata/good/misc.feature.md.pickles.ndjson | 4 ++-- testdata/good/tags.feature.md.pickles.ndjson | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testdata/good/datatables.feature.md.pickles.ndjson b/testdata/good/datatables.feature.md.pickles.ndjson index 75eeb8b7b..e7bafe766 100644 --- a/testdata/good/datatables.feature.md.pickles.ndjson +++ b/testdata/good/datatables.feature.md.pickles.ndjson @@ -1 +1 @@ -{"pickle":{"id":"d5ba0306-6c4b-4a27-8032-581b16fcfb94","uri":"../testdata/good/datatables.feature.md","astNodeIds":["86ac77b8-9928-4327-ac49-ee5bc7b92727"],"tags":[],"name":"minimalistic","language":"en","steps":[{"id":"c30710bb-7578-4a65-8538-3ca3a00364c2","text":"a simple data table","type":"Context","argument":{"dataTable":{"rows":[{"cells":[{"value":"foo"},{"value":"bar"}]},{"cells":[{"value":"boz"},{"value":"boo"}]}]}},"astNodeIds":["5191c118-2880-4a3b-a0b9-dad8040e2b46"]}]}} +{"pickle":{"astNodeIds":["3"],"id":"5","language":"en","name":"minimalistic","steps":[{"argument":{"dataTable":{"rows":[{"cells":[{"value":"foo"},{"value":"bar"}]},{"cells":[{"value":"boz"},{"value":"boo"}]}]}},"astNodeIds":["2"],"id":"4","text":"a simple data table","type":"Context"}],"tags":[],"uri":"../testdata/good/datatables.feature.md"}} diff --git a/testdata/good/docstrings.feature.md.pickles.ndjson b/testdata/good/docstrings.feature.md.pickles.ndjson index 3915c77f8..80cac1a47 100644 --- a/testdata/good/docstrings.feature.md.pickles.ndjson +++ b/testdata/good/docstrings.feature.md.pickles.ndjson @@ -1 +1 @@ -{"pickle":{"id":"24839cef-9f43-4ffa-9475-6ae2e533d8c9","uri":"../testdata/good/docstrings.feature.md","astNodeIds":["c11dca0c-4ca3-4c15-9036-6a01e080aa30"],"tags":[],"name":"minimalistic","language":"en","steps":[{"id":"70b74b30-a01b-423e-ad70-5ddb01f9235f","text":"a DocString with an implicitly escaped separator inside","type":"Unknown","argument":{"docString":{"content":"```"}},"astNodeIds":["52940dab-b378-4d20-970b-3967c427fc9b"]}]}} +{"pickle":{"astNodeIds":["1"],"id":"3","language":"en","name":"minimalistic","steps":[{"argument":{"docString":{"content":"```"}},"astNodeIds":["0"],"id":"2","text":"a DocString with an implicitly escaped separator inside","type":"Unknown"}],"tags":[],"uri":"../testdata/good/docstrings.feature.md"}} diff --git a/testdata/good/misc.feature.md.pickles.ndjson b/testdata/good/misc.feature.md.pickles.ndjson index 6080c88de..cca046e5a 100644 --- a/testdata/good/misc.feature.md.pickles.ndjson +++ b/testdata/good/misc.feature.md.pickles.ndjson @@ -1,2 +1,2 @@ -{"pickle":{"id":"64494de7-bbeb-440a-bafd-811fa58b0e14","uri":"../testdata/good/misc.feature.md","astNodeIds":["6bc6758d-5013-4dbe-9234-0980b2044d1e"],"tags":[],"name":"Something about math","language":"en","steps":[{"id":"acbbba34-bed3-4082-8dc4-f20abf273674","text":"step one","type":"Context","astNodeIds":["6cadcc23-9981-41de-9e33-9e8ade163b5f"]},{"id":"e85a565a-3936-4e76-8949-34db37567e3e","text":"step two","type":"Action","astNodeIds":["f9e5934e-8d46-4220-8ab3-1ed48a59a7e8"]},{"id":"3752db1e-c2fa-4db1-aec4-442e862ff678","text":"step three","type":"Outcome","astNodeIds":["3d44f4f1-9d04-47cf-aec7-71bc08c23a51"]}]}} -{"pickle":{"id":"cdf3e41c-5f12-4b42-a8c5-99e5a30391de","uri":"../testdata/good/misc.feature.md","astNodeIds":["c9e5cfc4-7e16-49bc-a266-729c654e78e7"],"tags":[],"name":"Something about gravity","language":"en","steps":[{"id":"be702179-8d6f-4bb9-ad71-f04d2e10dd96","text":"step one","type":"Context","astNodeIds":["4f146301-cfcd-47a8-b6f1-bc3bfd224e0c"]},{"id":"003e2077-0187-4626-b10d-d3fd4bef15d0","text":"step two","type":"Action","astNodeIds":["84ded463-4b37-4817-98e0-26e2b8ea16ae"]},{"id":"5fa0b049-fbf4-47c8-9c22-2e85621aeb7f","text":"step three","type":"Outcome","astNodeIds":["4a429963-a224-460a-a001-33ba6e70948f"]}]}} +{"pickle":{"astNodeIds":["3"],"id":"11","language":"en","name":"Something about math","steps":[{"astNodeIds":["0"],"id":"8","text":"step one","type":"Context"},{"astNodeIds":["1"],"id":"9","text":"step two","type":"Action"},{"astNodeIds":["2"],"id":"10","text":"step three","type":"Outcome"}],"tags":[],"uri":"../testdata/good/misc.feature.md"}} +{"pickle":{"astNodeIds":["7"],"id":"15","language":"en","name":"Something about gravity","steps":[{"astNodeIds":["4"],"id":"12","text":"step one","type":"Context"},{"astNodeIds":["5"],"id":"13","text":"step two","type":"Action"},{"astNodeIds":["6"],"id":"14","text":"step three","type":"Outcome"}],"tags":[],"uri":"../testdata/good/misc.feature.md"}} diff --git a/testdata/good/tags.feature.md.pickles.ndjson b/testdata/good/tags.feature.md.pickles.ndjson index 8d4d90a5d..a82bd44b9 100644 --- a/testdata/good/tags.feature.md.pickles.ndjson +++ b/testdata/good/tags.feature.md.pickles.ndjson @@ -1,6 +1,6 @@ -{"pickle":{"id":"3f92a2c2-1ad2-4575-931e-a99bf47dfb21","uri":"../testdata/good/tags.feature.md","astNodeIds":["2277e04c-1300-4100-abd6-063eabad0c38"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@scenario_tag1","astNodeId":"46212da8-98c6-4a5d-b2f1-d4f3ca93e97f"},{"name":"@scenario_tag2","astNodeId":"0b45ea80-2441-455b-9afc-78eacfa3898f"},{"name":"@scenario_tag3","astNodeId":"fceb7155-c9b2-4898-8cb2-2df415d880c8"}],"name":"minimalistic","language":"en","steps":[{"id":"4ffa994f-63cc-4fa7-a73a-477f55f458db","text":"the minimalism","type":"Context","astNodeIds":["a1830df8-62b7-4db2-ad62-d3cca34b7e0f"]}]}} -{"pickle":{"id":"94bcef16-e73c-4840-882b-3dfc5b2f9739","uri":"../testdata/good/tags.feature.md","astNodeIds":["3d3adf52-351d-4a70-ae8d-a65a6f812eee","51b3f85c-e844-41e8-a981-398e3d30ad17"],"name":"minimalistic outline","language":"en","steps":[{"id":"f93c39df-2a8a-4f62-904d-8144df5c7bb7","text":"the minimalism","type":"Context","astNodeIds":["45e8a2fa-c328-4287-81c5-e15c7b6cf976","51b3f85c-e844-41e8-a981-398e3d30ad17"]}],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@so_tag1","astNodeId":"fe85917e-4674-4fc3-8cdc-eefe1d481bb3"},{"name":"@so_tag2","astNodeId":"34e2c4ca-7580-4e92-b1d5-fb4c75df0118"},{"name":"@so_tag3","astNodeId":"2f7ab5be-0728-4ea8-b9db-33562c11eb2d"},{"name":"@ex_tag1","astNodeId":"5f8272ca-c756-416f-8217-5802a3ef96e4"},{"name":"@ex_tag2","astNodeId":"da57b49d-edac-449e-8313-264bee4e89e9"},{"name":"@ex_tag3","astNodeId":"41b605cb-6494-40d8-ac28-457cd4b33a96"}]}} -{"pickle":{"id":"81abe3b6-9f02-47b7-8f42-1f6a2be46e78","uri":"../testdata/good/tags.feature.md","astNodeIds":["3d3adf52-351d-4a70-ae8d-a65a6f812eee","b1495c13-2bf0-438c-ae09-ec41dfcdf22d"],"name":"minimalistic outline","language":"en","steps":[{"id":"85fc687c-7b39-4d06-8afc-7b692aab407e","text":"the more minimalism","type":"Context","astNodeIds":["45e8a2fa-c328-4287-81c5-e15c7b6cf976","b1495c13-2bf0-438c-ae09-ec41dfcdf22d"]}],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@so_tag1","astNodeId":"fe85917e-4674-4fc3-8cdc-eefe1d481bb3"},{"name":"@so_tag2","astNodeId":"34e2c4ca-7580-4e92-b1d5-fb4c75df0118"},{"name":"@so_tag3","astNodeId":"2f7ab5be-0728-4ea8-b9db-33562c11eb2d"},{"name":"@ex_tag4","astNodeId":"fd0630c2-43b4-462f-800d-436404154c34"},{"name":"@ex_tag5","astNodeId":"d7752694-7c4d-4d78-b34b-7a5be2cfd0d4"},{"name":"@ex_tag6","astNodeId":"c7c520ed-49e0-488c-94f9-1e519e2f08f6"}]}} -{"pickle":{"id":"522516c3-040d-4f2f-9263-d88ba42fb0e3","uri":"../testdata/good/tags.feature.md","astNodeIds":["ba2ec921-8506-4415-93ec-7c29fa724785"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@comment_tag1","astNodeId":"a4c1a4f3-3fd1-4e18-a013-3257d2ec5f89"}],"name":"comments","language":"en","steps":[]}} -{"pickle":{"id":"664b327b-bb2e-41e6-8990-38a10a14c84c","uri":"../testdata/good/tags.feature.md","astNodeIds":["92be8ee0-135d-44fe-9814-0a2bb8ff7150"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@comment_tag#2","astNodeId":"ccf9263d-2032-4eb5-87a0-05940efb31c8"}],"name":"hash in tags","language":"en","steps":[]}} -{"pickle":{"id":"925937d9-cc4b-4c0e-b8f2-25062f3be01a","uri":"../testdata/good/tags.feature.md","astNodeIds":["a5de7f97-8b0f-4235-80ac-4ce0057c87a3"],"tags":[{"name":"@feature_tag1","astNodeId":"0f025fc8-9fab-4cce-8d9d-5e0ce3da5aed"},{"name":"@feature_tag2","astNodeId":"89907a74-65f8-4a5c-9a29-8b04ea06eb38"},{"name":"@feature_tag3","astNodeId":"91ffb155-74a2-4664-9adb-dd931cdf54f1"},{"name":"@rule_tag","astNodeId":"b8caf880-2cd8-43ec-9b94-4158c112e992"},{"name":"@joined_tag3","astNodeId":"680d9c7e-95c3-486e-adf0-818ddcf6141f"},{"name":"@joined_tag4","astNodeId":"540d3559-ae0d-4f81-9ef0-bd8746e467f4"}],"name":"joined tags","language":"en","steps":[]}} +{"pickle":{"astNodeIds":["4"],"id":"35","language":"en","name":"minimalistic","steps":[{"astNodeIds":["0"],"id":"34","text":"the minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"1","name":"@scenario_tag1"},{"astNodeId":"2","name":"@scenario_tag2"},{"astNodeId":"3","name":"@scenario_tag3"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"astNodeIds":["21","7"],"id":"37","language":"en","name":"minimalistic outline","steps":[{"astNodeIds":["5","7"],"id":"36","text":"the minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"18","name":"@so_tag1"},{"astNodeId":"19","name":"@so_tag2"},{"astNodeId":"20","name":"@so_tag3"},{"astNodeId":"8","name":"@ex_tag1"},{"astNodeId":"9","name":"@ex_tag2"},{"astNodeId":"10","name":"@ex_tag3"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"astNodeIds":["21","13"],"id":"39","language":"en","name":"minimalistic outline","steps":[{"astNodeIds":["5","13"],"id":"38","text":"the more minimalism","type":"Context"}],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"18","name":"@so_tag1"},{"astNodeId":"19","name":"@so_tag2"},{"astNodeId":"20","name":"@so_tag3"},{"astNodeId":"14","name":"@ex_tag4"},{"astNodeId":"15","name":"@ex_tag5"},{"astNodeId":"16","name":"@ex_tag6"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"astNodeIds":["23"],"id":"40","language":"en","name":"comments","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"22","name":"@comment_tag1"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"astNodeIds":["25"],"id":"41","language":"en","name":"hash in tags","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"24","name":"@comment_tag#2"}],"uri":"../testdata/good/tags.feature.md"}} +{"pickle":{"astNodeIds":["28"],"id":"42","language":"en","name":"joined tags","steps":[],"tags":[{"astNodeId":"31","name":"@feature_tag1"},{"astNodeId":"32","name":"@feature_tag2"},{"astNodeId":"33","name":"@feature_tag3"},{"astNodeId":"29","name":"@rule_tag"},{"astNodeId":"26","name":"@joined_tag3"},{"astNodeId":"27","name":"@joined_tag4"}],"uri":"../testdata/good/tags.feature.md"}} From c0b94675e5ecb37d55e2c3613567b6e8bd4d2647 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:03:43 +0800 Subject: [PATCH 14/18] feat: Python - implement mediaType for markdown features --- python/gherkin/stream/source_events.py | 9 ++++++++- python/test/source_events_test.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 python/test/source_events_test.py diff --git a/python/gherkin/stream/source_events.py b/python/gherkin/stream/source_events.py index 946b6bc62..c5c5e1aa5 100644 --- a/python/gherkin/stream/source_events.py +++ b/python/gherkin/stream/source_events.py @@ -5,11 +5,18 @@ def source_event(path): 'source': { 'uri': path, 'data': io.open(path, 'r', encoding='utf8', newline='').read(), - 'mediaType': 'text/x.cucumber.gherkin+plain' + 'mediaType': _media_type(path) } } return event + +def _media_type(path): + if(path.endswith(".feature")): + return 'text/x.cucumber.gherkin+plain' + if(path.endswith(".feature.md")): + return 'text/x.cucumber.gherkin+markdown' + class SourceEvents: def __init__(self, paths): self.paths = paths diff --git a/python/test/source_events_test.py b/python/test/source_events_test.py new file mode 100644 index 000000000..c0d883897 --- /dev/null +++ b/python/test/source_events_test.py @@ -0,0 +1,11 @@ +from gherkin.stream.source_events import source_event + +from os import path + +def test_should_resolve_classic_feature(): + actual = source_event(path.join(path.dirname(__file__),"..","..","testdata","good","minimal.feature")) + assert actual['source']['mediaType'] == 'text/x.cucumber.gherkin+plain' + +def test_should_resolve_markdown_feature(): + actual = source_event(path.join(path.dirname(__file__),"..","..","testdata","good","minimal.feature.md")) + assert actual['source']['mediaType'] == 'text/x.cucumber.gherkin+markdown' From 16fbb63912d4865414ee4722088c23f7e3abad14 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:11:12 +0800 Subject: [PATCH 15/18] revert: re-generate parser.py using berp. No changes were necessary. --- python/gherkin/parser.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/gherkin/parser.py b/python/gherkin/parser.py index 617330bc6..1c0baad9b 100644 --- a/python/gherkin/parser.py +++ b/python/gherkin/parser.py @@ -250,11 +250,6 @@ def match_token_at_0(self, token, context): if self.match_Empty(context, token): self.build(context, token) return 0 - # FIXME - move into gherkin-python.razor - # if self.match_ScenarioLine(context,token): - # self.build(context, token) - # return 12 - state_comment = "State: 0 - Start" token.detach From 7f1790c94102aae29a62d2f7e830f7781d485d26 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:30:52 +0800 Subject: [PATCH 16/18] chore: remove dead code --- python/test/gherkin_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/test/gherkin_test.py b/python/test/gherkin_test.py index eea3cf506..13723ebca 100644 --- a/python/test/gherkin_test.py +++ b/python/test/gherkin_test.py @@ -136,7 +136,6 @@ def test_parsing_markdown_parses_a_feature_without_a_hash_Feature_header(): 'column': 1, }, 'language': 'en', - # 'keyword': None, 'name': '# Hello', 'description': '', 'children': [ @@ -161,8 +160,6 @@ def test_parsing_markdown_parses_a_feature_without_a_hash_Feature_header(): 'keyword': 'Given ', 'keywordType': "Context", 'text': 'a step', - # 'dataTable': None, - # 'docString': None, }, ], 'examples': [], From 6204f3007d51cb67a339c5e3a3848af41b45cdc3 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 17:39:14 +0800 Subject: [PATCH 17/18] fix: update markdown test data using Python implementation following support for matched_keyword_type --- testdata/good/docstrings.feature.md.ast.ndjson | 2 +- testdata/good/docstrings.feature.md.tokens | 2 +- testdata/good/misc.feature.md.ast.ndjson | 2 +- testdata/good/misc.feature.md.tokens | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/testdata/good/docstrings.feature.md.ast.ndjson b/testdata/good/docstrings.feature.md.ast.ndjson index f1fdb4a4d..7139bcd46 100644 --- a/testdata/good/docstrings.feature.md.ast.ndjson +++ b/testdata/good/docstrings.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"docString":{"content":"```","delimiter":"````","location":{"column":1,"line":6}},"id":"0","keyword":"And ","location":{"column":3,"line":5},"text":"a DocString with an implicitly escaped separator inside"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DocString variations","tags":[]},"uri":"../testdata/good/docstrings.feature.md"}} +{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":5,"line":3},"name":"minimalistic","steps":[{"docString":{"content":"```","delimiter":"````","location":{"column":1,"line":6}},"id":"0","keyword":"And ","keywordType":"Conjunction","location":{"column":3,"line":5},"text":"a DocString with an implicitly escaped separator inside"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":4,"line":1},"name":"DocString variations","tags":[]},"uri":"../testdata/good/docstrings.feature.md"}} diff --git a/testdata/good/docstrings.feature.md.tokens b/testdata/good/docstrings.feature.md.tokens index 990c0ead2..41bce1621 100644 --- a/testdata/good/docstrings.feature.md.tokens +++ b/testdata/good/docstrings.feature.md.tokens @@ -2,7 +2,7 @@ (2:1)Empty:// (3:5)ScenarioLine:()Scenario/minimalistic/ (4:1)Empty:// -(5:3)StepLine:()And /a DocString with an implicitly escaped separator inside/ +(5:3)StepLine:(Conjunction)And /a DocString with an implicitly escaped separator inside/ (6:1)DocStringSeparator:()````// (7:1)Other:/```/ (8:1)DocStringSeparator:()````// diff --git a/testdata/good/misc.feature.md.ast.ndjson b/testdata/good/misc.feature.md.ast.ndjson index 10254af10..f04397ef2 100644 --- a/testdata/good/misc.feature.md.ast.ndjson +++ b/testdata/good/misc.feature.md.ast.ndjson @@ -1 +1 @@ -{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":3,"line":12},"name":"Something about math","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"step one"},{"id":"1","keyword":"When ","location":{"column":3,"line":14},"text":"step two"},{"id":"2","keyword":"Then ","location":{"column":3,"line":15},"text":"step three"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":17},"name":"Something about gravity","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":4,"line":18},"text":"step one"},{"id":"5","keyword":"When ","location":{"column":4,"line":19},"text":"step two"},{"id":"6","keyword":"Then ","location":{"column":4,"line":20},"text":"step three"}],"tags":[]}}],"description":"","language":"en","location":{"column":1,"line":1},"name":"Markdown document without \"# Feature:\" header","tags":[]},"uri":"../testdata/good/misc.feature.md"}} +{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"3","keyword":"Scenario","location":{"column":3,"line":12},"name":"Something about math","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":13},"text":"step one"},{"id":"1","keyword":"When ","keywordType":"Action","location":{"column":3,"line":14},"text":"step two"},{"id":"2","keyword":"Then ","keywordType":"Outcome","location":{"column":3,"line":15},"text":"step three"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":17},"name":"Something about gravity","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":4,"line":18},"text":"step one"},{"id":"5","keyword":"When ","keywordType":"Action","location":{"column":4,"line":19},"text":"step two"},{"id":"6","keyword":"Then ","keywordType":"Outcome","location":{"column":4,"line":20},"text":"step three"}],"tags":[]}}],"description":"","language":"en","location":{"column":1,"line":1},"name":"Markdown document without \"# Feature:\" header","tags":[]},"uri":"../testdata/good/misc.feature.md"}} diff --git a/testdata/good/misc.feature.md.tokens b/testdata/good/misc.feature.md.tokens index dd8458784..f34c19738 100644 --- a/testdata/good/misc.feature.md.tokens +++ b/testdata/good/misc.feature.md.tokens @@ -11,13 +11,13 @@ (11:1)Empty:// (12:3)ScenarioLine:()Scenario/Something about math/ (13:3)StepLine:(Context)Given /step one/ -(14:3)StepLine:()When /step two/ -(15:3)StepLine:()Then /step three/ +(14:3)StepLine:(Action)When /step two/ +(15:3)StepLine:(Outcome)Then /step three/ (16:1)Empty:// (17:3)ScenarioLine:()Scenario/Something about gravity/ (18:4)StepLine:(Context)Given /step one/ -(19:4)StepLine:()When /step two/ -(20:4)StepLine:()Then /step three/ +(19:4)StepLine:(Action)When /step two/ +(20:4)StepLine:(Outcome)Then /step three/ (21:1)Empty:// (22:1)Empty:// (23:1)Empty:// From 18b1cf02a5363cea9abead1456fdc04fc4008f44 Mon Sep 17 00:00:00 2001 From: temyers Date: Tue, 27 Dec 2022 18:51:49 +0800 Subject: [PATCH 18/18] test: add tests to demonstrate inconsistency between Javascript and Python Markdown implementations --- javascript/Makefile | 1 + .../src/GherkinInMarkdownTokenMatcher.ts | 8 +- javascript/test/ParserTest.ts | 170 +++++++++++++++--- python/gherkin/token_matcher_markdown.py | 16 +- python/test/gherkin_test.py | 122 ++++++++++++- 5 files changed, 289 insertions(+), 28 deletions(-) diff --git a/javascript/Makefile b/javascript/Makefile index 1403bb31f..a6df5f96a 100644 --- a/javascript/Makefile +++ b/javascript/Makefile @@ -5,6 +5,7 @@ GHERKIN_PARSER = src/Parser.ts GHERKIN_RAZOR = gherkin-javascript.razor SOURCE_FILES = $(shell find . -name "*.js" | grep -v $(GHERKIN_PARSER)) +# HELP - is this correnct now? GHERKIN = npx gherkin-javascript GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature" -o -name "*.feature.md") diff --git a/javascript/src/GherkinInMarkdownTokenMatcher.ts b/javascript/src/GherkinInMarkdownTokenMatcher.ts index 742645a0f..5b7356411 100644 --- a/javascript/src/GherkinInMarkdownTokenMatcher.ts +++ b/javascript/src/GherkinInMarkdownTokenMatcher.ts @@ -130,7 +130,13 @@ export default class GherkinInMarkdownTokenMatcher implements ITokenMatcher { let parser: Parser beforeEach( () => - (parser = new Parser( - new AstBuilder(messages.IdGenerator.incrementing()), - new GherkinClassicTokenMatcher() - )) + (parser = new Parser( + new AstBuilder(messages.IdGenerator.incrementing()), + new GherkinClassicTokenMatcher() + )) ) it('parses a simple feature', function () { @@ -93,20 +94,20 @@ describe('Parser', function () { try { parser.parse( '# a comment\n' + - 'Feature: Foo\n' + - ' Scenario: Bar\n' + - ' Given x\n' + - ' ```\n' + - ' unclosed docstring\n' + 'Feature: Foo\n' + + ' Scenario: Bar\n' + + ' Given x\n' + + ' ```\n' + + ' unclosed docstring\n' ) } catch (expected) { ast = parser.parse( 'Feature: Foo\n' + - ' Scenario: Bar\n' + - ' Given x\n' + - ' """\n' + - ' closed docstring\n' + - ' """' + ' Scenario: Bar\n' + + ' Given x\n' + + ' """\n' + + ' closed docstring\n' + + ' """' ) } @@ -155,11 +156,11 @@ describe('Parser', function () { it('interpolates data tables', function () { const envelopes = generateMessages( 'Feature: Foo\n' + - ' Scenario Outline: Parenthesis\n' + - ' Given the thing and has \n' + - ' Examples:\n' + - ' | is (not) triggered | value |\n' + - ' | is triggered | foo |\n ', + ' Scenario Outline: Parenthesis\n' + + ' Given the thing and has \n' + + ' Examples:\n' + + ' | is (not) triggered | value |\n' + + ' | is triggered | foo |\n ', '', messages.SourceMediaType.TEXT_X_CUCUMBER_GHERKIN_PLAIN, { includePickles: true, newId: messages.IdGenerator.incrementing() } @@ -194,10 +195,10 @@ describe('Parser', function () { let parser: Parser beforeEach( () => - (parser = new Parser( - new AstBuilder(messages.IdGenerator.incrementing()), - new GherkinInMarkdownTokenMatcher() - )) + (parser = new Parser( + new AstBuilder(messages.IdGenerator.incrementing()), + new GherkinInMarkdownTokenMatcher() + )) ) it('does not parse a feature description', function () { @@ -308,5 +309,128 @@ description assert.strictEqual(pickle.steps[0].argument.docString.content, '```what') }) + + it("parses Markdown data tables with headers", () => { + const markdown = `## Feature: DataTables + +### Scenario: minimalistic + +* Given a simple data table + | foo | bar | + | --- | --- | + | boz | boo | +` + const ast = parser.parse(markdown) + const gherkinDocument: messages.GherkinDocument = { + "comments": [ + { + "location": { + "column": 3, + "line": 7 + }, + "text": undefined + } + ], + "feature": { + "children": [ + { + "scenario": { + "description": "", + "examples": [], + "id": "3", + "keyword": "Scenario", + "location": { + "column": 5, + "line": 3 + }, + "name": "minimalistic", + "steps": [ + { + "dataTable": { + "location": { + "column": 3, + "line": 6 + }, + "rows": [ + { + "cells": [ + { + "location": { + "column": 5, + "line": 6 + }, + "value": "foo" + }, + { + "location": { + "column": 11, + "line": 6 + }, + "value": "bar" + } + ], + "id": "0", + "location": { + "column": 3, + "line": 6 + } + }, + { + "cells": [ + { + "location": { + "column": 5, + "line": 8 + }, + "value": "boz" + }, + { + "location": { + "column": 11, + "line": 8 + }, + "value": "boo" + } + ], + "id": "1", + "location": { + "column": 3, + "line": 8 + } + } + ] + }, + "id": "2", + docString: undefined, + "keyword": "Given ", + "keywordType": StepKeywordType.CONTEXT, + "location": { + "column": 3, + "line": 5 + }, + "text": "a simple data table" + } + ], + "tags": [] + } + } + ], + "description": "", + "keyword": "Feature", + "language": "en", + "location": { + "column": 4, + "line": 1 + }, + "name": "DataTables", + "tags": [] + } + } + assert.deepStrictEqual(ast, gherkinDocument) + }) + }) + + }) + diff --git a/python/gherkin/token_matcher_markdown.py b/python/gherkin/token_matcher_markdown.py index 02c8e930e..9197080c2 100644 --- a/python/gherkin/token_matcher_markdown.py +++ b/python/gherkin/token_matcher_markdown.py @@ -81,12 +81,13 @@ def match_StepLine(self, token): return self._match_title_line(KEYWORD_PREFIX_BULLET, nonStarStepKeywords, '', token, 'StepLine') def match_Comment(self, token): + result = False if(token.line.startswith('|')): table_cells = token.line.table_cells if(self._is_gfm_table_separator(table_cells)): - self._set_token_matched(token,"Comment") - return True - return self._set_token_matched(token,None) + result= True + self._set_token_matched(token, "Comment") + return result def match_Empty(self, token): @@ -196,6 +197,15 @@ def _match_title_line(self, prefix, keywords, keywordSuffix, token, token_type): return True return False + def _set_token_matched2(self, token, matched, indent=None, ): + token.matched_gherkin_dialect = self.dialect_name + if indent is not None: + token.matched_indent = indent + else: + token.matched_indent = token.line.indent if token.line else 0 + token.location['column'] = token.matched_indent + 1 + return matched + def _set_token_matched(self, token, matched_type, text=None, keyword=None, keyword_type=None, indent=None, items=None): if items is None: diff --git a/python/test/gherkin_test.py b/python/test/gherkin_test.py index 13723ebca..71edf9f2e 100644 --- a/python/test/gherkin_test.py +++ b/python/test/gherkin_test.py @@ -169,4 +169,124 @@ def test_parsing_markdown_parses_a_feature_without_a_hash_Feature_header(): }, 'comments': [], } - assert ast == expected \ No newline at end of file + assert ast == expected + +def test_it_parses_markdown_data_tables_with_headers(): + parser = Parser() + matcher = GherkinInMarkdownTokenMatcher() + markdown = """## Feature: DataTables + +### Scenario: minimalistic + +* Given a simple data table + | foo | bar | + | --- | --- | + | boz | boo | +""" + ast = parser.parse(TokenScanner(markdown), matcher) + expected = { + "comments": [ + { + "location": { + "column": 3, + "line": 7 + }, + "text": None + } + ], + "feature": { + "children": [ + { + "scenario": { + "description": "", + "examples": [], + "id": "3", + "keyword": "Scenario", + "location": { + "column": 5, + "line": 3 + }, + "name": "minimalistic", + "steps": [ + { + "dataTable": { + "location": { + "column": 3, + "line": 6 + }, + "rows": [ + { + "cells": [ + { + "location": { + "column": 5, + "line": 6 + }, + "value": "foo" + }, + { + "location": { + "column": 11, + "line": 6 + }, + "value": "bar" + } + ], + "id": "0", + "location": { + "column": 3, + "line": 6 + } + }, + { + "cells": [ + { + "location": { + "column": 5, + "line": 8 + }, + "value": "boz" + }, + { + "location": { + "column": 11, + "line": 8 + }, + "value": "boo" + } + ], + "id": "1", + "location": { + "column": 3, + "line": 8 + } + } + ] + }, + "id": "2", + # "docString": None, + "keyword": "Given ", + "keywordType": "Context", + "location": { + "column": 3, + "line": 5 + }, + "text": "a simple data table" + } + ], + "tags": [] + } + } + ], + "description": "", + "keyword": "Feature", + "language": "en", + "location": { + "column": 4, + "line": 1 + }, + "name": "DataTables", + "tags": [] + } + } + assert ast == expected