diff --git a/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs b/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs index ad51abe2..5fea1947 100644 --- a/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs +++ b/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs @@ -385,7 +385,10 @@ public bool InEmbed(string marker) public bool IsInNestedEmbed(string marker) { - return _inNestedEmbed || (marker.StartsWith("+") && marker.Length > 1 && IsEmbedPart(marker.Substring(1))); + return _inNestedEmbed + || ( + !(marker is null) && marker.StartsWith("+") && marker.Length > 1 && IsEmbedPart(marker.Substring(1)) + ); } private static bool IsNoteText(string marker) @@ -400,7 +403,7 @@ public static bool IsEmbedPart(string marker) private static bool IsEmbedCharacter(string marker) { - return marker.IsOneOf("f", "fe", "fig", "fm", "x"); + return !(marker is null) && marker.IsOneOf("f", "fe", "fig", "fm", "x"); } } } diff --git a/tests/SIL.Machine.Tests/Corpora/TestData/usfm/Tes/41MATTes.SFM b/tests/SIL.Machine.Tests/Corpora/TestData/usfm/Tes/41MATTes.SFM index 24ce7004..88259f13 100644 --- a/tests/SIL.Machine.Tests/Corpora/TestData/usfm/Tes/41MATTes.SFM +++ b/tests/SIL.Machine.Tests/Corpora/TestData/usfm/Tes/41MATTes.SFM @@ -1,13 +1,13 @@ \id MAT - Test \h Matthew \mt Matthew -\ip An introduction to Matthew with an empty comment\fe + \ft \fe* +\ip An introduction to Matthew\fe + \ft This is an endnote.\fe* \p \rq MAT 1\rq* Here is another paragraph. \p and with a \w keyword|a special concept\w* in it. \p and a \weirdtaglookingthing that is not an actual tag. \c 1 \s Chapter One -\v 1 Chapter \pn one\+pro WON\+pro*\pn*, verse \f + \fr 1:1: \ft This is a footnote for v1.\f*one. +\v 1 Chapter \pn one\+pro WON\+pro*\pn*, verse one.\f + \fr 1:1: \ft This is a footnote for v1.\f* \li1 \v 2 \bd C\bd*hapter one, \li2 verse\f + \fr 1:2: \ft This is a footnote for v2.\f* two. @@ -26,7 +26,7 @@ \s1 Chapter \it Two \it* \p \p -\v 1 Chapter \add two\add*, verse \f + \fr 2:1: \ft This is a \bd footnote.\bd*\f*one. +\v 1 Chapter \add two\add*, verse \f + \fr 2:1: \ft This is a footnote.\f*one. \v 2-3 Chapter two, // verse \fm ∆\fm*two. \esb \ms This is a sidebar @@ -37,9 +37,9 @@ \p \v 6 Chapter two, verse \w six|strong="12345" \w*. \p -\v 6 Bad verse. \x - \xo 2:3-4 \xt Cool Book 3:24 \xta The annotation \x* and more content. +\v 6 Bad verse. \x - \xo abc\xt 123\x* and more content. \p -\v 5 Chapter two, verse five\rq (MAT 3:1)\rq*. +\v 5 Chapter two, verse five \rq (MAT 3:1)\rq*. \v 7a Chapter two, verse seven A, \s Section header \ts-s\* \p @@ -54,7 +54,6 @@ \v 11-12 \restore restore information \c 3 -\r (Mark 1:2-3; Luke 4:5-6) \cl PSALM 3 \s1 Section 1 \mt1 Major Title 1 diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index 3a81e17d..40641fe8 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -45,13 +45,25 @@ public void GetUsfm_PreferExisting() { var rows = new List<(IReadOnlyList, string)> { - (ScrRef("MAT 1:6"), "Text 6"), - (ScrRef("MAT 1:7"), "Text 7"), + (ScrRef("MAT 1:6"), "Update 1"), + (ScrRef("MAT 1:7"), "Update 2"), }; - string target = UpdateUsfm(rows, textBehavior: UpdateUsfmTextBehavior.PreferExisting); - Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); - Assert.That(target, Contains.Substring("\\v 6 Verse 6 content.\r\n")); - Assert.That(target, Contains.Substring("\\v 7 Text 7\r\n")); + var usfm = + @"\id MAT - Test +\c 1 +\v 1 Some text +\v 2 +\v 3 Other text +"; + string target = UpdateUsfm(rows, usfm, textBehavior: UpdateUsfmTextBehavior.PreferExisting); + var result = + @"\id MAT - Test +\c 1 +\v 1 Some text +\v 2 Update 2 +\v 3 Other text +"; + Assess(target, result); } [Test] @@ -81,74 +93,25 @@ public void GetUsfm_Verse_StripNote() } [Test] - public void GetUsfm_Verse_StripNotesWithUpdatedVerseText() - { - var rows = new List<(IReadOnlyList, string)> - { - (ScrRef("MAT 1:1"), "First verse of the first chapter.") - }; - - string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); - Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); - Assert.That( - target, - Contains.Substring("\\ip An introduction to Matthew with an empty comment\\fe + \\ft \\fe*") - ); - Assert.That(target, Contains.Substring("\\v 1 First verse of the first chapter.\r\n\\li1\r\n\\v 2")); - } - - [Test] - public void GetUsfm_Verse_ReplaceNoteKeepReference() + public void GetUsfm_Verse_ReplaceNote() { var rows = new List<(IReadOnlyList, string)> { - (ScrRef("MAT 2:1"), "First verse of the second chapter."), - (ScrRef("MAT 2:1/1:f"), "This is a new footnote.") + (ScrRef("MAT 1:1"), "updated text"), + (ScrRef("MAT 1:1/1:f"), "This is a new footnote.") }; - - string target = UpdateUsfm(rows); - Assert.That( - target, - Contains.Substring( - "\\v 1 First verse of the second chapter. \\f + \\fr 2:1: \\ft This is a new footnote. \\f*\r\n" - ) - ); - } - - [Test] - public void GetUsfm_Verse_PreserveFiguresAndReferences() - { - var rows = new List<(IReadOnlyList, string)> - { - // fig - (ScrRef("MAT 1:5"), "Fifth verse of the first chapter."), - (ScrRef("MAT 1:5/1:fig"), "figure text not updated"), - // r - (ScrRef("MAT 2:0/1:r"), "parallel reference not updated"), - // rq - (ScrRef("MAT 2:5/1:rq"), "quote reference not updated"), - // xo - (ScrRef("MAT 2:6/3:xo"), "Cross reference not update"), - // xt - (ScrRef("MAT 2:6/4:xt"), "cross reference - target reference not updated"), - // xta - (ScrRef("MAT 2:6/5:xta"), "cross reference annotation updated"), - }; - - string target = UpdateUsfm(rows); - Assert.That( - target, - Contains.Substring( - "\\v 5 Fifth verse of the first chapter.\r\n\\li2 \\fig Figure 1|src=\"image1.png\" size=\"col\" ref=\"1:5\"\\fig*\r\n\\v 6" - ) - ); - Assert.That(target, Contains.Substring("\\r (Mark 1:2-3; Luke 4:5-6)\r\n")); - Assert.That( - target, - Contains.Substring( - "\\v 6 Bad verse. \\x - \\xo 2:3-4 \\xt Cool Book 3:24 \\xta The annotation \\x* and more content.\r\n" - ) - ); + var usfm = + @"\id MAT - Test +\c 1 +\v 1 Chapter \add one\add*, verse \f + \fr 2:1: \ft This is a footnote.\f*one. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 updated text \f + \fr 2:1: \ft This is a new footnote. \f* +"; + Assess(target, result); } [Test] @@ -163,7 +126,7 @@ public void GetUsfm_Verse_RowVerseSegment() Assert.That( target, Contains.Substring( - "\\v 1 First verse of the second chapter. \\f + \\fr 2:1: \\ft This is a \\bd footnote.\\bd*\\f*\r\n" + "\\v 1 First verse of the second chapter. \\f + \\fr 2:1: \\ft This is a footnote.\\f*\r\n" ) ); } @@ -420,18 +383,6 @@ public void GetUsfm_NonVerse_Milestone() Assert.That(target, Contains.Substring("\\s A new section header. \\ts-s\\*\r\n")); } - [Test] - public void GetUsfm_NonVerse_KeepNote() - { - var rows = new List<(IReadOnlyList, string)> - { - (ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.") - }; - - string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Preserve); - Assert.That(target, Contains.Substring("\\ip The introductory paragraph. \\fe + \\ft \\fe*\r\n")); - } - [Test] public void GetUsfm_NonVerse_SkipNote() { @@ -513,7 +464,235 @@ public void GetUsfm_Verse_PretranslationsBeforeText() }; string target = UpdateUsfm(rows); - Assert.That(target, Contains.Substring("\\ip The introductory paragraph. \\fe + \\ft \\fe*\r\n")); + Assert.That( + target, + Contains.Substring("\\ip The introductory paragraph. \\fe + \\ft This is an endnote.\\fe*\r\n") + ); + } + + [Test] + public void EmbedStylePreservation() + { + var rows = new List<(IReadOnlyList, string)> + { + (ScrRef("MAT 1:1"), "Update the greeting"), + (ScrRef("MAT 1:1/1:f"), "Update the comment"), + (ScrRef("MAT 1:2"), "Update the greeting only"), + (ScrRef("MAT 1:3/1:f"), "Update the comment only"), + }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 Hello \f \fr 1.1 \ft Some \+bd note\+bd* \f*\bd World \bd* +\v 2 Good \f \fr 1.2 \ft Some other \+bd note\+bd* \f*\bd Morning \bd* +\v 3 Pleasant \f \fr 1.3 \ft A third \+bd note\+bd* \f*\bd Evening \bd* +"; + var target = UpdateUsfm( + rows, + usfm, + embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmIntraVerseMarkerBehavior.Preserve + ); + var resultPp = + @"\id MAT - Test +\c 1 +\v 1 Update the greeting \f \fr 1.1 \ft Update the comment \+bd \+bd*\f*\bd \bd* +\v 2 Update the greeting only \f \fr 1.2 \ft Some other \+bd note\+bd* \f*\bd \bd* +\v 3 Pleasant \f \fr 1.3 \ft Update the comment only \+bd \+bd*\f*\bd Evening \bd* +"; + Assess(target, resultPp); + + target = UpdateUsfm( + rows, + usfm, + embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip + ); + var resultPs = + @"\id MAT - Test +\c 1 +\v 1 Update the greeting \f \fr 1.1 \ft Update the comment \f* +\v 2 Update the greeting only \f \fr 1.2 \ft Some other \+bd note\+bd* \f* +\v 3 Pleasant \f \fr 1.3 \ft Update the comment only \f*\bd Evening \bd* +"; + Assess(target, resultPs); + + target = UpdateUsfm( + rows, + usfm, + embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip, + styleBehavior: UpdateUsfmIntraVerseMarkerBehavior.Preserve + ); + var resultSp = + @"\id MAT - Test +\c 1 +\v 1 Update the greeting \bd \bd* +\v 2 Update the greeting only \bd \bd* +\v 3 Pleasant \bd Evening \bd* +"; + Assess(target, resultSp); + + target = UpdateUsfm( + rows, + usfm, + embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip, + styleBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip + ); + var resultSs = + @"\id MAT - Test +\c 1 +\v 1 Update the greeting +\v 2 Update the greeting only +\v 3 Pleasant \bd Evening \bd* +"; + Assess(target, resultSs); + } + + [Test] + public void EmptyNote() + { + var rows = new List<(IReadOnlyList, string)> { (ScrRef("MAT 1:1/1:f"), "Update the note") }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 Empty Note \f \fr 1.1 \ft \f* +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Empty Note \f \fr 1.1 \ft Update the note \f* +"; + Assess(target, result); + } + + [Test] + public void CrossReferenceDontUpdate() + { + var rows = new List<(IReadOnlyList, string)> + { + (ScrRef("MAT 1:1/1:x"), "Update the cross reference"), + }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 Cross reference verse \x - \xo 2:3-4 \xt Cool Book 3:24 \xta The annotation \x* and more content. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Cross reference verse \x - \xo 2:3-4 \xt Cool Book 3:24 \xta The annotation \x* and more content. +"; + Assess(target, result); + } + + [Test] + public void PreserveFigAndFm() + { + var rows = new List<(IReadOnlyList, string)> { (ScrRef("MAT 1:1"), "Update"), }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 initial text \fig stuff\fig* more text \fm * \fm* and more. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Update \fig stuff\fig*\fm * \fm* +"; + Assess(target, result); + } + + [Test] + public void NestedXt() + { + var rows = new List<(IReadOnlyList, string)> + { + (ScrRef("MAT 1:1"), "Update text"), + (ScrRef("MAT 1:1/1:f"), "Update note"), + }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 initial text \f + \fr 15.8 \ft Text (\+xt reference\+xt*). And more.\f* and the end. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Update text \f + \fr 15.8 \ft Update note \+xt reference\+xt*\f* +"; + Assess(target, result); + + target = UpdateUsfm(rows, usfm, embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); + var result2 = + @"\id MAT - Test +\c 1 +\v 1 Update text +"; + Assess(target, result2); + } + + [Test] + public void NonNestedXt() + { + var rows = new List<(IReadOnlyList, string)> + { + (ScrRef("MAT 1:1"), "Update text"), + (ScrRef("MAT 1:1/1:f"), "Update note"), + }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 initial text \f + \fr 15.8 \ft Text \xt reference\f* and the end. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Update text \f + \fr 15.8 \ft Update note \xt reference\f* +"; + Assess(target, result); + + target = UpdateUsfm(rows, usfm, embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); + var result2 = + @"\id MAT - Test +\c 1 +\v 1 Update text +"; + Assess(target, result2); + } + + [Test] + public void MultipleFtOnlyUpdateFirst() + { + var rows = new List<(IReadOnlyList, string)> + { + (ScrRef("MAT 1:1"), "Update text"), + (ScrRef("MAT 1:1/1:f"), "Update note"), + }; + var usfm = + @"\id MAT - Test +\c 1 +\v 1 initial text \f + \fr 15.8 \ft first note \ft second note\f* and the end. +"; + var target = UpdateUsfm(rows, usfm); + var result = + @"\id MAT - Test +\c 1 +\v 1 Update text \f + \fr 15.8 \ft Update note \ft second note\f* +"; + Assess(target, result); + + target = UpdateUsfm(rows, usfm, embedBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); + var result2 = + @"\id MAT - Test +\c 1 +\v 1 Update text +"; + Assess(target, result2); } private static ScriptureRef[] ScrRef(params string[] refs) @@ -543,4 +722,15 @@ private static string UpdateUsfm( return updater.GetUsfm(); } } + + private static void Assess(string target, string truth) + { + Assert.That(target, Is.Not.Null); + var target_lines = target.Split(new[] { "\r\n" }, StringSplitOptions.None); + var truth_lines = truth.Split(new[] { "\n" }, StringSplitOptions.None); + for (int i = 0; i < truth_lines.Length; i++) + { + Assert.That(target_lines[i], Is.EqualTo(truth_lines[i])); + } + } } diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmFileTextTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmFileTextTests.cs index a6998201..40391b66 100644 --- a/tests/SIL.Machine.Tests/Corpora/UsfmFileTextTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UsfmFileTextTests.cs @@ -73,7 +73,7 @@ public void GetRows_NonEmptyText_AllText() IText text = corpus["MAT"]; TextRow[] rows = text.GetRows().ToArray(); - Assert.That(rows, Has.Length.EqualTo(51)); + Assert.That(rows, Has.Length.EqualTo(50)); Assert.That(rows[0].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:0/1:h", corpus.Versification))); Assert.That(rows[0].Text, Is.EqualTo("Matthew")); @@ -85,7 +85,7 @@ public void GetRows_NonEmptyText_AllText() Assert.That(rows[2].Text, Is.EqualTo("An introduction to Matthew with an empty comment")); Assert.That(rows[3].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:0/3:ip/1:fe", corpus.Versification))); - Assert.That(rows[3].Text, Is.EqualTo("")); + Assert.That(rows[3].Text, Is.EqualTo("This is an endnote.")); Assert.That(rows[4].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:0/4:p", corpus.Versification))); Assert.That(rows[4].Text, Is.EqualTo("MAT 1 Here is another paragraph.")); @@ -185,7 +185,7 @@ public void GetRows_IncludeMarkers() Assert.That( rows[0].Text, Is.EqualTo( - "Chapter \\pn one\\+pro WON\\+pro*\\pn*, verse \\f + \\fr 1:1: \\ft This is a footnote for v1.\\f*one." + "Chapter \\pn one\\+pro WON\\+pro*\\pn*, verse one.\\f + \\fr 1:1: \\ft This is a footnote for v1.\\f*" ) ); @@ -206,7 +206,7 @@ public void GetRows_IncludeMarkers() Assert.That(rows[8].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:1", corpus.Versification))); Assert.That( rows[8].Text, - Is.EqualTo("Chapter \\add two\\add*, verse \\f + \\fr 2:1: \\ft This is a \\bd footnote.\\bd*\\f*one.") + Is.EqualTo("Chapter \\add two\\add*, verse \\f + \\fr 2:1: \\ft This is a footnote.\\f*one.") ); Assert.That(rows[9].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:2", corpus.Versification))); @@ -231,7 +231,7 @@ public void GetRows_IncludeMarkers() Assert.That(rows[12].Text, Is.EqualTo("Chapter two, verse four.")); Assert.That(rows[13].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:5", corpus.Versification))); - Assert.That(rows[13].Text, Is.EqualTo("Chapter two, verse five\\rq (MAT 3:1)\\rq*.")); + Assert.That(rows[13].Text, Is.EqualTo("Chapter two, verse five \\rq (MAT 3:1)\\rq*.")); Assert.That(rows[14].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:6", corpus.Versification))); Assert.That(rows[14].Text, Is.EqualTo("Chapter two, verse \\w six|strong=\"12345\" \\w*.")); @@ -259,13 +259,13 @@ public void GetRows_IncludeMarkers_AllText() Assert.That(rows, Has.Length.EqualTo(47)); Assert.That(rows[2].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:0/3:ip", corpus.Versification))); - Assert.That(rows[2].Text, Is.EqualTo("An introduction to Matthew with an empty comment\\fe + \\ft \\fe*")); + Assert.That(rows[2].Text, Is.EqualTo("An introduction to Matthew\\fe + \\ft This is an endnote.\\fe*")); Assert.That(rows[8].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:1", corpus.Versification))); Assert.That( rows[8].Text, Is.EqualTo( - "Chapter \\pn one\\+pro WON\\+pro*\\pn*, verse \\f + \\fr 1:1: \\ft This is a footnote for v1.\\f*one." + "Chapter \\pn one\\+pro WON\\+pro*\\pn*, verse one.\\f + \\fr 1:1: \\ft This is a footnote for v1.\\f*" ) ); @@ -289,7 +289,7 @@ public void GetRows_IncludeMarkers_AllText() Assert.That(rows[22].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:1", corpus.Versification))); Assert.That( rows[22].Text, - Is.EqualTo("Chapter \\add two\\add*, verse \\f + \\fr 2:1: \\ft This is a \\bd footnote.\\bd*\\f*one.") + Is.EqualTo("Chapter \\add two\\add*, verse \\f + \\fr 2:1: \\ft This is a footnote.\\f*one.") ); Assert.That(rows[26].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 2:3/2:esb/2:p", corpus.Versification))); diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs index cad05380..7138eda2 100644 --- a/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs @@ -11,7 +11,7 @@ public void Tokenize() string usfm = ReadUsfm(); var tokenizer = new UsfmTokenizer(); IReadOnlyList tokens = tokenizer.Tokenize(usfm); - Assert.That(tokens, Has.Count.EqualTo(236)); + Assert.That(tokens, Has.Count.EqualTo(230)); Assert.That(tokens[0].Type, Is.EqualTo(UsfmTokenType.Book)); Assert.That(tokens[0].Marker, Is.EqualTo("id")); @@ -19,22 +19,22 @@ public void Tokenize() Assert.That(tokens[0].LineNumber, Is.EqualTo(1)); Assert.That(tokens[0].ColumnNumber, Is.EqualTo(1)); - Assert.That(tokens[30].Type, Is.EqualTo(UsfmTokenType.Text)); - Assert.That(tokens[30].Text, Is.EqualTo("Chapter One ")); - Assert.That(tokens[30].LineNumber, Is.EqualTo(9)); - Assert.That(tokens[30].ColumnNumber, Is.EqualTo(4)); + Assert.That(tokens[31].Type, Is.EqualTo(UsfmTokenType.Text)); + Assert.That(tokens[31].Text, Is.EqualTo("Chapter One ")); + Assert.That(tokens[31].LineNumber, Is.EqualTo(9)); + Assert.That(tokens[31].ColumnNumber, Is.EqualTo(4)); - Assert.That(tokens[31].Type, Is.EqualTo(UsfmTokenType.Verse)); - Assert.That(tokens[31].Marker, Is.EqualTo("v")); - Assert.That(tokens[31].Data, Is.EqualTo("1")); - Assert.That(tokens[31].LineNumber, Is.EqualTo(10)); - Assert.That(tokens[31].ColumnNumber, Is.EqualTo(1)); + Assert.That(tokens[32].Type, Is.EqualTo(UsfmTokenType.Verse)); + Assert.That(tokens[32].Marker, Is.EqualTo("v")); + Assert.That(tokens[32].Data, Is.EqualTo("1")); + Assert.That(tokens[32].LineNumber, Is.EqualTo(10)); + Assert.That(tokens[32].ColumnNumber, Is.EqualTo(1)); - Assert.That(tokens[40].Type, Is.EqualTo(UsfmTokenType.Note)); - Assert.That(tokens[40].Marker, Is.EqualTo("f")); - Assert.That(tokens[40].Data, Is.EqualTo("+")); - Assert.That(tokens[40].LineNumber, Is.EqualTo(10)); - Assert.That(tokens[40].ColumnNumber, Is.EqualTo(48)); + Assert.That(tokens[41].Type, Is.EqualTo(UsfmTokenType.Note)); + Assert.That(tokens[41].Marker, Is.EqualTo("f")); + Assert.That(tokens[41].Data, Is.EqualTo("+")); + Assert.That(tokens[41].LineNumber, Is.EqualTo(10)); + Assert.That(tokens[41].ColumnNumber, Is.EqualTo(52)); } [Test]