Skip to content

Commit

Permalink
Fixed tab expansion around block markers. Fixes #80
Browse files Browse the repository at this point in the history
  • Loading branch information
@Knagis
Knagis committed Apr 20, 2016
1 parent df75c9e commit 6dae3ab
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 45 deletions.
7 changes: 7 additions & 0 deletions CommonMark.Tests/ListTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ public void Example210WithPositionTracking()
", s);
}

[TestMethod]
[TestCategory("Container blocks - List items")]
public void ListWithTabs()
{
Helpers.ExecuteTest("*\tbar", "<ul>\n<li>bar</li>\n</ul>");
}

[TestMethod]
[TestCategory("Container blocks - List items")]
public void UnicodeBulletEscape()
Expand Down
150 changes: 105 additions & 45 deletions CommonMark/Parser/BlockMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ internal static class BlockMethods
{
private const int CODE_INDENT = 4;
private const int TabSize = 4;
private const string Spaces = " ";

#if OptimizeFor45
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
Expand All @@ -31,7 +32,7 @@ private static bool AcceptsLines(BlockTag block_type)
block_type == BlockTag.FencedCode);
}

private static void AddLine(Block block, LineInfo lineInfo, string ln, int offset, int length = -1)
private static void AddLine(Block block, LineInfo lineInfo, string ln, int offset, int remainingSpaces, int length = -1)
{
if (!block.IsOpen)
throw new CommonMarkException(string.Format(CultureInfo.InvariantCulture, "Attempted to add line '{0}' to closed container ({1}).", ln, block.Tag));
Expand All @@ -51,6 +52,7 @@ private static void AddLine(Block block, LineInfo lineInfo, string ln, int offse
if (lineInfo.IsTrackingPositions)
curSC.PositionTracker.AddOffset(lineInfo, offset, len);

curSC.Append(Spaces, 0, remainingSpaces);
curSC.Append(ln, offset, len);
}

Expand Down Expand Up @@ -347,7 +349,7 @@ private static int ParseListMarker(string ln, int pos, out ListData data)
if (c == '+' || c == '•' || ((c == '*' || c == '-') && 0 == Scanner.scan_thematic_break(ln, pos, len)))
{
pos++;
if (pos == len || (ln[pos] != ' ' && ln[pos] != '\n'))
if (pos == len || !Utilities.IsWhitespace(ln[pos]))
return 0;

data = new ListData();
Expand All @@ -373,7 +375,7 @@ private static int ParseListMarker(string ln, int pos, out ListData data)
return 0;

pos++;
if (pos == len || (ln[pos] != ' ' && ln[pos] != '\n'))
if (pos == len || !Utilities.IsWhitespace(ln[pos]))
return 0;

data = new ListData();
Expand All @@ -399,8 +401,53 @@ private static bool ListsMatch(ListData listData, ListData itemData)
listData.BulletChar == itemData.BulletChar);
}

private static void AdvanceOffset(string line, int count, bool columns, ref int offset, ref int column)
private static bool AdvanceOptionalSpace(string line, ref int offset, ref int column, ref int remainingSpaces)
{
if (remainingSpaces > 0)
{
remainingSpaces--;
return true;
}

var c = line[offset];
if (c == ' ')
{
offset++;
column++;
return true;
}
else if (c == '\t')
{
offset++;
var chars_to_tab = 4 - (column % TabSize);
column += chars_to_tab;
remainingSpaces = chars_to_tab - 1;
return true;
}

return false;
}

private static void AdvanceOffset(string line, int count, bool columns, ref int offset, ref int column, ref int remainingSpaces)
{
if (columns)
{
if (remainingSpaces > count)
{
remainingSpaces -= count;
count = 0;
}
else
{
count -= remainingSpaces;
remainingSpaces = 0;
}
}
else
{
remainingSpaces = 0;
}

char c;
while (count > 0 && (c = line[offset]) != '\n')
{
Expand All @@ -410,6 +457,11 @@ private static void AdvanceOffset(string line, int count, bool columns, ref int
column += chars_to_tab;
offset += 1;
count -= columns ? chars_to_tab : 1;

if (count < 0)
{
remainingSpaces = 0 - count;
}
}
else
{
Expand All @@ -435,6 +487,9 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
// column is the virtual position in the line that takes TAB expansion into account
var column = 0;

// the adjustment to the virtual position `column` that points to the number of spaces from the TAB that have not been included in any indent.
var remainingSpaces = 0;

// the char position of the first non-space char
int first_nonspace;

Expand Down Expand Up @@ -462,7 +517,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)

FindFirstNonspace(ln, offset, column, out first_nonspace, out first_nonspace_column, out curChar);

indent = first_nonspace_column - column;
indent = first_nonspace_column - column + remainingSpaces;
blank = curChar == '\n';

switch (container.Tag)
Expand All @@ -471,9 +526,8 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
{
if (indent <= 3 && curChar == '>')
{
AdvanceOffset(ln, indent + 1, true, ref offset, ref column);
if (ln[offset] == ' ')
offset++;
AdvanceOffset(ln, indent + 1, true, ref offset, ref column, ref remainingSpaces);
AdvanceOptionalSpace(ln, ref offset, ref column, ref remainingSpaces);
}
else
{
Expand All @@ -487,14 +541,14 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
{
if (indent >= container.ListData.MarkerOffset + container.ListData.Padding)
{
AdvanceOffset(ln, container.ListData.MarkerOffset + container.ListData.Padding, true, ref offset, ref column);
AdvanceOffset(ln, container.ListData.MarkerOffset + container.ListData.Padding, true, ref offset, ref column, ref remainingSpaces);
}
else if (blank && container.FirstChild != null)
{
// if container->first_child is NULL, then the opening line
// of the list item was blank after the list marker; in this
// case, we are done with the list item.
AdvanceOffset(ln, first_nonspace - offset, false, ref offset, ref column);
AdvanceOffset(ln, first_nonspace - offset, false, ref offset, ref column, ref remainingSpaces);
}
else
{
Expand All @@ -507,9 +561,9 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
case BlockTag.IndentedCode:
{
if (indent >= CODE_INDENT)
AdvanceOffset(ln, CODE_INDENT, true, ref offset, ref column);
AdvanceOffset(ln, CODE_INDENT, true, ref offset, ref column, ref remainingSpaces);
else if (blank)
AdvanceOffset(ln, first_nonspace - offset, false, ref offset, ref column);
AdvanceOffset(ln, first_nonspace - offset, false, ref offset, ref column, ref remainingSpaces);
else
all_matched = false;

Expand Down Expand Up @@ -606,21 +660,16 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
if (!indented && curChar == '>')
{

AdvanceOffset(ln, first_nonspace + 1 - offset, false, ref offset, ref column);
// optional following character
if (ln[offset] == ' ')
{
offset++;
column++;
}
AdvanceOffset(ln, first_nonspace + 1 - offset, false, ref offset, ref column, ref remainingSpaces);
AdvanceOptionalSpace(ln, ref offset, ref column, ref remainingSpaces);

container = CreateChildBlock(container, line, BlockTag.BlockQuote, first_nonspace);

}
else if (!indented && curChar == '#' && 0 != (matched = Scanner.scan_atx_heading_start(ln, first_nonspace, ln.Length, out i)))
{

AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column);
AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column, ref remainingSpaces);
container = CreateChildBlock(container, line, BlockTag.AtxHeading, first_nonspace);
container.Heading = new HeadingData(i);

Expand All @@ -634,7 +683,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
container.FencedCodeData.FenceLength = matched;
container.FencedCodeData.FenceOffset = first_nonspace - offset;

AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column);
AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column, ref remainingSpaces);

}
else if (!indented && curChar == '<' &&
Expand All @@ -654,7 +703,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)

container.Tag = BlockTag.SetextHeading;
container.Heading = new HeadingData(matched);
AdvanceOffset(ln, ln.Length - 1 - offset, false, ref offset, ref column);
AdvanceOffset(ln, ln.Length - 1 - offset, false, ref offset, ref column, ref remainingSpaces);

}
else if (!indented
Expand All @@ -666,33 +715,45 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
container = CreateChildBlock(container, line, BlockTag.ThematicBreak, first_nonspace);
Finalize(container, line);
container = container.Parent;
AdvanceOffset(ln, ln.Length - 1 - offset, false, ref offset, ref column);
AdvanceOffset(ln, ln.Length - 1 - offset, false, ref offset, ref column, ref remainingSpaces);

}
else if ((!indented || container.Tag == BlockTag.List)
&& 0 != (matched = ParseListMarker(ln, first_nonspace, out data)))
{

// compute padding:
AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column);
i = 0;
while (i <= 5 && ln[offset + i] == ' ')
i++;
AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column, ref remainingSpaces);

var prevOffset = offset;
var prevColumn = column;
var prevRemainingSpaces = remainingSpaces;

while (column - prevColumn <= CODE_INDENT)
{
if (!AdvanceOptionalSpace(ln, ref offset, ref column, ref remainingSpaces))
break;
}

// i = number of spaces after marker, up to 5
if (i >= 5 || i < 1 || ln[offset] == '\n')
if (column == prevColumn)
{
// no spaces at all
data.Padding = matched + 1;
if (i > 0)
{
column++;
offset++;
}
}
else if (column - prevColumn > CODE_INDENT || ln[offset] == '\n')
{
data.Padding = matched + 1;

// too many (or none) spaces, ignoring everything but the first one
offset = prevOffset;
column = prevColumn;
remainingSpaces = prevRemainingSpaces;
AdvanceOptionalSpace(ln, ref offset, ref column, ref remainingSpaces);
}
else
{
data.Padding = matched + i;
AdvanceOffset(ln, i, true, ref offset, ref column);
data.Padding = matched + column - prevColumn;
}

// check container; if it's a list, see if this list item
Expand All @@ -712,7 +773,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
}
else if (indented && !maybeLazy && !blank)
{
AdvanceOffset(ln, CODE_INDENT, true, ref offset, ref column);
AdvanceOffset(ln, CODE_INDENT, true, ref offset, ref column, ref remainingSpaces);
container = CreateChildBlock(container, line, BlockTag.IndentedCode, offset);
}
else
Expand Down Expand Up @@ -767,7 +828,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
cur.StringContent.Length > 0)
{

AddLine(cur, line, ln, offset);
AddLine(cur, line, ln, offset, remainingSpaces);

}
else
Expand All @@ -787,8 +848,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)

if (container.Tag == BlockTag.IndentedCode)
{

AddLine(container, line, ln, offset);
AddLine(container, line, ln, offset, remainingSpaces);

}
else if (container.Tag == BlockTag.FencedCode)
Expand All @@ -803,14 +863,14 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
}
else
{
AddLine(container, line, ln, offset);
AddLine(container, line, ln, offset, remainingSpaces);
}

}
else if (container.Tag == BlockTag.HtmlBlock)
{

AddLine(container, line, ln, offset);
AddLine(container, line, ln, offset, remainingSpaces);

if (Scanner.scan_html_block_end(container.HtmlBlockType, ln, first_nonspace, ln.Length))
{
Expand Down Expand Up @@ -842,23 +902,23 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
if (p < 0 || ln[p] != ' ')
p = ln.Length - 1;

AddLine(container, line, ln, first_nonspace, p - first_nonspace + 1);
AddLine(container, line, ln, first_nonspace, remainingSpaces, p - first_nonspace + 1);
Finalize(container, line);
container = container.Parent;

}
else if (AcceptsLines(container.Tag))
{

AddLine(container, line, ln, first_nonspace);
AddLine(container, line, ln, first_nonspace, remainingSpaces);

}
else if (container.Tag != BlockTag.ThematicBreak && container.Tag != BlockTag.SetextHeading)
{

// create paragraph container for line
container = CreateChildBlock(container, line, BlockTag.Paragraph, first_nonspace);
AddLine(container, line, ln, first_nonspace);
AddLine(container, line, ln, first_nonspace, remainingSpaces);

}
else
Expand All @@ -875,7 +935,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
private static void FindFirstNonspace(string ln, int offset, int column, out int first_nonspace,
out int first_nonspace_column, out char curChar)
{
var chars_to_tab = TabSize - (column%TabSize);
var chars_to_tab = TabSize - (column % TabSize);
first_nonspace = offset;
first_nonspace_column = column;
while ((curChar = ln[first_nonspace]) != '\n')
Expand Down

0 comments on commit 6dae3ab

Please sign in to comment.