Merge pull request #197 from creachadair/muckthebits

Prevent tokenization from modifying its input.

internal/tokenizer/tokenize.go

@@ -13,6 +13,10 @@ func Tokenize(content []byte) []string {
 		content = content[:byteLimit]
 	}
 
+	// Copy the input so that changes wrought by the tokenization steps do not
+	// modify the caller's copy of the input. See #196.
+	content = append([]byte(nil), content...)
+
 	tokens := make([][]byte, 0, 50)
 	for _, extract := range extractTokens {
 		var extractedTokens [][]byte

internal/tokenizer/tokenize_test.go

@@ -102,7 +102,10 @@ func TestTokenize(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			before := string(test.content)
 			tokens := Tokenize(test.content)
+			after := string(test.content)
+			assert.Equal(t, before, after, "the input slice was modified")
 			assert.Equal(t, len(test.expected), len(tokens), fmt.Sprintf("token' slice length = %v, want %v", len(test.expected), len(tokens)))
 			for i, expectedToken := range test.expected {
 				assert.Equal(t, expectedToken, tokens[i], fmt.Sprintf("token = %v, want %v", tokens[i], expectedToken))