diff --git a/.eslintignore b/.eslintignore index 134f308..853a55b 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,7 +1,4 @@ **/coverage/ **/dist/ -**/test/specs/expect/ -**/test/specs/fixtures/ -**/test/specs/parsers/js/ -**/test/v223/ -**/test/perf.js +**/test/ +!/test/runner.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6343a..488be15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Compiler Changes +### v3.2.3 +- Fixes various issues with literal regexes. + ### v3.1.4 - Fix avoid the `filename` option for the babel-standalone parser diff --git a/Makefile b/Makefile index facb507..8398dda 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ test: build test-mocha build: clean eslint pre-build # build riot and es6 versions + @ mkdir -p $(DIST) @ $(JSPP) $(JSPP_RIOT_FLAGS) src/_riot.js > $(DIST)riot.compiler.js @ $(JSPP) $(JSPP_ES6_FLAGS) src/_es6.js > $(DIST)es6.compiler.js @@ -32,7 +33,6 @@ clean: pre-build: # build the node version - @ mkdir -p $(DIST) @ $(JSPP) $(JSPP_NODE_FLAGS) src/core.js > $(LIB)compiler.js @ $(JSPP) $(JSPP_NODE_FLAGS) src/safe-regex.js > $(LIB)safe-regex.js diff --git a/lib/brackets.js b/lib/brackets.js index a21f085..d261cfa 100644 --- a/lib/brackets.js +++ b/lib/brackets.js @@ -4,7 +4,9 @@ * Brackets support for the node.js version of the riot-compiler * @module */ -var safeRegex = require('./safe-regex.js') + +var safeRegex = require('./safe-regex') +var skipRegex = require('./skip-regex') /** * Matches valid, multiline JavaScript comments in almost all its forms. @@ -34,6 +36,11 @@ var S_QBLOCKS = R_STRINGS.source + '|' + /(?:\breturn\s+|(?:[$\w\)\]]|\+\+|--)\s*(\/)(?![*\/]))/.source + '|' + /\/(?=[^*\/])[^[\/\\]*(?:(?:\[(?:\\.|[^\]\\]*)*\]|\\.)[^[\/\\]*)*?([^<]\/)[gim]*/.source +/* + JS/ES6 quoted strings and start of regex (basic ES6 does not supports nested backquotes). +*/ +var S_QBLOCK2 = R_STRINGS.source + '|' + /(\/)(?![*\/])/.source + /** * Hash of regexes for matching JavaScript brackets out of quoted strings and literal * regexes. Used by {@link module:brackets.split|split}, these are heavy, but their @@ -41,9 +48,9 @@ var S_QBLOCKS = R_STRINGS.source + '|' + * @const {object} */ var FINDBRACES = { - '(': RegExp('([()])|' + S_QBLOCKS, 'g'), - '[': RegExp('([[\\]])|' + S_QBLOCKS, 'g'), - '{': RegExp('([{}])|' + S_QBLOCKS, 'g') + '(': RegExp('([()])|' + S_QBLOCK2, 'g'), + '[': RegExp('([[\\]])|' + S_QBLOCK2, 'g'), + '{': RegExp('([{}])|' + S_QBLOCK2, 'g') } /** @@ -92,7 +99,9 @@ function _rewrite (re) { module.exports = { R_STRINGS: R_STRINGS, R_MLCOMMS: R_MLCOMMS, - S_QBLOCKS: S_QBLOCKS + S_QBLOCKS: S_QBLOCKS, + S_QBLOCK2: S_QBLOCK2, + skipRegex: skipRegex } /** @@ -140,13 +149,18 @@ module.exports.split = function split (str, _, _bp) { $1: optional escape character, $2: opening js bracket `{[(`, $3: closing riot bracket, - $4 & $5: qblocks + $4: opening slashes of regex */ if (match[2]) { // if have a javascript opening bracket, re.lastIndex = skipBraces(str, match[2], re.lastIndex) continue // skip the bracketed block and loop } + if (!match[3]) { // if don't have a closing bracket + // look here if this "regex" is a real regex (riot#2361) + if (match[4]) { + re.lastIndex = skipRegex(str, match.index) + } continue // search again } } @@ -206,8 +220,12 @@ module.exports.split = function split (str, _, _bp) { rr.lastIndex = ix ix = 1 while ((mm = rr.exec(s))) { - if (mm[1] && - !(mm[1] === ch ? ++ix : --ix)) break + if (mm[1]) { + if (mm[1] === ch) ++ix + else if (!--ix) break + } else if (mm[2]) { + rr.lastIndex = skipRegex(str, mm.index) + } } if (ix) { diff --git a/lib/compiler.js b/lib/compiler.js index aa8b591..9615ab0 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -1,8 +1,8 @@ /** - * The riot-compiler WIP + * The riot-compiler v3.2.3 * * @module compiler - * @version WIP + * @version v3.2.3 * @license MIT * @copyright Muut Inc. + contributors */ @@ -354,7 +354,7 @@ var JS_ES6END = RegExp('[{}]|' + brackets.S_QBLOCKS, 'g') * {@link module:brackets.S_QBLOCKS|brackets.S_QBLOCKS} to skip literal string and regexes. * @const {RegExp} */ -var JS_COMMS = RegExp(brackets.R_MLCOMMS.source + '|//[^\r\n]*|' + brackets.S_QBLOCKS, 'g') +var JS_COMMS = RegExp(brackets.R_MLCOMMS.source + '|//[^\r\n]*|' + brackets.S_QBLOCK2, 'g') /** * Default parser for JavaScript, supports ES6-like method syntax @@ -404,9 +404,11 @@ function riotjs (js) { function rmComms (s, r, m) { r.lastIndex = 0 while ((m = r.exec(s))) { - if (m[0][0] === '/' && !m[1] && !m[2]) { - s = RE.leftContext + ' ' + RE.rightContext - r.lastIndex = m[3] + 1 + if (m[1]) { + r.lastIndex = brackets.skipRegex(s, m.index) + } else if (m[0][0] === '/') { + s = s.slice(0, m.index) + ' ' + s.slice(r.lastIndex) + r.lastIndex = m.index + 1 } } return s @@ -1008,5 +1010,5 @@ module.exports = { css: compileCSS, js: compileJS, parsers: parsers, - version: 'WIP' + version: 'v3.2.3' } diff --git a/lib/skip-regex.js b/lib/skip-regex.js new file mode 100644 index 0000000..5a642c3 --- /dev/null +++ b/lib/skip-regex.js @@ -0,0 +1,109 @@ +/* + Regex detection. + From: https://github.com/riot/parser/blob/master/src/skip-regex.js +*/ +//#if NODE +'use strict' +//#endif +//#if 0 +/* eslint no-unused-vars: [2, {args: "after-used", varsIgnorePattern: "^skipRegex"}] */ +//#endif + +var skipRegex = (function () { + + // safe characters to precced a regex (including `=>`, `**`, and `...`) + var beforeReChars = '[{(,;:?=|&!^~>%*/' + + // keyword that can preceed a regex (`in` is handled as special case) + var beforeReWords = [ + 'case', + 'default', + 'do', + 'else', + 'in', + 'instanceof', + 'prefix', + 'return', + 'typeof', + 'void', + 'yield' + ] + + var wordsLastChar = beforeReWords.reduce(function (s, w) { + return s + w.slice(-1) + }, '') + + // The string to test can't include line-endings + var RE_REGEX = /^\/(?=[^*>/])[^[/\\]*(?:(?:\\.|\[(?:\\.|[^\]\\]*)*\])[^[\\/]*)*?\/[gimuy]*/ + var RE_VN_CHAR = /[$\w]/ + + // Searches the position of the previous non-blank character inside `code`, + // starting with `pos - 1`. + function prev (code, pos) { + while (--pos >= 0 && /\s/.test(code[pos])); + return pos + } + + /** + * Check if the code in the `start` position can be a regex. + * + * @param {string} code - Buffer to test in + * @param {number} start - Position the first slash inside `code` + * @returns {number} position of the char following the regex. + */ + function _skipRegex (code, start) { + + // `exec()` will extract from the slash to the end of line and the + // chained `match()` will match the possible regex. + var re = /.*/g + var pos = re.lastIndex = start++ + var match = re.exec(code)[0].match(RE_REGEX) + + if (match) { + var next = pos + match[0].length // result is not from `re.exec` + + pos = prev(code, pos) + var c = code[pos] + + // start of buffer or safe prefix? + if (pos < 0 || ~beforeReChars.indexOf(c)) { + return next + } + + // from here, `pos` is >= 0 and `c` is code[pos] + // is-tanbul ignore next: This is for ES6 + if (c === '.') { + // can be `...` or something silly like 5./2 + if (code[pos - 1] === '.') { + start = next + } + + } else if (c === '+' || c === '-') { + // tricky case + if (code[--pos] !== c || // if have a single operator or + (pos = prev(code, pos)) < 0 || // ...have `++` and no previous token or + !RE_VN_CHAR.test(code[pos])) { // ...the token is not a JS var/number + start = next // ...this is a regex + } + + } else if (~wordsLastChar.indexOf(c)) { + // keyword? + var end = pos + 1 + + while (--pos >= 0 && RE_VN_CHAR.test(code[pos])); + if (~beforeReWords.indexOf(code.slice(pos + 1, end))) { + start = next + } + } + } + + return start + } + + return _skipRegex + +})() + +//#if NODE +module.exports = skipRegex +//#endif diff --git a/package.json b/package.json index 0ace16f..4fb0f7d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "riot-compiler", - "version": "3.2.2", + "version": "3.2.3", "description": "Compiler for riot .tag files", "main": "lib/compiler.js", "jsnext:main": "dist/es6.compiler.js", @@ -32,12 +32,12 @@ "compiler" ], "devDependencies": { - "coveralls": "^2.11.16", - "eslint": "^3.15.0", + "coveralls": "^2.13.1", + "eslint": "^3.19.0", "expect.js": "^0.3.1", "istanbul": "^0.4.5", "jspreproc": "^0.2.7", - "mocha": "^3.2.0", + "mocha": "^3.4.2", "riot-bump": "^1.0.0" }, "author": "Riot maintainers team + smart people from all over the world", diff --git a/src/core.js b/src/core.js index 5cd63d3..578f38d 100644 --- a/src/core.js +++ b/src/core.js @@ -385,7 +385,7 @@ var JS_ES6END = RegExp('[{}]|' + brackets.S_QBLOCKS, 'g') * {@link module:brackets.S_QBLOCKS|brackets.S_QBLOCKS} to skip literal string and regexes. * @const {RegExp} */ -var JS_COMMS = RegExp(brackets.R_MLCOMMS.source + '|//[^\r\n]*|' + brackets.S_QBLOCKS, 'g') +var JS_COMMS = RegExp(brackets.R_MLCOMMS.source + '|//[^\r\n]*|' + brackets.S_QBLOCK2, 'g') /** * Default parser for JavaScript, supports ES6-like method syntax @@ -436,12 +436,15 @@ function riotjs (js) { return parts.length ? parts.join('') + js : js // 2016-01-18: remove comments without touching qblocks (avoid reallocation) + // 2017-96-03: Fixes riot#2361 & riot#2369 using skipRegex from riot/parser function rmComms (s, r, m) { r.lastIndex = 0 while ((m = r.exec(s))) { - if (m[0][0] === '/' && !m[1] && !m[2]) { // $1:div, $2:regex - s = RE.leftContext + ' ' + RE.rightContext - r.lastIndex = m[3] + 1 // $3:matchOffset + if (m[1]) { + r.lastIndex = brackets.skipRegex(s, m.index) + } else if (m[0][0] === '/') { + s = s.slice(0, m.index) + ' ' + s.slice(r.lastIndex) + r.lastIndex = m.index + 1 } } return s diff --git a/test/specs/expect/issue_2361.js b/test/specs/expect/issue_2361.js new file mode 100644 index 0000000..65d3d06 --- /dev/null +++ b/test/specs/expect/issue_2361.js @@ -0,0 +1,2 @@ +riot.tag2('my-tag', '