Skip to content

Commit

Permalink
Improve Ruby some more
Browse files Browse the repository at this point in the history
  • Loading branch information
@jart
jart committed Nov 9, 2024
1 parent 5b0fff1 commit 33a057e
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 32 deletions.
124 changes: 96 additions & 28 deletions llamafile/highlight_ruby.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,6 @@
#include <cosmo.h>
#include <ctype.h>

// ruby lexical syntax is bananas and isn't even formally documented
//
// known issues
//
// 1. we can't tell this backtick apart from command substitution
//
// def `(command)
// return "just testing a backquote override"
// end
//
// 2. we don't know <<arg isn't a heredoc. emacs doesn't get this right
// either, but somehow vim does.
//
// when /\.*\.h/
// options[:includes] <<arg; true
// when /--(\w+)=\"?(.*)\"?/
// options[$1.to_sym] = $2; true
//

enum {
NORMAL,
WORD,
Expand Down Expand Up @@ -69,6 +50,9 @@ enum {
PERCENT,
PERCENT2,
PERCENT_STRING,
PERCENT_HASH,
PERCENT_HASH_DOLLAR,
PERCENT_HASH_DOLLAR_WORD,
MULTICOM,
MULTICOM_BOL,
REGEX,
Expand Down Expand Up @@ -155,6 +139,23 @@ static bool is_dollar_one(int c) {
}
}

static bool is_percent_literal(int c) {
switch (c) {
case 'q':
case 'Q':
case 'r':
case 's':
case 'w':
case 'W':
case 'x':
case 'i':
case 'I':
return true;
default:
return false;
}
}

static bool isident(int c) {
return !isascii(c) || //
isalnum(c) || //
Expand Down Expand Up @@ -251,6 +252,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
lf::append_wchar(r, c);
expect_ = EXPECT_OPERATOR;
} else if (c == '#') {
expect_ = EXPECT_VALUE;
*r += HI_COMMENT;
lf::append_wchar(r, c);
t_ = COMMENT;
Expand Down Expand Up @@ -281,9 +283,9 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
t_ = HEREDOC_BOL;
i_ = 0;
}
} else if (c == ']') {
} else if (c == ']' || c == ')') {
expect_ = EXPECT_OPERATOR;
*r += ']';
lf::append_wchar(r, c);
is_definition_ = false;
} else if (ispunct(c)) {
expect_ = EXPECT_VALUE;
Expand Down Expand Up @@ -511,7 +513,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;

case PERCENT:
if (c == 'q' || c == 'Q') {
if (is_percent_literal(c)) {
q_ = c;
t_ = PERCENT2;
} else if (ispunct(c)) {
Expand Down Expand Up @@ -547,15 +549,72 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
}
break;

case PERCENT_HASH:
if (c == '{' && nesti_ < sizeof(nest_)) {
*r += HI_BOLD;
*r += '#';
*r += HI_UNBOLD;
*r += '{';
*r += HI_RESET;
expect_ = EXPECT_VALUE;
nest_[nesti_++] = PERCENT_STRING;
t_ = NORMAL;
} else if (c == '$') {
t_ = PERCENT_HASH_DOLLAR;
} else {
*r += '#';
t_ = PERCENT_STRING;
goto PercentString;
}
break;

case PERCENT_HASH_DOLLAR:
if (is_dollar_one(c)) {
*r += '#';
*r += HI_BOLD;
*r += '$';
lf::append_wchar(r, c);
*r += HI_UNBOLD;
t_ = PERCENT_STRING;
} else if (isalpha(c)) {
*r += '#';
*r += HI_BOLD;
*r += '$';
lf::append_wchar(r, c);
t_ = PERCENT_HASH_DOLLAR_WORD;
} else {
*r += '#';
*r += '$';
t_ = PERCENT_STRING;
goto PercentString;
}
break;

case PERCENT_HASH_DOLLAR_WORD:
if (isident(c)) {
lf::append_wchar(r, c);
} else {
*r += HI_UNBOLD;
t_ = PERCENT_STRING;
goto PercentString;
}
break;

PercentString:
case PERCENT_STRING:
lf::append_wchar(r, c);
if (c == opener_ && opener_ != closer_) {
lf::append_wchar(r, c);
++level_;
} else if (c == '#') {
t_ = PERCENT_HASH;
} else if (c == closer_) {
lf::append_wchar(r, c);
if (!--level_) {
*r += HI_RESET;
t_ = NORMAL;
}
} else {
lf::append_wchar(r, c);
}
break;

Expand Down Expand Up @@ -713,7 +772,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;

case LT_LT:
if (c == '-') {
if (c == '-' || c == '~') {
indented_heredoc_ = true;
lf::append_wchar(r, c);
} else if (c == '\'' || c == '`' || c == '"') {
Expand All @@ -723,7 +782,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
lf::append_wchar(r, c);
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
heredoc_ += c;
lf::append_wchar(&heredoc_, c);
lf::append_wchar(r, c);
} else {
t_ = NORMAL;
Expand All @@ -734,7 +793,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case LT_LT_NAME:
if (isalnum(c) || c == '_') {
t_ = LT_LT_NAME;
heredoc_ += c;
lf::append_wchar(&heredoc_, c);
lf::append_wchar(r, c);
} else if (c == '\n') {
lf::append_wchar(r, c);
Expand All @@ -755,7 +814,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
pending_heredoc_ = true;
t_ = NORMAL;
} else {
heredoc_ += c;
lf::append_wchar(&heredoc_, c);
}
break;

Expand All @@ -767,7 +826,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
*r += HI_RESET;
}
i_ = 0;
} else if (c == '\t' && indented_heredoc_) {
} else if (isblank(c) && indented_heredoc_) {
// do nothing
} else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
i_++;
Expand Down Expand Up @@ -859,6 +918,14 @@ void HighlightRuby::flush(std::string *r) {
*r += '%';
*r += q_;
break;
case PERCENT_HASH:
*r += '#';
*r += HI_RESET;
break;
case PERCENT_HASH_DOLLAR:
*r += "#$";
*r += HI_RESET;
break;
case QUESTION:
*r += '?';
break;
Expand All @@ -885,6 +952,7 @@ void HighlightRuby::flush(std::string *r) {
case REGEX_BACKSLASH:
case REGEX_HASH_DOLLAR_WORD:
case PERCENT_STRING:
case PERCENT_HASH_DOLLAR_WORD:
case AT_WORD:
case DOLLAR_WORD:
case TICK:
Expand Down
2 changes: 1 addition & 1 deletion llamafile/server/www/ctype.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ function isspace(c) {
function isblank(c) {
switch (c) {
case ' ':
case '\n':
case '\t':
return true;
default:
return false;
Expand Down
86 changes: 83 additions & 3 deletions llamafile/server/www/highlight_ruby.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,9 @@ class HighlightRuby extends Highlighter {
static DQUOTE_HASH = 36;
static DQUOTE_HASH_DOLLAR = 37;
static DQUOTE_HASH_DOLLAR_WORD = 38;
static PERCENT_HASH = 39;
static PERCENT_HASH_DOLLAR = 40;
static PERCENT_HASH_DOLLAR_WORD = 41;

static EXPECT_VALUE = 0;
static EXPECT_OPERATOR = 1;
Expand Down Expand Up @@ -245,6 +248,23 @@ class HighlightRuby extends Highlighter {
}
}

static is_percent_literal(c) {
switch (c) {
case 'q':
case 'Q':
case 'r':
case 's':
case 'w':
case 'W':
case 'x':
case 'i':
case 'I':
return true;
default:
return false;
}
}

static isident(c) {
return !isascii(c) ||
isalnum(c) ||
Expand Down Expand Up @@ -332,6 +352,7 @@ class HighlightRuby extends Highlighter {
this.push("span", "comment");
this.append(c);
this.state = HighlightRuby.COMMENT;
this.expect = HighlightRuby.EXPECT_VALUE;
} else if (c == '<' && this.expect == HighlightRuby.EXPECT_VALUE) {
this.append(c);
this.state = HighlightRuby.LT;
Expand Down Expand Up @@ -578,7 +599,7 @@ class HighlightRuby extends Highlighter {
break;

case HighlightRuby.PERCENT:
if (c == 'q' || c == 'Q') {
if (HighlightRuby.is_percent_literal(c)) {
this.q = c;
this.state = HighlightRuby.PERCENT2;
} else if (ispunct(c)) {
Expand Down Expand Up @@ -612,10 +633,60 @@ class HighlightRuby extends Highlighter {
}
break;

case HighlightRuby.PERCENT_HASH:
if (c == '{') {
this.push("strong", "");
this.append('#');
this.pop();
this.append('{');
this.pop();
this.expect = HighlightRuby.EXPECT_VALUE;
this.nest.push(HighlightRuby.PERCENT_STRING);
this.state = HighlightRuby.NORMAL;
} else if (c == '$') {
this.state = HighlightRuby.PERCENT_HASH_DOLLAR;
} else {
this.append('#');
this.epsilon(HighlightRuby.PERCENT_STRING);
}
break;

case PERCENT_HASH_DOLLAR:
if (HighlightRuby.is_dollar_one(c)) {
this.append('#');
this.push("strong", "");
this.append('$');
this.append(c);
this.pop();
this.state = HighlightRuby.PERCENT_STRING;
} else if (isalpha(c)) {
this.append('#');
this.push("strong", "");
this.append('$');
this.append(c);
this.state = HighlightRuby.PERCENT_HASH_DOLLAR_WORD;
} else {
this.append('#');
this.append('$');
this.epsilon(HighlightRuby.PERCENT_STRING);
}
break;

case PERCENT_HASH_DOLLAR_WORD:
if (HighlightRuby.isident(c)) {
this.append(c);
} else {
this.push("strong", "");
this.epsilon(HighlightRuby.PERCENT_STRING);
}
break;

case HighlightRuby.PERCENT_STRING:
this.append(c);
if (c == this.opener && this.opener != this.closer) {
++this.level;
} else if (c == '#') {
this.state = HighlightRuby.PERCENT_HASH;
} else if (c == this.closer) {
if (!--this.level) {
this.pop();
Expand Down Expand Up @@ -770,7 +841,7 @@ class HighlightRuby extends Highlighter {
break;

case HighlightRuby.LT_LT:
if (c == '-') {
if (c == '-' || c == '~') {
this.indented_heredoc = true;
this.append(c);
} else if (c == '\'' || c == '`' || c == '"') {
Expand Down Expand Up @@ -822,7 +893,7 @@ class HighlightRuby extends Highlighter {
this.pop();
}
this.i = 0;
} else if (c == '\t' && this.indented_heredoc) {
} else if (isblank(c) && this.indented_heredoc) {
// do nothing
} else if (this.i < this.heredoc.length && this.heredoc[this.i] == c) {
this.i++;
Expand Down Expand Up @@ -913,6 +984,14 @@ class HighlightRuby extends Highlighter {
this.append('%');
this.append(this.q);
break;
case HighlightRuby.PERCENT_HASH:
this.append('#');
this.pop();
break;
case HighlightRuby.PERCENT_HASH_DOLLAR:
this.append("#$");
this.pop();
break;
case HighlightRuby.QUESTION:
this.append('?');
break;
Expand All @@ -939,6 +1018,7 @@ class HighlightRuby extends Highlighter {
case HighlightRuby.REGEX_BACKSLASH:
case HighlightRuby.REGEX_HASH_DOLLAR_WORD:
case HighlightRuby.PERCENT_STRING:
case HighlightRuby.PERCENT_HASH_DOLLAR_WORD:
case HighlightRuby.AT_WORD:
case HighlightRuby.DOLLAR_WORD:
case HighlightRuby.TICK:
Expand Down

0 comments on commit 33a057e

Please sign in to comment.