diff --git a/lib/rouge/demos/davi b/lib/rouge/demos/davi new file mode 100644 index 0000000000..82ebc427ef --- /dev/null +++ b/lib/rouge/demos/davi @@ -0,0 +1,3 @@ + diff --git a/lib/rouge/lexers/davi.rb b/lib/rouge/lexers/davi.rb new file mode 100644 index 0000000000..5a89f79942 --- /dev/null +++ b/lib/rouge/lexers/davi.rb @@ -0,0 +1,377 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + class DAVI < TemplateLexer + title "DAVI" + desc "The DaVinci scripting language (davinciscript.com)" + tag 'davi' + filenames '*.davi' + mimetypes 'text/x-davi' + + option :start_inline, 'Whether to start with inline davi or require . (default: best guess)' + option :funcnamehighlighting, 'Whether to highlight builtin functions (default: true)' + option :disabledmodules, 'Disable certain modules from being highlighted as builtins (default: empty)' + + def initialize(*) + super + + # if truthy, the lexer starts highlighting with davi code + # (no / do + token Comment::Preproc + reset_stack + end + end + + state :return do + rule(//) { pop! } + end + + state :start do + # We enter this state if we aren't sure whether the DAVI in the text is + # delimited by \/@-]+/, Operator + end + + state :string do + rule %r/"/, Str::Double, :pop! + rule %r/[^\\{$"]+/, Str::Double + rule %r/\\u\{[0-9a-fA-F]+\}/, Str::Escape + rule %r/\\([efrntv\"$\\]|[0-7]{1,3}|[xX][0-9a-fA-F]{1,2})/, Str::Escape + rule %r/\$#{id}(\[\S+\]|->#{id})?/, Name::Variable + + rule %r/\{\$\{/, Str::Interpol, :string_interp_double + rule %r/\{(?=\$)/, Str::Interpol, :string_interp_single + rule %r/(\{)(\S+)(\})/ do + groups Str::Interpol, Name::Variable, Str::Interpol + end + + rule %r/[${\\]+/, Str::Double + end + + state :string_interp_double do + rule %r/\}\}/, Str::Interpol, :pop! + mixin :davi + end + + state :string_interp_single do + rule %r/\}/, Str::Interpol, :pop! + mixin :davi + end + + state :values do + # heredocs + rule %r/<<<(["']?)(#{id})\1\n.*?\n\s*\2;?/im, Str::Heredoc + + # numbers + rule %r/(\d[_\d]*)?\.(\d[_\d]*)?(e[+-]?\d[_\d]*)?/i, Num::Float + rule %r/0[0-7][0-7_]*/, Num::Oct + rule %r/0b[01][01_]*/i, Num::Bin + rule %r/0x[a-f0-9][a-f0-9_]*/i, Num::Hex + rule %r/\d[_\d]*/, Num::Integer + + # strings + rule %r/'([^'\\]*(?:\\.[^'\\]*)*)'/, Str::Single + rule %r/`([^`\\]*(?:\\.[^`\\]*)*)`/, Str::Backtick + rule %r/"/, Str::Double, :string + + # functions + rule %r/(function|fn)\b/i do + push :in_function_return + push :in_function_params + push :in_function_name + token Keyword + end + + # constants + rule %r/(true|false|null)\b/i, Keyword::Constant + + # objects + rule %r/new\b/i, Keyword, :in_new + end + + state :variables do + rule %r/\$\{\$+#{id}\}/, Name::Variable + rule %r/\$+#{id}/, Name::Variable + end + + state :whitespace do + rule %r/\s+/, Text + rule %r/#[^\[].*?$/, Comment::Single + rule %r(//.*?$), Comment::Single + rule %r(/\*\*(?!/).*?\*/)m, Comment::Doc + rule %r(/\*.*?\*/)m, Comment::Multiline + end + + state :root do + rule %r/<\?(davi|=)?/i, Comment::Preproc, :davi + rule(/.*?(?=<\?)|.*/m) { delegate parent } + end + + state :davi do + mixin :escape + + mixin :whitespace + mixin :variables + mixin :values + + rule %r/(namespace) + (\s+) + (#{id_with_ns})/ix do |m| + groups Keyword::Namespace, Text, Name::Namespace + end + + rule %r/#\[.*\]$/, Name::Attribute + + rule %r/(class|interface|trait|extends|implements) + (\s+) + (#{id_with_ns})/ix do |m| + groups Keyword::Declaration, Text, Name::Class + end + + mixin :names + + rule %r/[;,\(\)\{\}\[\]]/, Punctuation + + mixin :operators + rule %r/[=?]/, Operator + end + + state :in_assign do + rule %r/,/, Punctuation, :pop! + rule %r/[\[\]]/, Punctuation + rule %r/\(/, Punctuation, :in_assign_function + mixin :escape + mixin :whitespace + mixin :values + mixin :variables + mixin :names + mixin :operators + mixin :return + end + + state :in_assign_function do + rule %r/\)/, Punctuation, :pop! + rule %r/,/, Punctuation + mixin :in_assign + end + + state :in_catch do + rule %r/\(/, Punctuation + rule %r/\|/, Operator + rule id_with_ns, Name::Class + mixin :escape + mixin :whitespace + mixin :return + end + + state :in_const do + rule id, Name::Constant + rule %r/=/, Operator, :in_assign + mixin :escape + mixin :whitespace + mixin :return + end + + state :in_function_body do + rule %r/{/, Punctuation, :push + rule %r/}/, Punctuation, :pop! + mixin :davi + end + + state :in_function_name do + rule %r/&/, Operator + rule id, Name + rule %r/\(/, Punctuation, :pop! + mixin :escape + mixin :whitespace + mixin :return + end + + state :in_function_params do + rule %r/\)/, Punctuation, :pop! + rule %r/,/, Punctuation + rule %r/[.]{3}/, Punctuation + rule %r/=/, Operator, :in_assign + rule %r/\b(?:public|protected|private|readonly)\b/i, Keyword + rule %r/\??#{id}/, Keyword::Type, :in_assign + mixin :escape + mixin :whitespace + mixin :variables + mixin :return + end + + state :in_function_return do + rule %r/:/, Punctuation + rule %r/use\b/i, Keyword, :in_function_use + rule %r/\??#{id}/, Keyword::Type, :in_assign + rule %r/\{/ do + token Punctuation + goto :in_function_body + end + mixin :escape + mixin :whitespace + mixin :return + end + + state :in_function_use do + rule %r/[,\(]/, Punctuation + rule %r/&/, Operator + rule %r/\)/, Punctuation, :pop! + mixin :escape + mixin :whitespace + mixin :variables + mixin :return + end + + state :in_new do + rule %r/class\b/i do + token Keyword::Declaration + goto :in_new_class + end + rule id_with_ns, Name::Class, :pop! + mixin :escape + mixin :whitespace + mixin :return + end + + state :in_new_class do + rule %r/\}/, Punctuation, :pop! + rule %r/\{/, Punctuation + mixin :davi + end + + state :in_use do + rule %r/[,\}]/, Punctuation + rule %r/(function|const)\b/i, Keyword + rule %r/(#{ns})(\{)/ do + groups Name::Namespace, Punctuation + end + rule %r/#{id_with_ns}(_#{id})+/, Name::Function + mixin :escape + mixin :whitespace + mixin :names + mixin :return + end + + state :in_visibility do + rule %r/\b(?:readonly|static)\b/i, Keyword + rule %r/(?=(abstract|const|function)\b)/i, Keyword, :pop! + rule %r/\??#{id}/, Keyword::Type, :pop! + mixin :escape + mixin :whitespace + mixin :return + end + end + end +end diff --git a/lib/rouge/lexers/davi/keywords.rb b/lib/rouge/lexers/davi/keywords.rb new file mode 100644 index 0000000000..a42c2251bb --- /dev/null +++ b/lib/rouge/lexers/davi/keywords.rb @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +# DO NOT EDIT +# This file is automatically generated by `rake builtins:davi`. +# See tasks/builtins/davi.rake for more info. + +module Rouge + module Lexers + class DAVI + def self.builtins + @builtins ||= {}.tap do |b| + b["Array"] = Set.new ["append", "range", "slice", "sort"] + b["Conversion"] = Set.new ["int", "str"] + b["File System"] = Set.new ["fileGetContents"] + b["HTTP"] = Set.new ["httpListen", "httpRegister"] + b["String"] = Set.new ["camelCase", "char", "dotCase", "explode", "find", "join", "kebabCase", "len", "lower", "lowerFirst", "lowerWords", "pascalCase", "rune", "snakeCase", "split", "type", "upFirst", "upWords", "upper"] + b["System"] = Set.new ["args", "echo", "exit", "read", "time"] + end + end + end + end +end \ No newline at end of file diff --git a/spec/lexers/davi_spec.rb b/spec/lexers/davi_spec.rb new file mode 100644 index 0000000000..42827a4849 --- /dev/null +++ b/spec/lexers/davi_spec.rb @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::DAVI do + let(:subject) { Rouge::Lexers::DAVI.new } + + describe 'guessing' do + include Support::Guessing + + it 'Guesses files containing ' 'foo.davi', :source => ' 'foo.davi', :source => ' ' ' '.davi', :source => 'It's html outside davi!

+ + + + + + + + + + + + + + + + + + +I can close davi tags and get html! + + +other_thing()}" + + +/* Functions */ + +function &byref() { + $x = array(); + return $x; +} + +public abstract function thing(mysqli $mysqli, int FOO + BAR, ?mysqli $mysqli): mysqli $mysqli; + +function sort_on(array &$array, string $key) { + usort($array, function ($a, $b) use($key) { return $a <=> $b; }); +} + +// Function-like keywords. +isset($foo); +empty($bar); +unset($baz); +die($qux); + +// No "use" +$example = function () { + some_fn($message); +}; + +// Inherit $message +$example = function () use ($message) { + some_fn($message); +}; + +// A generator +$gen = (function() { + yield 1; + yield 2; + + return 3; +})(); + +$fn = fn($x) => fn($y) => $x * $y + $z; +fn(array $x) => $x; +static fn(): int => $x; +fn($x = 42) => $x; +fn(&$x) => $x; +fn&($x) => $x; +fn($x, ...$rest) => $rest; + + +/* Classes */ + +class Foo extends AbstractFoo implements Bar { +} + +//Anonymous class example +$app->setLogger(new class implements Logger { + public function log(string $msg) { + echo $msg; + } +}); + +class ClassB +{ + use TraitA, TraitB, TraitC { + TraitB::foo insteadof TraitA; + } +} + +class Zip\Zipp { + +} + +class Zip extends Archive { + function out($filename = false) { + // Empty output + $file_data = array(); // Data of the file part + $cd_data = array(); // Data of the central directory + } +} + +class User +{ + public int $id; + public ?string $name; + + public function __construct(int $id, ?string $name) + { + $this->id = $id; + $this->name = $name; + } +} + +class C { + private readonly $x; + protected readonly int $y; + private string $z; + + public static ?string $a = ""; + public static int $b = 0; + + public function __construct( + public $foo, + protected readonly $bar, + private readonly array $baz = [], + private string $qux = "", + ) { + } +} + + +/* Traits */ + +trait SomeTrait { + // Some visibility + const PUBLIC_CONST_A = 1; + public const PUBLIC_CONST_B = 2; + protected const PROTECTED_CONST = 3; + private const PRIVATE_CONST = 4; + + function afunc(string $arg, SomeInterface $arg2, callable $arg3, object $arg4): void { + echo "hello"; + if (1 <=> 1) { + echo "yep!"; + } + } +} + + +/* Interfaces */ + +interface SomeInterface { + function interfaceFunc(bool $arg): iterable; + function nullableTypes(?bool $arg): ?iterable; +} + + +/* Imports */ + +use some\namespace\ClassA; +use some\namespace\ClassB as B; +use function some\namespace\fn_a; +use const some\namespace\ConstA; +use Class1, /*Class2, */Class3; + +use some\namespace\{ClassA, ClassB, ClassC as C}; +use function some\namespace\{fn_a, fn_b, fn_c}; +use const some\namespace\{ConstA, ConstB, ConstC}; +use some\name\{function some_fn, const Foo\BAR, SomeClass}; + +use const some\namespace\{ + ConstA, + #ConstB, + ConstC +}; + + +/* Namespaces */ + +namespace some\namespace; + + +/* Assignments */ + +list($id1, $name1) = $data[0]; +[$id1, $name1] = $data[0]; +const A = f(1, 2, g(3, 4, $my_var[0])), B = "abc"; + + +/* Conditionals */ + +if(!defined('UNLOCK') || !UNLOCK) + die(); + +if(function_exists('gzdeflate')) { + $method = 8; + $compressed_data = gzdeflate($content); +} elseif(function_exists('bzcompress')) { + $method = 12; + $compressed_data = bzcompress($content); +} else { + $compressed_data = $content; +} + +switch($header_info['compression_method']) { + case 0: + $content = $data; + break; + + case 8: + $content = gzinflate($data); + break; + + case 12: + if(!function_exists('bzdecompress')) + return false; + break; + + default: + return false; +} + +$y = match ($x) { + 0 => 1, + 1 => -1, + default => 0, +}; + +/* Loops */ + +foreach($this->files as $name => $file) { + $content = $file[0]; + $fd .= "\x50\x4b\x03\x04"; // Local file header signature + $fd .= pack("v", 20); // Version needed to extract + $fd .= pack("V", crc32($content)); // crc-32 + $fd .= pack("V", strlen($compressed_data)); // Compressed size +} + +while($pos < strlen($cdr)) { + if(substr($cdr, $pos, 4) == "\x50\x4b\x05\x05") { + $tmp_info = unpack('vsize', substr($cdr, $pos + 4, 2)); + $digital_sig = substr($header, $pos + 6, $tmp_info['size']); + break; + } +} + + +/* Exceptions */ + +class Test { + public function testing() { + try { + throw new MyException(); + } catch (MyException | MyOtherException $e) { + var_dump(get_class($e)); + } catch (\Exception $x) { + throw $x; + } + } +} + +?> + + + + +

Hello world!

+ + + +

Hello world!

+ + +

it's html here at the end, too.

+ + +/* Attributes */ + + "value"))] +#[MyAttribute(100 + 200)] +class SetUp { + #[JsonSerialize('Setup')] + public const version = '0.0.1'; + + #[SetUp] + public function fileExists() {} +} + +function f(\Ns1\Ns2\Ns3\Class $x) { +} + +?> diff --git a/tasks/builtins/davi.rake b/tasks/builtins/davi.rake new file mode 100644 index 0000000000..587406cf41 --- /dev/null +++ b/tasks/builtins/davi.rake @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +DAVI_DOCS_URI = "https://raw.githubusercontent.com/davinci-script/davi/main/docs/docs/.vuepress/dist/davi-details.json" +DAVI_KEYWORDS_FILE = "./lib/rouge/lexers/davi/keywords.rb" + +namespace :builtins do + task :davi do + generator = Rouge::Tasks::Builtins::DAVI.new + + files = generator.download_docs(DAVI_DOCS_URI) + keywords = generator.extract_keywords(files.values) + output = generator.render_output(keywords) + + File.write(DAVI_KEYWORDS_FILE, output) + + end +end + +module Rouge + module Tasks + module Builtins + class DAVI + def download_docs(input) + files = Hash.new + + system "mkdir -p /tmp/rouge" + Dir.chdir "/tmp/rouge" do + system "wget #{input} -O davi-details.json -q " + files["davi-details.json"] = File.read("davi-details.json") + end + + files + end + + def extract_keywords(files) + keywords = Hash.new { |h,k| h[k] = Array.new } + json = JSON.parse(files[0]) + + json.each do |k, v| + keywords[k] = json[k].keys + end + + keywords + end + + def render_output(keywords, &b) + return enum_for(:render_output, keywords).to_a.join("\n") unless b + + yield "# -*- coding: utf-8 -*- #" + yield "# frozen_string_literal: true" + yield "" + yield "# DO NOT EDIT" + yield "# This file is automatically generated by `rake builtins:davi`." + yield "# See tasks/builtins/davi.rake for more info." + yield "" + yield "module Rouge" + yield " module Lexers" + yield " class DAVI" + yield " def self.builtins" + yield " @builtins ||= {}.tap do |b|" + keywords.each do |n, fs| + yield " b[#{n.inspect}] = Set.new #{fs.inspect}" + end + yield " end" + yield " end" + yield " end" + yield " end" + yield "end" + end + end + end + end +end + +def davi_references(&b) + return enum_for :davi_references unless block_given? + + davi_manual_url = 'http://us3.davi.net/distributions/manual/davi_manual_en.tar.gz' + + sh 'mkdir -p /tmp/rouge', { verbose: false } + Dir.chdir '/tmp/rouge' do + sh "wget -qO- #{davi_manual_url} | tar -xz", { verbose: false } + Dir.chdir './davi-chunked-xhtml' do + Dir.glob('./ref.*').sort.each { |x| yield File.read(x) } + end + end +end + +def davi_functions(&b) + return enum_for :davi_functions unless block_given? + + davi_references do |file| + file =~ %r((.*?) Functions) + name = $1 + + next unless name + + functions = file.scan %r((.*?)) + + yield [name, functions] + end +end + +def davi_builtins_source + yield "# -*- coding: utf-8 -*- #" + yield "# frozen_string_literal: true" + yield "" + yield "# automatically generated by `rake builtins:davi`" + yield "module Rouge" + yield " module Lexers" + yield " class DAVI" + yield " def self.builtins" + yield " @builtins ||= {}.tap do |b|" + davi_functions do |n, fs| + yield " b[#{n.inspect}] = Set.new %w(#{fs.join(' ')})" + end + yield " end" + yield " end" + yield " end" + yield " end" + yield "end" +end