block comment lexer failed

[dpull]

test.lua :

#!/usr/bin/env lua
local pretty = require("pl.pretty")
local lexer = require("pl.lexer")

-- 1234
--- 5678
--[=[
    local function get_1212121212(file_path)
    end
]=]
function parse(lua_code)
    local lexical, f = lexer.lua(lua_code) --, {space=true, comments=false})
    local items = {}
    
    for t, v in lexical do
        local line = lexer.lineno(lexical)
        table.insert(items, {t, v, line})
    end
    return items
end

local function get_luafunctions(file_path)
    local lines = io.lines(file_path)
    local file = assert(io.open(file_path))
    local lua_lexical = parse(file)
    pretty.dump(lua_lexical)
end

local file = arg[1]
get_luafunctions(file)

execute on lua5.1.4, ./test.lua test.lua

--[=[
    local function get_1212121212(file_path)
    end
]=]

at line 7:10 lexer result:

  {
    "keyword",
    "local",
    8
  },
  {
    "keyword",
    "function",
    8
  },
  {
    "iden",
    "get_1212121212",
    8
  },
  {
    "(",
    "(",
    8
  },
  {
    "iden",
    "file_path",
    8
  },
  {
    ")",
    ")",
    8
  },
  {
    "keyword",
    "end",
    9
  },
  {
    [=[]]=],
    [=[]]=],
    10
  },
  {
    "=",
    "=",
    10
  },
  {
    [=[]]=],
    [=[]]=],
    10
  },

[Tieske]

The problem is here that it reads from a file, and parses line by line. Hence when matching it doesn't match the block-comments since the terminator is on a line that hasn't been read yet. The next in line is the regular -- token, which then will match and produces this output.

Load the full file in memory as a string and parse it, then it will work.