From a599ce9cbf37f4ae150ec22235c0a68ef66ce878 Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 12:49:08 +0100 Subject: [PATCH 1/6] apply formatting --- lib/eden.ex | 12 +-- lib/eden/decode.ex | 54 ++++++---- lib/eden/encode.ex | 36 ++++--- lib/eden/exception.ex | 28 ++++-- lib/eden/lexer.ex | 203 ++++++++++++++++++++++---------------- lib/eden/parser.ex | 81 +++++++++------ lib/eden/parser/node.ex | 18 ++-- test/eden/lexer_test.exs | 114 ++++++++++++--------- test/eden/parser_test.exs | 130 ++++++++++-------------- test/eden_test.exs | 15 +-- test/test_helper.exs | 2 +- 11 files changed, 393 insertions(+), 300 deletions(-) diff --git a/lib/eden.ex b/lib/eden.ex index 8ed9257..4cb9177 100644 --- a/lib/eden.ex +++ b/lib/eden.ex @@ -10,8 +10,7 @@ defmodule Eden do alias Eden.Decode alias Eden.Exception, as: Ex - @default_handlers %{"inst" => &Eden.Tag.inst/1, - "uuid" => &Eden.Tag.uuid/1} + @default_handlers %{"inst" => &Eden.Tag.inst/1, "uuid" => &Eden.Tag.uuid/1} @doc """ Encodes an *Elixir* term that implements the `Eden.Encode` protocol. @@ -55,7 +54,7 @@ defmodule Eden do iex> Eden.encode({:a, 1}) {:error, Protocol.UndefinedError} """ - @spec encode(Encode.t) :: {:ok, String.t} | {:error, atom} + @spec encode(Encode.t()) :: {:ok, String.t()} | {:error, atom} def encode(data) do try do {:ok, encode!(data)} @@ -70,7 +69,7 @@ defmodule Eden do Returns the function result otherwise. """ - @spec encode!(Encode.t) :: String.t + @spec encode!(Encode.t()) :: String.t() def encode!(data) do Encode.encode(data) end @@ -102,7 +101,7 @@ defmodule Eden do iex> Eden.decode("nil true false .") {:error, Eden.Exception.UnexpectedInputError} """ - @spec decode(String.t, Keyword.t) :: {:ok, any} | {:error, atom} + @spec decode(String.t(), Keyword.t()) :: {:ok, any} | {:error, atom} def decode(input, opts \\ []) do try do {:ok, decode!(input, opts)} @@ -117,11 +116,12 @@ defmodule Eden do Returns the function result otherwise. """ - @spec decode!(String.t, Keyword.t) :: any + @spec decode!(String.t(), Keyword.t()) :: any def decode!(input, opts \\ []) do tree = parse(input, location: true) handlers = Map.merge(@default_handlers, opts[:handlers] || %{}) opts = [handlers: handlers] + case Decode.decode(tree, opts) do [] -> raise Ex.EmptyInputError, input [data] -> data diff --git a/lib/eden/decode.ex b/lib/eden/decode.ex index cdaf9ff..358735a 100644 --- a/lib/eden/decode.ex +++ b/lib/eden/decode.ex @@ -9,81 +9,101 @@ defmodule Eden.Decode do def decode(children, opts) when is_list(children) do Enum.map(children, fn x -> decode(x, opts) end) end + def decode(%Node{type: :root, children: children}, opts) do decode(children, opts) end - def decode(%Node{type: :nil}, _opts) do - :nil + + def decode(%Node{type: nil}, _opts) do + nil end - def decode(%Node{type: :true}, _opts) do - :true + + def decode(%Node{type: true}, _opts) do + true end - def decode(%Node{type: :false}, _opts) do - :false + + def decode(%Node{type: false}, _opts) do + false end + def decode(%Node{type: :string, value: value}, _opts) do value end + def decode(%Node{type: :character, value: value}, _opts) do %Character{char: value} end + def decode(%Node{type: :symbol, value: value}, _opts) do %Symbol{name: value} end + def decode(%Node{type: :keyword, value: value}, _opts) do String.to_atom(value) end + def decode(%Node{type: :integer, value: value}, _opts) do value = String.trim_trailing(value, "N") :erlang.binary_to_integer(value) end + def decode(%Node{type: :float, value: value}, _opts) do value = String.trim_trailing(value, "M") # Elixir/Erlang don't convert to float if there # is no decimal part. - final_value = if not String.contains?(value, ".") do - if String.match?(value, ~r/[eE]/) do - String.replace(value, ~r/[eE]/, ".0E") + final_value = + if not String.contains?(value, ".") do + if String.match?(value, ~r/[eE]/) do + String.replace(value, ~r/[eE]/, ".0E") + else + value <> ".0" + end else - value <> ".0" + value end - else - value - end + :erlang.binary_to_float(final_value) end + def decode(%Node{type: :list, children: children}, opts) do decode(children, opts) end + def decode(%Node{type: :vector, children: children}, opts) do children |> decode(opts) - |> Array.from_list + |> Array.from_list() end + def decode(%Node{type: :map, children: children} = node, opts) do - if Integer.is_odd(length children) do + if Integer.is_odd(length(children)) do raise Ex.OddExpressionCountError, node end + children |> decode(opts) |> Enum.chunk_every(2) |> Enum.map(fn [a, b] -> {a, b} end) |> Enum.into(%{}) end + def decode(%Node{type: :set, children: children}, opts) do children |> decode(opts) - |> Enum.into(MapSet.new) + |> Enum.into(MapSet.new()) end + def decode(%Node{type: :tag, value: name, children: [child]}, opts) do case Map.get(opts[:handlers], name) do nil -> %Tag{name: name, value: decode(child, opts)} + handler -> handler.(decode(child, opts)) end end + def decode(%Node{type: type}, _opts) do - raise "Unrecognized node type: #{inspect type}" + raise "Unrecognized node type: #{inspect(type)}" end end diff --git a/lib/eden/encode.ex b/lib/eden/encode.ex index 45710bb..149693d 100644 --- a/lib/eden/encode.ex +++ b/lib/eden/encode.ex @@ -8,43 +8,54 @@ alias Eden.Tag defprotocol Eden.Encode do @fallback_to_any true - @spec encode(any) :: String.t + @spec encode(any) :: String.t() def encode(value) end defmodule Eden.Encode.Utils do - def wrap(str, first, last )do + def wrap(str, first, last) do first <> str <> last end end defimpl Encode, for: Atom do def encode(atom) when atom in [nil, true, false] do - Atom.to_string atom + Atom.to_string(atom) end + def encode(atom) do - ":" <> Atom.to_string atom + ":" <> Atom.to_string(atom) end end defimpl Encode, for: Symbol do - def encode(symbol) do symbol.name end + def encode(symbol) do + symbol.name + end end defimpl Encode, for: BitString do - def encode(string) do "\"#{string}\"" end + def encode(string) do + "\"#{string}\"" + end end defimpl Encode, for: Character do - def encode(char) do "\\#{char.char}" end + def encode(char) do + "\\#{char.char}" + end end defimpl Encode, for: Integer do - def encode(int) do "#{inspect int}" end + def encode(int) do + "#{inspect(int)}" + end end defimpl Encode, for: Float do - def encode(float) do "#{inspect float}" end + def encode(float) do + "#{inspect(float)}" + end end defimpl Encode, for: List do @@ -59,7 +70,7 @@ end defimpl Encode, for: Array do def encode(array) do array - |> Array.to_list + |> Array.to_list() |> Enum.map(&Encode.encode/1) |> Enum.join(", ") |> Utils.wrap("[", "]") @@ -69,7 +80,7 @@ end defimpl Encode, for: Map do def encode(map) do map - |> Map.to_list + |> Map.to_list() |> Enum.map(fn {k, v} -> Encode.encode(k) <> " " <> Encode.encode(v) end) |> Enum.join(", ") |> Utils.wrap("{", "}") @@ -109,7 +120,8 @@ defimpl Encode, for: Any do def encode(struct) when is_map(struct) do Encode.encode(Map.from_struct(struct)) end - def encode(value) do + + def encode(value) do raise %Protocol.UndefinedError{protocol: Encode, value: value} end end diff --git a/lib/eden/exception.ex b/lib/eden/exception.ex index 2f962a2..069e952 100644 --- a/lib/eden/exception.ex +++ b/lib/eden/exception.ex @@ -1,11 +1,11 @@ defmodule Eden.Exception do - defmodule Util do def token_message(token) do - msg = "(#{inspect token.type}) #{inspect token.value}" + msg = "(#{inspect(token.type)}) #{inspect(token.value)}" + if Map.has_key?(token, :location) do loc = token.location - msg <> " at line #{inspect loc.line} and column #{inspect loc.col}." + msg <> " at line #{inspect(loc.line)} and column #{inspect(loc.col)}." else msg end @@ -34,6 +34,7 @@ defmodule Eden.Exception do defmodule UnexpectedTokenError do defexception [:message] + def exception(token) do %UnexpectedTokenError{message: Util.token_message(token)} end @@ -41,29 +42,33 @@ defmodule Eden.Exception do defmodule UnbalancedDelimiterError do defexception [:message] + def exception(msg) do - %UnbalancedDelimiterError{message: "#{inspect msg}"} + %UnbalancedDelimiterError{message: "#{inspect(msg)}"} end end defmodule OddExpressionCountError do defexception [:message] + def exception(msg) do - %OddExpressionCountError{message: "#{inspect msg}"} + %OddExpressionCountError{message: "#{inspect(msg)}"} end end defmodule IncompleteTagError do defexception [:message] + def exception(msg) do - %IncompleteTagError{message: "#{inspect msg}"} + %IncompleteTagError{message: "#{inspect(msg)}"} end end defmodule MissingDiscardExpressionError do defexception [:message] + def exception(msg) do - %MissingDiscardExpressionError{message: "#{inspect msg}"} + %MissingDiscardExpressionError{message: "#{inspect(msg)}"} end end @@ -71,19 +76,22 @@ defmodule Eden.Exception do defmodule EmptyInputError do defexception [:message] + def exception(msg) do - %EmptyInputError{message: "#{inspect msg}"} + %EmptyInputError{message: "#{inspect(msg)}"} end end defmodule NotImplementedError do defexception [:message] + def exception(msg) when is_binary(msg) do %NotImplementedError{message: msg} end + def exception({function, arity}) do - function = Atom.to_string function - %NotImplementedError{message: "#{function}/#{inspect arity}"} + function = Atom.to_string(function) + %NotImplementedError{message: "#{function}/#{inspect(arity)}"} end end end diff --git a/lib/eden/lexer.ex b/lib/eden/lexer.ex index 293b4dc..088d24d 100644 --- a/lib/eden/lexer.ex +++ b/lib/eden/lexer.ex @@ -1,5 +1,6 @@ defmodule Eden.Lexer do alias Eden.Exception, as: Ex + @moduledoc """ A module that implements a lexer for the edn format through its only function `tokenize/1`. @@ -31,11 +32,14 @@ defmodule Eden.Lexer do [%Eden.Lexer.Token{location: %{col: 0, line: 1}, type: nil, value: "nil"}] """ def tokenize(input, opts \\ [location: false]) do - initial_state = %{state: :new, - tokens: [], - current: nil, - opts: opts, - location: %{line: 1, col: 0}} + initial_state = %{ + state: :new, + tokens: [], + current: nil, + opts: opts, + location: %{line: 1, col: 0} + } + _tokenize(initial_state, input) end @@ -49,40 +53,46 @@ defmodule Eden.Lexer do |> valid? |> add_token(state.current) |> Map.get(:tokens) - |> Enum.reverse + |> Enum.reverse() end # Comment - defp _tokenize(state = %{state: :new}, <<";" :: utf8, rest :: binary>>) do + defp _tokenize(state = %{state: :new}, <<";"::utf8, rest::binary>>) do token = token(:comment, "") start_token(state, :comment, token, ";", rest) end - defp _tokenize(state = %{state: :comment}, <>) - when <> in ["\n", "\r"] do - end_token(state, <>, rest) + + defp _tokenize(state = %{state: :comment}, <>) + when <> in ["\n", "\r"] do + end_token(state, <>, rest) end - defp _tokenize(state = %{state: :comment}, <<";" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :comment}, <<";"::utf8, rest::binary>>) do skip_char(state, ";", rest) end - defp _tokenize(state = %{state: :comment}, <>) do - consume_char(state, <>, rest) + + defp _tokenize(state = %{state: :comment}, <>) do + consume_char(state, <>, rest) end # Literals - defp _tokenize(state = %{state: :new}, <<"nil" :: utf8, rest :: binary>>) do - token = token(:nil, "nil") + defp _tokenize(state = %{state: :new}, <<"nil"::utf8, rest::binary>>) do + token = token(nil, "nil") start_token(state, :check_literal, token, "nil", rest) end - defp _tokenize(state = %{state: :new}, <<"true" :: utf8, rest :: binary>>) do - token = token(:true, "true") + + defp _tokenize(state = %{state: :new}, <<"true"::utf8, rest::binary>>) do + token = token(true, "true") start_token(state, :check_literal, token, "true", rest) end - defp _tokenize(state = %{state: :new}, <<"false" :: utf8, rest :: binary>>) do - token = token(:false, "false") + + defp _tokenize(state = %{state: :new}, <<"false"::utf8, rest::binary>>) do + token = token(false, "false") start_token(state, :check_literal, token, "false", rest) end - defp _tokenize(state = %{state: :check_literal}, <> = input) do - if separator?(<>) do + + defp _tokenize(state = %{state: :check_literal}, <> = input) do + if separator?(<>) do end_token(state, "", input) else token = token(:symbol, state.current.value) @@ -91,142 +101,159 @@ defmodule Eden.Lexer do end # String - defp _tokenize(state = %{state: :new}, <<"\"" :: utf8, rest :: binary>>) do + defp _tokenize(state = %{state: :new}, <<"\""::utf8, rest::binary>>) do token = token(:string, "") start_token(state, :string, token, "\"", rest) end - defp _tokenize(state = %{state: :string}, <<"\\" :: utf8, char :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :string}, <<"\\"::utf8, char::utf8, rest::binary>>) do # TODO: this will cause the line count to get corrupted, # either use the original or send the real content as # an optional argument. - consume_char(state, escaped_char(<>), rest, <<"\\" :: utf8, char :: utf8>>) + consume_char(state, escaped_char(<>), rest, <<"\\"::utf8, char::utf8>>) end - defp _tokenize(state = %{state: :string}, <<"\"" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :string}, <<"\""::utf8, rest::binary>>) do end_token(state, "\"", rest) end - defp _tokenize(state = %{state: :string}, <>) do - consume_char(state, <>, rest) + + defp _tokenize(state = %{state: :string}, <>) do + consume_char(state, <>, rest) end # Character - defp _tokenize(state = %{state: :new}, <<"\\" :: utf8, char :: utf8, rest :: binary>>) do - token = token(:character, <>) - end_token(state, token, "\\" <> <>, rest) + defp _tokenize(state = %{state: :new}, <<"\\"::utf8, char::utf8, rest::binary>>) do + token = token(:character, <>) + end_token(state, token, "\\" <> <>, rest) end # Keyword and Symbol - defp _tokenize(state = %{state: :new}, <<":" :: utf8, rest :: binary>>) do + defp _tokenize(state = %{state: :new}, <<":"::utf8, rest::binary>>) do token = token(:keyword, "") start_token(state, :symbol, token, ":", rest) end - defp _tokenize(state = %{state: :symbol}, <<"/" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :symbol}, <<"/"::utf8, rest::binary>>) do if not String.contains?(state.current.value, "/") do - consume_char(state, <<"/" :: utf8>>, rest) + consume_char(state, <<"/"::utf8>>, rest) else raise Ex.UnexpectedInputError, "/" end end - defp _tokenize(state = %{state: :symbol}, <> = input) do - if symbol_char?(<>) do - consume_char(state, <>, rest) + + defp _tokenize(state = %{state: :symbol}, <> = input) do + if symbol_char?(<>) do + consume_char(state, <>, rest) else end_token(state, "", input) end end # Integers & Float - defp _tokenize(state = %{state: :new}, <>) - when <> in ["-", "+"] do + defp _tokenize(state = %{state: :new}, <>) + when <> in ["-", "+"] do token = token(:integer, <>) - start_token(state, :number, token, <>, rest) + start_token(state, :number, token, <>, rest) end - defp _tokenize(state = %{state: :exponent}, <>) - when <> in ["-", "+"] do - consume_char(state, <>, rest) + + defp _tokenize(state = %{state: :exponent}, <>) + when <> in ["-", "+"] do + consume_char(state, <>, rest) end - defp _tokenize(state = %{state: :number}, <<"N" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :number}, <<"N"::utf8, rest::binary>>) do state = append_to_current(state, "N") end_token(state, "N", rest) end - defp _tokenize(state = %{state: :number}, <<"M" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :number}, <<"M"::utf8, rest::binary>>) do state = append_to_current(state, "M") token = token(:float, state.current.value) end_token(state, token, "M", rest) end - defp _tokenize(state = %{state: :number}, <<"." :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :number}, <<"."::utf8, rest::binary>>) do state = append_to_current(state, ".") token = token(:float, state.current.value) start_token(state, :fraction, token, ".", rest) end - defp _tokenize(state = %{state: :number}, <>) - when <> in ["e", "E"] do - state = append_to_current(state, <>) + + defp _tokenize(state = %{state: :number}, <>) + when <> in ["e", "E"] do + state = append_to_current(state, <>) token = token(:float, state.current.value) - start_token(state, :exponent, token, <>, rest) + start_token(state, :exponent, token, <>, rest) end - defp _tokenize(state = %{state: s}, <> = input) - when s in [:number, :exponent, :fraction] do + + defp _tokenize(state = %{state: s}, <> = input) + when s in [:number, :exponent, :fraction] do cond do - digit?(<>) -> + digit?(<>) -> state |> set_state(:number) - |> consume_char(<>, rest) - s in [:exponent, :fraction] and separator?(<>) -> + |> consume_char(<>, rest) + + s in [:exponent, :fraction] and separator?(<>) -> raise Ex.UnfinishedTokenError, state.current - separator?(<>) -> + + separator?(<>) -> end_token(state, "", input) + true -> - raise Ex.UnexpectedInputError, <> + raise Ex.UnexpectedInputError, <> end end # Delimiters - defp _tokenize(state = %{state: :new}, <>) - when <> in ["{", "}", "[", "]", "(", ")"] do + defp _tokenize(state = %{state: :new}, <>) + when <> in ["{", "}", "[", "]", "(", ")"] do delim = <> token = token(delim_type(delim), delim) end_token(state, token, delim, rest) end - defp _tokenize(state = %{state: :new}, <<"#\{" :: utf8, rest :: binary>>) do + + defp _tokenize(state = %{state: :new}, <<"#\{"::utf8, rest::binary>>) do token = token(:set_open, "#\{") end_token(state, token, "#\{", rest) end # Whitespace - defp _tokenize(state = %{state: :new}, <>) - when <> in [" ", "\t", "\r", "\n", ","] do + defp _tokenize(state = %{state: :new}, <>) + when <> in [" ", "\t", "\r", "\n", ","] do skip_char(state, <>, rest) end # Discard - defp _tokenize(state = %{state: :new}, <<"#_" :: utf8, rest :: binary>>) do + defp _tokenize(state = %{state: :new}, <<"#_"::utf8, rest::binary>>) do token = token(:discard, "#_") end_token(state, token, "#_", rest) end # Tags - defp _tokenize(state = %{state: :new}, <<"#" :: utf8, rest :: binary>>) do + defp _tokenize(state = %{state: :new}, <<"#"::utf8, rest::binary>>) do token = token(:tag, "") start_token(state, :symbol, token, "#", rest) end # Symbol, Integer or Invalid input - defp _tokenize(state = %{state: :new}, <>) do + defp _tokenize(state = %{state: :new}, <>) do cond do - alpha?(<>) -> - token = token(:symbol, <>) - start_token(state, :symbol, token, <>, rest) - digit?(<>) -> - token = token(:integer, <>) - start_token(state, :number, token, <>, rest) + alpha?(<>) -> + token = token(:symbol, <>) + start_token(state, :symbol, token, <>, rest) + + digit?(<>) -> + token = token(:integer, <>) + start_token(state, :number, token, <>, rest) + true -> - raise Ex.UnexpectedInputError, <> + raise Ex.UnexpectedInputError, <> end end # Unexpected Input - defp _tokenize(_, <>) do - raise Ex.UnexpectedInputError, <> + defp _tokenize(_, <>) do + raise Ex.UnexpectedInputError, <> end ############################################################################## @@ -271,16 +298,19 @@ defmodule Eden.Lexer do defp update_location(state, "") do state end - defp update_location(state, <<"\n" :: utf8, rest :: binary>>) do + + defp update_location(state, <<"\n"::utf8, rest::binary>>) do state |> put_in([:location, :line], state.location.line + 1) |> put_in([:location, :col], 0) |> update_location(rest) end - defp update_location(state, <<"\r" :: utf8, rest :: binary>>) do + + defp update_location(state, <<"\r"::utf8, rest::binary>>) do update_location(state, rest) end - defp update_location(state, <<_ :: utf8, rest :: binary>>) do + + defp update_location(state, <<_::utf8, rest::binary>>) do state |> put_in([:location, :col], state.location.col + 1) |> update_location(rest) @@ -291,11 +321,13 @@ defmodule Eden.Lexer do end defp set_token(state, token) do - token = if state.opts[:location] do - Map.put(token, :location, state.location) - else - token - end + token = + if state.opts[:location] do + Map.put(token, :location, state.location) + else + token + end + Map.put(state, :current, token) end @@ -309,15 +341,14 @@ defmodule Eden.Lexer do end defp reset(state) do - %{state | - state: :new, - current: nil} + %{state | state: :new, current: nil} end defp valid?(%{state: state, current: current}) - when state in [:string, :exponent, :character, :fraction] do + when state in [:string, :exponent, :character, :fraction] do raise Ex.UnfinishedTokenError, current end + defp valid?(state) do state end @@ -325,10 +356,12 @@ defmodule Eden.Lexer do defp add_token(state, nil) do state end + defp add_token(state, token) do if token.type == :keyword and token.value == "" do raise Ex.UnfinishedTokenError, token end + %{state | tokens: [token | state.tokens]} end diff --git a/lib/eden/parser.ex b/lib/eden/parser.ex index 99b708b..762b2d3 100644 --- a/lib/eden/parser.ex +++ b/lib/eden/parser.ex @@ -51,9 +51,11 @@ defmodule Eden.Parser do tokens = Lexer.tokenize(input, opts) state = %{tokens: tokens, node: new_node(:root)} state = exprs(state) + if not Enum.empty?(state.tokens) do raise Ex.UnexpectedTokenError, List.first(state.tokens) end + Node.reverse_children(state.node) end @@ -71,27 +73,29 @@ defmodule Eden.Parser do defp expr(%{tokens: []}) do nil end + defp expr(state) do - terminal(state, :nil) - || terminal(state, :false) - || terminal(state, :true) - || terminal(state, :symbol) - || terminal(state, :keyword) - || terminal(state, :integer) - || terminal(state, :float) - || terminal(state, :string) - || terminal(state, :character) - || map_begin(state) - || vector_begin(state) - || list_begin(state) - || set_begin(state) - || tag(state) - || discard(state) - || comment(state) + terminal(state, nil) || + terminal(state, false) || + terminal(state, true) || + terminal(state, :symbol) || + terminal(state, :keyword) || + terminal(state, :integer) || + terminal(state, :float) || + terminal(state, :string) || + terminal(state, :character) || + map_begin(state) || + vector_begin(state) || + list_begin(state) || + set_begin(state) || + tag(state) || + discard(state) || + comment(state) end defp terminal(state, type) do {state, token} = pop_token(state) + if token?(token, type) do node = new_node(type, token, true) add_node(state, node) @@ -102,6 +106,7 @@ defmodule Eden.Parser do defp map_begin(state) do {state, token} = pop_token(state) + if token?(token, :curly_open) do state |> set_node(new_node(:map, token)) @@ -132,9 +137,11 @@ defmodule Eden.Parser do defp map_end(state) do {state, token} = pop_token(state) + if not token?(token, :curly_close) do raise Ex.UnbalancedDelimiterError, state.node end + state end @@ -142,6 +149,7 @@ defmodule Eden.Parser do defp vector_begin(state) do {state, token} = pop_token(state) + if token?(token, :bracket_open) do state |> set_node(new_node(:vector, token)) @@ -153,9 +161,11 @@ defmodule Eden.Parser do defp vector_end(state) do {state, token} = pop_token(state) + if not token?(token, :bracket_close) do raise Ex.UnbalancedDelimiterError, state.node end + state end @@ -163,6 +173,7 @@ defmodule Eden.Parser do defp list_begin(state) do {state, token} = pop_token(state) + if token?(token, :paren_open) do state |> set_node(new_node(:list, token)) @@ -174,9 +185,11 @@ defmodule Eden.Parser do defp list_end(state) do {state, token} = pop_token(state) + if not token?(token, :paren_close) do raise Ex.UnbalancedDelimiterError, state.node end + state end @@ -184,6 +197,7 @@ defmodule Eden.Parser do defp set_begin(state) do {state, token} = pop_token(state) + if token?(token, :set_open) do state |> set_node(new_node(:set, token)) @@ -195,9 +209,11 @@ defmodule Eden.Parser do defp set_end(state) do {state, token} = pop_token(state) + if not token?(token, :curly_close) do raise Ex.UnbalancedDelimiterError, state.node end + state end @@ -205,8 +221,10 @@ defmodule Eden.Parser do defp tag(state) do {state, token} = pop_token(state) + if token?(token, :tag) do node = new_node(:tag, token, true) + state |> set_node(node) |> expr @@ -219,6 +237,7 @@ defmodule Eden.Parser do defp discard(state) do {state, token} = pop_token(state) + if token?(token, :discard) do state |> set_node(new_node(:discard, token)) @@ -232,6 +251,7 @@ defmodule Eden.Parser do defp comment(state) do {state, token} = pop_token(state) + if token?(token, :comment) do state end @@ -244,16 +264,17 @@ defmodule Eden.Parser do ## Node defp new_node(type, token \\ nil, use_value? \\ false) do - location = if token && Map.has_key?(token, :location) do - token.location - end - value = if token && use_value? do - token.value - end - %Node{type: type, - location: location, - value: value, - children: []} + location = + if token && Map.has_key?(token, :location) do + token.location + end + + value = + if token && use_value? do + token.value + end + + %Node{type: type, location: location, value: value, children: []} end defp add_node(state, node) do @@ -269,6 +290,7 @@ defmodule Eden.Parser do defp restore_node(new_state, old_state, add_child? \\ true) do child_node = Node.reverse_children(new_state.node) old_state = Map.put(new_state, :node, old_state.node) + if add_child? do add_node(old_state, child_node) else @@ -279,11 +301,10 @@ defmodule Eden.Parser do ## Token defp token?(nil, _), do: false - defp token?(token, type), do: (token.type == type) + defp token?(token, type), do: token.type == type defp pop_token(state) do - {update_in(state, [:tokens], &tail/1), - List.first(state.tokens)} + {update_in(state, [:tokens], &tail/1), List.first(state.tokens)} end ## Utils @@ -297,7 +318,7 @@ defmodule Eden.Parser do end defp raise_when(x, ex, msg, pred?) do - if(pred?.(x), do: (raise ex, msg), else: x) + if(pred?.(x), do: raise(ex, msg), else: x) end defp tail([]), do: [] diff --git a/lib/eden/parser/node.ex b/lib/eden/parser/node.ex index 4a471f1..2e4cfbd 100644 --- a/lib/eden/parser/node.ex +++ b/lib/eden/parser/node.ex @@ -26,20 +26,22 @@ defmodule Eden.Parser.Node do value_str = if node.value, do: "\"" <> node.value <> "\" ", else: "" loc = node.location + location_str = - if loc do - concat ["(", Integer.to_string(loc.line), ",", - Integer.to_string(loc.col), ")"] - else - "" - end + if loc do + concat(["(", Integer.to_string(loc.line), ",", Integer.to_string(loc.col), ")"]) + else + "" + end level = Map.get(opts, :level, 0) opts = Map.put(opts, :level, level + 2) padding = String.duplicate(" ", level) - concat [padding, "",type_str , " ", value_str, location_str, "\n"] - ++ Enum.map(node.children, fn x -> to_doc(x, opts)end ) + concat( + [padding, "", type_str, " ", value_str, location_str, "\n"] ++ + Enum.map(node.children, fn x -> to_doc(x, opts) end) + ) end end diff --git a/test/eden/lexer_test.exs b/test/eden/lexer_test.exs index 71aee2f..4c5b66e 100644 --- a/test/eden/lexer_test.exs +++ b/test/eden/lexer_test.exs @@ -14,29 +14,31 @@ defmodule Eden.LexerTest do end test "nil, true, false" do - assert tokenize("nil") == [token(:nil, "nil")] - assert tokenize(" nil ") == [token(:nil, "nil")] - assert tokenize("true") == [token(:true, "true")] - assert tokenize(" true ") == [token(:true, "true")] - assert tokenize("false") == [token(:false, "false")] - assert tokenize(" false ") == [token(:false, "false")] - - assert List.first(tokenize(" nil{ ")) == token(:nil, "nil") + assert tokenize("nil") == [token(nil, "nil")] + assert tokenize(" nil ") == [token(nil, "nil")] + assert tokenize("true") == [token(true, "true")] + assert tokenize(" true ") == [token(true, "true")] + assert tokenize("false") == [token(false, "false")] + assert tokenize(" false ") == [token(false, "false")] + + assert List.first(tokenize(" nil{ ")) == token(nil, "nil") assert List.first(tokenize(" nilo ")) == token(:symbol, "nilo") - assert List.first(tokenize(" true} ")) == token(:true, "true") + assert List.first(tokenize(" true} ")) == token(true, "true") assert List.first(tokenize(" truedetective ")) == token(:symbol, "truedetective") - assert List.first(tokenize(" false{ ")) == token(:false, "false") + assert List.first(tokenize(" false{ ")) == token(false, "false") assert List.first(tokenize(" falsette ")) == token(:symbol, "falsette") end test "String" do assert tokenize(" \"this is a string\" ") == [token(:string, "this is a string")] assert tokenize(" \"this is a \\\" string\" ") == [token(:string, "this is a \" string")] + assert_raise Ex.UnfinishedTokenError, fn -> tokenize(" \"this is an unfinished string ") end + assert_raise Ex.UnfinishedTokenError, fn -> tokenize(" \"this is an unfinished string\\\"") end @@ -70,6 +72,7 @@ defmodule Eden.LexerTest do assert_raise Ex.UnexpectedInputError, fn -> tokenize(" question?\\") end + assert_raise Ex.UnexpectedInputError, fn -> tokenize("ns/name/ss") end @@ -105,86 +108,109 @@ defmodule Eden.LexerTest do assert_raise Ex.UnexpectedInputError, fn -> assert tokenize("1234.a") end + assert_raise Ex.UnexpectedInputError, fn -> assert tokenize("1234.121a ") end + assert_raise Ex.UnexpectedInputError, fn -> assert tokenize("1234E0a1") end + assert_raise Ex.UnfinishedTokenError, fn -> tokenize("1234E") end + assert_raise Ex.UnfinishedTokenError, fn -> tokenize("1234.") end + assert_raise Ex.UnfinishedTokenError, fn -> tokenize("1234. :kw") end end test "Delimiters" do - assert tokenize("{[#\{}]} )()") == [token(:curly_open, "{"), - token(:bracket_open, "["), - token(:set_open, "#\{"), - token(:curly_close, "}"), - token(:bracket_close, "]"), - token(:curly_close, "}"), - token(:paren_close, ")"), - token(:paren_open, "("), - token(:paren_close, ")")] + assert tokenize("{[#\{}]} )()") == [ + token(:curly_open, "{"), + token(:bracket_open, "["), + token(:set_open, "#\{"), + token(:curly_close, "}"), + token(:bracket_close, "]"), + token(:curly_close, "}"), + token(:paren_close, ")"), + token(:paren_open, "("), + token(:paren_close, ")") + ] end test "Discard" do assert tokenize("#_ ") == [token(:discard, "#_")] - assert tokenize("1 #_ :kw") == [token(:integer, "1"), - token(:discard, "#_"), - token(:keyword, "kw")] + + assert tokenize("1 #_ :kw") == [ + token(:integer, "1"), + token(:discard, "#_"), + token(:keyword, "kw") + ] end test "Tag" do assert tokenize("#ns/name") == [token(:tag, "ns/name")] assert tokenize("#whatever") == [token(:tag, "whatever")] - assert tokenize(" #whatever :kw") == [token(:tag, "whatever"), - token(:keyword, "kw")] + assert tokenize(" #whatever :kw") == [token(:tag, "whatever"), token(:keyword, "kw")] end test "Comment" do - assert tokenize("1 ;; hello") == [token(:integer, "1"), - token(:comment, " hello")] - assert tokenize("1 ;; hello\n\r") == [token(:integer, "1"), - token(:comment, " hello")] - assert tokenize("1 ;; hello\n\r bla") == [token(:integer, "1"), - token(:comment, " hello"), - token(:symbol, "bla")] + assert tokenize("1 ;; hello") == [token(:integer, "1"), token(:comment, " hello")] + assert tokenize("1 ;; hello\n\r") == [token(:integer, "1"), token(:comment, " hello")] + + assert tokenize("1 ;; hello\n\r bla") == [ + token(:integer, "1"), + token(:comment, " hello"), + token(:symbol, "bla") + ] end test "Line and Column Information" do - tokens = [token(:integer, "1", %{line: 1, col: 0}), - token(:comment, " hello", %{line: 1, col: 2})] + tokens = [ + token(:integer, "1", %{line: 1, col: 0}), + token(:comment, " hello", %{line: 1, col: 2}) + ] + assert tokenize("1 ;; hello", location: true) == tokens assert tokenize("1 ;; hello\r\n", location: true) == tokens - tokens = [token(:integer, "1", %{line: 1, col: 0}), - token(:comment, " hello", %{line: 1, col: 2}), - token(:symbol, "bla", %{line: 2, col: 1})] + tokens = [ + token(:integer, "1", %{line: 1, col: 0}), + token(:comment, " hello", %{line: 1, col: 2}), + token(:symbol, "bla", %{line: 2, col: 1}) + ] + assert tokenize("1 ;; hello\r\n bla", location: true) == tokens assert tokenize("1 ;; hello\n\r bla", location: true) == tokens - tokens = [token(:integer, "1", %{line: 1, col: 0}), - token(:string, "hello \n world", %{line: 2, col: 0}), - token(:keyword, "kw", %{line: 3, col: 8})] + tokens = [ + token(:integer, "1", %{line: 1, col: 0}), + token(:string, "hello \n world", %{line: 2, col: 0}), + token(:keyword, "kw", %{line: 3, col: 8}) + ] + assert tokenize("1 \n\"hello \n world\" :kw ", location: true) == tokens - tokens = [token(:integer, "1", %{line: 1, col: 0}), - token(:string, "hello \n \" world", %{line: 2, col: 0}), - token(:keyword, "kw", %{line: 3, col: 1})] + tokens = [ + token(:integer, "1", %{line: 1, col: 0}), + token(:string, "hello \n \" world", %{line: 2, col: 0}), + token(:keyword, "kw", %{line: 3, col: 1}) + ] + assert tokenize("1 \n\"hello \\n \\\" world\"\n :kw ", location: true) == tokens end defp token(type, value, location \\ nil) do token = %Lexer.Token{type: type, value: value} + if location, - do: Map.put(token, :location, location), - else: token + do: Map.put(token, :location, location), + else: token end end diff --git a/test/eden/parser_test.exs b/test/eden/parser_test.exs index 9117092..7ef2011 100644 --- a/test/eden/parser_test.exs +++ b/test/eden/parser_test.exs @@ -10,9 +10,7 @@ defmodule Eden.ParserTest do end test "Literals" do - root = node(:root, nil, [node(:nil, "nil"), - node(:true, "true"), - node(:false, "false")]) + root = node(:root, nil, [node(nil, "nil"), node(true, "true"), node(false, "false")]) assert parse("nil true false") == root root = node(:root, nil, [node(:integer, "1")]) @@ -25,25 +23,20 @@ defmodule Eden.ParserTest do root = node(:root, nil, [node(:float, "1.33")]) assert parse("1.33") == root - root = node(:root, nil, - [node(:symbol, "nilo"), - node(:symbol, "truthy"), - node(:symbol, "falsey")]) + root = + node(:root, nil, [node(:symbol, "nilo"), node(:symbol, "truthy"), node(:symbol, "falsey")]) + assert parse("nilo truthy falsey") == root - root = node(:root, nil, - [node(:keyword, "nilo"), - node(:string, "truthy"), - node(:keyword, "falsey")]) + root = + node(:root, nil, [node(:keyword, "nilo"), node(:string, "truthy"), node(:keyword, "falsey")]) + assert parse(":nilo \"truthy\" :falsey") == root - root = node(:root, nil, - [node(:character, "h"), - node(:character, "i")]) + root = node(:root, nil, [node(:character, "h"), node(:character, "i")]) assert parse("\\h\\i") == root - root = node(:root, nil, - [node(:string, "Thaïlande")]) + root = node(:root, nil, [node(:string, "Thaïlande")]) assert parse("\"Thaïlande\"") == root assert_raise Ex.UnfinishedTokenError, fn -> @@ -59,25 +52,26 @@ defmodule Eden.ParserTest do root = node(:root, nil, [node(:map, nil)]) assert parse("{}") == root - root = node(:root, nil, - [node(:map, nil, - [node(:keyword, "name"), - node(:string, "John")])]) + root = node(:root, nil, [node(:map, nil, [node(:keyword, "name"), node(:string, "John")])]) assert parse("{:name \"John\"}") == root - root = node(:root, nil, - [node(:map, nil, - [node(:keyword, "name"), - node(:string, "John"), - node(:keyword, "age"), - node(:integer, "120")])]) + root = + node(:root, nil, [ + node(:map, nil, [ + node(:keyword, "name"), + node(:string, "John"), + node(:keyword, "age"), + node(:integer, "120") + ]) + ]) + assert parse("{:name \"John\", :age 120}") == root assert_raise Ex.OddExpressionCountError, fn -> parse("{nil true false}") end - assert_raise Ex.UnbalancedDelimiterError, fn -> + assert_raise Ex.UnbalancedDelimiterError, fn -> parse("{nil true ") end end @@ -86,17 +80,14 @@ defmodule Eden.ParserTest do root = node(:root, nil, [node(:vector, nil)]) assert parse("[]") == root - root = node(:root, nil, - [node(:vector, nil, - [node(:keyword, "name"), - node(:string, "John")])]) + root = node(:root, nil, [node(:vector, nil, [node(:keyword, "name"), node(:string, "John")])]) assert parse("[:name, \"John\"]") == root - root = node(:root, nil, - [node(:vector, nil, - [node(:keyword, "name"), - node(:string, "John"), - node(:integer, "120")])]) + root = + node(:root, nil, [ + node(:vector, nil, [node(:keyword, "name"), node(:string, "John"), node(:integer, "120")]) + ]) + assert parse("[:name, \"John\", 120]") == root assert_raise Ex.UnbalancedDelimiterError, fn -> @@ -108,17 +99,14 @@ defmodule Eden.ParserTest do root = node(:root, nil, [node(:list, nil)]) assert parse("()") == root - root = node(:root, nil, - [node(:list, nil, - [node(:keyword, "name"), - node(:string, "John")])]) + root = node(:root, nil, [node(:list, nil, [node(:keyword, "name"), node(:string, "John")])]) assert parse("(:name, \"John\")") == root - root = node(:root, nil, - [node(:list, nil, - [node(:keyword, "name"), - node(:string, "John"), - node(:integer, "120")])]) + root = + node(:root, nil, [ + node(:list, nil, [node(:keyword, "name"), node(:string, "John"), node(:integer, "120")]) + ]) + assert parse("(:name, \"John\", 120)") == root assert_raise Ex.UnbalancedDelimiterError, fn -> @@ -130,17 +118,14 @@ defmodule Eden.ParserTest do root = node(:root, nil, [node(:set, nil)]) assert parse("#\{}") == root - root = node(:root, nil, - [node(:set, nil, - [node(:keyword, "name"), - node(:string, "John")])]) + root = node(:root, nil, [node(:set, nil, [node(:keyword, "name"), node(:string, "John")])]) assert parse("#\{:name, \"John\"}") == root - root = node(:root, nil, - [node(:set, nil, - [node(:keyword, "name"), - node(:string, "John"), - node(:integer, "120")])]) + root = + node(:root, nil, [ + node(:set, nil, [node(:keyword, "name"), node(:string, "John"), node(:integer, "120")]) + ]) + assert parse("#\{:name, \"John\", 120}") == root assert_raise Ex.UnbalancedDelimiterError, fn -> @@ -149,16 +134,14 @@ defmodule Eden.ParserTest do end test "Tag" do - root = node(:root, nil, - [node(:tag, "inst", - [node(:string, "1985-04-12T23:20:50.52Z")])]) + root = node(:root, nil, [node(:tag, "inst", [node(:string, "1985-04-12T23:20:50.52Z")])]) assert parse("#inst \"1985-04-12T23:20:50.52Z\"") == root - root = node(:root, nil, - [node(:tag, "some/tag", - [node(:map, nil, - [node(:keyword, "a"), - node(:integer, "1")])])]) + root = + node(:root, nil, [ + node(:tag, "some/tag", [node(:map, nil, [node(:keyword, "a"), node(:integer, "1")])]) + ]) + assert parse("#some/tag {:a 1}") == root assert_raise Ex.IncompleteTagError, fn -> @@ -167,28 +150,18 @@ defmodule Eden.ParserTest do end test "Discard" do - root = node(:root, nil, - [node(:set, nil, - [node(:keyword, "name")])]) + root = node(:root, nil, [node(:set, nil, [node(:keyword, "name")])]) assert parse("#\{:name, #_ \"John\"}") == root - root = node(:root, nil, - [node(:set, nil, - [node(:string, "John"), - node(:integer, "120")])]) + root = node(:root, nil, [node(:set, nil, [node(:string, "John"), node(:integer, "120")])]) assert parse("#\{#_:name, \"John\", 120}") == root end test "Comment" do - root = node(:root, nil, - [node(:set, nil, - [node(:keyword, "name")])]) + root = node(:root, nil, [node(:set, nil, [node(:keyword, "name")])]) assert parse("#\{:name, \n ;; \"John\" \n}") == root - root = node(:root, nil, - [node(:set, nil, - [node(:string, "John"), - node(:integer, "120")])]) + root = node(:root, nil, [node(:set, nil, [node(:string, "John"), node(:integer, "120")])]) assert parse("#\{\n;; :name, \n \"John\", 120}") == root end @@ -199,9 +172,6 @@ defmodule Eden.ParserTest do end defp node(type, value, children \\ []) do - %Parser.Node{type: type, - value: value, - children: children, - location: nil} + %Parser.Node{type: type, value: value, children: children, location: nil} end end diff --git a/test/eden_test.exs b/test/eden_test.exs index 98d6c16..8a80633 100644 --- a/test/eden_test.exs +++ b/test/eden_test.exs @@ -16,7 +16,6 @@ defmodule EdenTest do assert_raise Ex.EmptyInputError, fn -> decode!("") end - end test "Decode Literals" do @@ -32,7 +31,7 @@ defmodule EdenTest do assert decode!("\\z") == %Character{char: "z"} assert decode!("a-symbol") == %Symbol{name: "a-symbol"} - assert decode!(":the-keyword") == :'the-keyword' + assert decode!(":the-keyword") == :"the-keyword" assert decode!("42") == 42 assert decode!("42N") == 42 @@ -64,14 +63,17 @@ defmodule EdenTest do end test "Decode Set" do - set = Enum.into([:name, "John", :age, 42], MapSet.new) + set = Enum.into([:name, "John", :age, 42], MapSet.new()) assert decode!("#\{:name \"John\" :age 42}") == set end test "Decode Tag" do date = Timex.parse!("1985-04-12T23:20:50.52Z", "{RFC3339z}") assert decode!("#inst \"1985-04-12T23:20:50.52Z\"") == date - assert decode!("#uuid \"f81d4fae-7dec-11d0-a765-00a0c91e6bf6\"") == %UUID{value: "f81d4fae-7dec-11d0-a765-00a0c91e6bf6"} + + assert decode!("#uuid \"f81d4fae-7dec-11d0-a765-00a0c91e6bf6\"") == %UUID{ + value: "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" + } assert decode!("#custom/tag (1 2 3)") == %Tag{name: "custom/tag", value: [1, 2, 3]} handlers = %{"custom/tag" => &custom_tag_handler/1} @@ -100,7 +102,7 @@ defmodule EdenTest do assert encode!(42.0e3) == "4.2e4" assert encode!(42.0e-3) == "0.042" assert encode!(42.0e-1) == "4.2" - assert encode!(42.01E+1) == "420.1" + assert encode!(42.01e+1) == "420.1" end test "Encode List" do @@ -118,7 +120,7 @@ defmodule EdenTest do end test "Encode Set" do - set = Enum.into([:name, "John", :age, 42], MapSet.new) + set = Enum.into([:name, "John", :age, 42], MapSet.new()) assert encode!(set) == "#\{42, :age, :name, \"John\"}" end @@ -153,7 +155,6 @@ defmodule EdenTest do assert e.protocol == Eden.Encode assert e.value == self() end - end defp custom_tag_handler(value) when is_list(value) do diff --git a/test/test_helper.exs b/test/test_helper.exs index 7d746e0..818ec61 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,4 +1,4 @@ require Logger -Logger.configure([level: :error]) +Logger.configure(level: :error) ExUnit.start() From bef731a8d4d60d91f8a2fca863df1b87cdf1f20e Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 13:09:59 +0100 Subject: [PATCH 2/6] update deps --- mix.exs | 3 +-- mix.lock | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/mix.exs b/mix.exs index e5e362e..08c8dd1 100644 --- a/mix.exs +++ b/mix.exs @@ -25,8 +25,7 @@ defmodule Eden.Mixfile do defp deps do [ {:elixir_array, "~> 2.1.0"}, - {:timex, "~> 3.1"}, - {:exreloader, github: "jfacorro/exreloader", tag: "master", only: :dev}, + {:timex, "~> 3.7"}, {:ex_doc, "~> 0.23", only: :dev}, {:earmark, ">= 0.0.0", only: :dev} ] diff --git a/mix.lock b/mix.lock index ce0e898..320b785 100644 --- a/mix.lock +++ b/mix.lock @@ -1,22 +1,22 @@ %{ - "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, + "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, "elixir_array": {:hex, :elixir_array, "2.1.0", "580293f82afdd63be880d69d3b1cc829fe6994f4b1442ee809ccb7199ee7fa13", [:mix], [], "hexpm", "9425e43cf7df7eaea6ac03e25e317835b52212ec374d71029a8fd02c1c32648c"}, "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"}, - "exreloader": {:git, "https://github.com/jfacorro/exreloader.git", "c54f341284597d9efbeb0fe05001ff442a87074c", [tag: "master"]}, - "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"}, - "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, - "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, + "expo": {:hex, :expo, "0.5.2", "beba786aab8e3c5431813d7a44b828e7b922bfa431d6bfbada0904535342efe2", [:mix], [], "hexpm", "8c9bfa06ca017c9cb4020fabe980bc7fdb1aaec059fd004c2ab3bff03b1c599c"}, + "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, + "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, + "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, - "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, - "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, + "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, + "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } From 0e656492133a9ee43523ea7b41bc3ebc6fffaef2 Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 13:10:45 +0100 Subject: [PATCH 3/6] allow | in symbols & keywords --- lib/eden/lexer.ex | 2 +- test/eden/lexer_test.exs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/eden/lexer.ex b/lib/eden/lexer.ex index 088d24d..b44951b 100644 --- a/lib/eden/lexer.ex +++ b/lib/eden/lexer.ex @@ -382,7 +382,7 @@ defmodule Eden.Lexer do defp digit?(char), do: String.match?(char, ~r/[0-9]/) - defp symbol_char?(char), do: String.match?(char, ~r/[_?a-zA-Z0-9.*+!\-$%&=<>\#:]/) + defp symbol_char?(char), do: String.match?(char, ~r/[_?a-zA-Z0-9.*+!\-$%&=<>\#:|]/) defp whitespace?(char), do: String.match?(char, ~r/[\s,]/) diff --git a/test/eden/lexer_test.exs b/test/eden/lexer_test.exs index 4c5b66e..8846c8f 100644 --- a/test/eden/lexer_test.exs +++ b/test/eden/lexer_test.exs @@ -9,7 +9,7 @@ defmodule Eden.LexerTest do assert tokenize(" \n \t, \r") == [] assert_raise Ex.UnexpectedInputError, fn -> - tokenize(" \n \t, \r a| ,,,") + tokenize(" \n \t, \r / ,,,") end end @@ -55,6 +55,7 @@ defmodule Eden.LexerTest do assert tokenize(" :question? ") == [token(:keyword, "question?")] assert tokenize(":question?{") == [token(:keyword, "question?"), token(:curly_open, "{")] assert tokenize(":k?+._-!7><$&=*") == [token(:keyword, "k?+._-!7><$&=*")] + assert tokenize(":a-1|2") == [token(:keyword, "a-1|2")] assert_raise Ex.UnexpectedInputError, fn -> tokenize(" :question?\\") @@ -68,6 +69,7 @@ defmodule Eden.LexerTest do assert tokenize("question?{") == [token(:symbol, "question?"), token(:curly_open, "{")] assert tokenize("k?+._-!7><$&=*") == [token(:symbol, "k?+._-!7><$&=*")] assert tokenize("ns/name") == [token(:symbol, "ns/name")] + assert tokenize("a-1|2") == [token(:symbol, "a-1|2")] assert_raise Ex.UnexpectedInputError, fn -> tokenize(" question?\\") From fa473d8e60a7c74317047d58571c8e83b83dfd0c Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 13:11:34 +0100 Subject: [PATCH 4/6] support of namespaced maps decoding --- lib/eden/decode.ex | 29 +++++++++++++++++++++++++++++ lib/eden/lexer.ex | 6 ++++++ lib/eden/parser.ex | 20 ++++++++++++++++++++ test/eden_test.exs | 5 +++++ 4 files changed, 60 insertions(+) diff --git a/lib/eden/decode.ex b/lib/eden/decode.ex index 358735a..e16e35d 100644 --- a/lib/eden/decode.ex +++ b/lib/eden/decode.ex @@ -87,6 +87,17 @@ defmodule Eden.Decode do |> Enum.into(%{}) end + def decode(%Node{type: :ns_map, value: value, children: children} = node, opts) do + if Integer.is_odd(length(children)) do + raise Ex.OddExpressionCountError, node + end + + children + |> Enum.chunk_every(2) + |> Enum.map(fn [a, b] -> {decode_ns_map_key(a, value, opts), decode(b, opts)} end) + |> Enum.into(%{}) + end + def decode(%Node{type: :set, children: children}, opts) do children |> decode(opts) @@ -106,4 +117,22 @@ defmodule Eden.Decode do def decode(%Node{type: type}, _opts) do raise "Unrecognized node type: #{inspect(type)}" end + + defp decode_ns_map_key(%Node{type: :keyword, value: value}, ns, _opts) do + if not String.contains?(value, "/") do + (ns <> "/" <> value) |> String.to_atom() + else + case value do + "_/" <> kw -> + kw |> String.to_atom() + + _ -> + value |> String.to_atom() + end + end + end + + defp decode_ns_map_key(node, _ns, opts) do + decode(node, opts) + end end diff --git a/lib/eden/lexer.ex b/lib/eden/lexer.ex index b44951b..6d58241 100644 --- a/lib/eden/lexer.ex +++ b/lib/eden/lexer.ex @@ -229,6 +229,12 @@ defmodule Eden.Lexer do end_token(state, token, "#_", rest) end + # NS Maps + defp _tokenize(state = %{state: :new}, <<"#:"::utf8, rest::binary>>) do + token = token(:ns_map, "") + start_token(state, :symbol, token, "#", rest) + end + # Tags defp _tokenize(state = %{state: :new}, <<"#"::utf8, rest::binary>>) do token = token(:tag, "") diff --git a/lib/eden/parser.ex b/lib/eden/parser.ex index 762b2d3..2a92486 100644 --- a/lib/eden/parser.ex +++ b/lib/eden/parser.ex @@ -85,6 +85,7 @@ defmodule Eden.Parser do terminal(state, :string) || terminal(state, :character) || map_begin(state) || + ns_map_begin(state) || vector_begin(state) || list_begin(state) || set_begin(state) || @@ -145,6 +146,25 @@ defmodule Eden.Parser do state end + defp ns_map_begin(state) do + {state, token} = pop_token(state) + + if token?(token, :ns_map) do + node = new_node(:ns_map, token, true) + {state, token} = pop_token(state) + + if not token?(token, :curly_open) do + raise Ex.UnexpectedTokenError, token + end + + state + |> set_node(node) + |> pairs + |> map_end + |> restore_node(state) + end + end + ## Vector defp vector_begin(state) do diff --git a/test/eden_test.exs b/test/eden_test.exs index 8a80633..119ee6e 100644 --- a/test/eden_test.exs +++ b/test/eden_test.exs @@ -32,6 +32,7 @@ defmodule EdenTest do assert decode!("a-symbol") == %Symbol{name: "a-symbol"} assert decode!(":the-keyword") == :"the-keyword" + assert decode!(":ns/the-keyword") == :"ns/the-keyword" assert decode!("42") == 42 assert decode!("42N") == 42 @@ -62,6 +63,10 @@ defmodule EdenTest do end end + test "Decode Namespaced Map" do + assert decode!("#:ns{:foo 1, :bar/zoo 2 :_/baz 3}") == %{"ns/foo": 1, "bar/zoo": 2, baz: 3} + end + test "Decode Set" do set = Enum.into([:name, "John", :age, 42], MapSet.new()) assert decode!("#\{:name \"John\" :age 42}") == set From 33e1fd7eb12dff64f69fa937e6a3ed073f77bf7c Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 13:11:42 +0100 Subject: [PATCH 5/6] test fixes --- test/eden_test.exs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/eden_test.exs b/test/eden_test.exs index 119ee6e..92af7ed 100644 --- a/test/eden_test.exs +++ b/test/eden_test.exs @@ -104,7 +104,7 @@ defmodule EdenTest do assert encode!(42) == "42" assert encode!(42.0) == "42.0" - assert encode!(42.0e3) == "4.2e4" + assert encode!(42.0e3) == "42000.0" assert encode!(42.0e-3) == "0.042" assert encode!(42.0e-1) == "4.2" assert encode!(42.01e+1) == "420.1" @@ -121,12 +121,14 @@ defmodule EdenTest do test "Encode Map" do map = %{name: "John", age: 42} - assert encode!(map) == "{:age 42, :name \"John\"}" + assert encode!(map) == "{:name \"John\", :age 42}" end test "Encode Set" do - set = Enum.into([:name, "John", :age, 42], MapSet.new()) - assert encode!(set) == "#\{42, :age, :name, \"John\"}" + assert encode!(MapSet.new([:name])) == "#\{:name}" + + set = MapSet.new([:name, "John", :age, 42]) + assert encode!(set) |> decode!() == set end test "Encode Tag" do From f4ac1d57d070bed14c50c0dafc3c5c68b462d440 Mon Sep 17 00:00:00 2001 From: Andrey Rudenko Date: Fri, 22 Mar 2024 13:19:38 +0100 Subject: [PATCH 6/6] special support for :true, :false, :nil --- lib/eden/decode.ex | 4 ++++ lib/eden/encode.ex | 6 ++++++ test/eden_test.exs | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/lib/eden/decode.ex b/lib/eden/decode.ex index e16e35d..c19c055 100644 --- a/lib/eden/decode.ex +++ b/lib/eden/decode.ex @@ -38,6 +38,10 @@ defmodule Eden.Decode do %Symbol{name: value} end + def decode(%Node{type: :keyword, value: value}, _opts) when value in ["nil", "true", "false"] do + {:keyword, String.to_atom(value)} + end + def decode(%Node{type: :keyword, value: value}, _opts) do String.to_atom(value) end diff --git a/lib/eden/encode.ex b/lib/eden/encode.ex index 149693d..89a35ef 100644 --- a/lib/eden/encode.ex +++ b/lib/eden/encode.ex @@ -28,6 +28,12 @@ defimpl Encode, for: Atom do end end +defimpl Encode, for: Tuple do + def encode({:keyword, atom}) when atom in [nil, true, false] do + ":" <> Atom.to_string(atom) + end +end + defimpl Encode, for: Symbol do def encode(symbol) do symbol.name diff --git a/test/eden_test.exs b/test/eden_test.exs index 92af7ed..2abc18b 100644 --- a/test/eden_test.exs +++ b/test/eden_test.exs @@ -33,6 +33,8 @@ defmodule EdenTest do assert decode!("a-symbol") == %Symbol{name: "a-symbol"} assert decode!(":the-keyword") == :"the-keyword" assert decode!(":ns/the-keyword") == :"ns/the-keyword" + assert decode!(":true") == {:keyword, true} + assert decode!(":nil") == {:keyword, nil} assert decode!("42") == 42 assert decode!("42N") == 42 @@ -101,6 +103,8 @@ defmodule EdenTest do assert encode!(Symbol.new("a-symbol")) == "a-symbol" assert encode!(:"the-keyword") == ":the-keyword" + assert encode!({:keyword, true}) == ":true" + assert encode!(42) == "42" assert encode!(42.0) == "42.0"