From 167f06f1836ca8fc7072ac686207d8fc4b3fc9db Mon Sep 17 00:00:00 2001 From: Gio Lucas Torres Date: Mon, 19 Feb 2024 16:51:46 +0100 Subject: [PATCH 1/5] Yamel.Encoder protocol --- README.md | 5 +- lib/yamel.ex | 72 ++----- lib/yaml/encoder.ex | 289 +++++++++++++++++++++++++++++ mix.exs | 4 + test/support/test_encode_struct.ex | 13 ++ test/yamel/encoder_test.exs | 182 ++++++++++++++++++ test/yamel_test.exs | 1 - 7 files changed, 503 insertions(+), 63 deletions(-) create mode 100644 lib/yaml/encoder.ex create mode 100644 test/support/test_encode_struct.ex create mode 100644 test/yamel/encoder_test.exs diff --git a/README.md b/README.md index be77348..b5a197b 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,12 @@ zoo: - vee """ -Yamel.decode!(yaml_string) +Yamel.decode!(yaml_string) # equivalent to Yamel.decode!(yaml_string, keys: :string) => %{"foo" => "bar", "zoo" => ["caa", "boo", "vee"]} +Yamel.decode!(yaml_string, keys: :atom) +=> %{foo: "bar", zoo: ["caa", "boo", "vee"]} + Yamel.encode!(["caa", :boo, :"\"foo\""]) => "- caa\n- boo\n- \"foo\"\n\n" diff --git a/lib/yamel.ex b/lib/yamel.ex index 798bdf9..8c946f7 100644 --- a/lib/yamel.ex +++ b/lib/yamel.ex @@ -6,9 +6,7 @@ defmodule Yamel do @type t :: map() | list() @type yaml :: String.t() @type keys :: :atom | :string | :atoms | :strings - @type quotable_types :: :atom | :string | :number | :boolean @type decode_opts :: [keys: keys] | [] - @type encode_opts :: [quote: quotable_types] | [] @type parse_error :: YamlElixir.ParsingError.t() @doc ~S""" @@ -98,12 +96,13 @@ defmodule Yamel do "- foo\n- \"bar\"\n- 12.3\n- \"true\"\n\n" """ - @spec encode!(Yamel.t(), encode_opts()) :: yaml() - def encode!(map_or_list, opts \\ []) + @spec encode!(Yamel.t(), Yamel.Encoder.opts()) :: yaml() + def encode!(map_or_list_or_tuple, opts \\ []) - def encode!(map_or_list, opts) - when is_map(map_or_list) or is_list(map_or_list), - do: to_yaml!(map_or_list, opts) + def encode!(map_or_list_or_tuple, opts) + when is_map(map_or_list_or_tuple) or is_list(map_or_list_or_tuple) or + is_tuple(map_or_list_or_tuple), + do: to_yaml!(map_or_list_or_tuple, opts) def encode!(value, _opts), do: raise(ArgumentError, "Unsupported value: #{inspect(value)}") @@ -130,7 +129,7 @@ defmodule Yamel do {:ok, "- foo\n- \"bar\"\n- 12.3\n- \"true\"\n\n"} """ - @spec encode(Yamel.t(), encode_opts()) :: {:ok, yaml()} | {:error, reason :: String.t()} + @spec encode(Yamel.t(), Yamel.Encoder.opts()) :: {:ok, yaml()} | {:error, reason :: String.t()} def encode(map_or_list, opts \\ []) def encode(map_or_list, opts) when is_map(map_or_list) or is_list(map_or_list), @@ -166,68 +165,19 @@ defmodule Yamel do defp to_yaml!(map_or_list, opts) defp to_yaml!(map_or_list, opts) do - opts_map = + options = opts |> Enum.map(fn opt when is_tuple(opt) -> opt opt -> {opt, true} end) |> Map.new() - |> Map.update(:indentation, "", & &1) |> Map.update(:quote, [], & &1) + |> Map.update(:indent_size, 2, & &1) + |> Map.update(:node_level, 0, & &1) map_or_list - |> serialize(opts_map) - |> Enum.join() + |> Yamel.Encoder.encode(options) |> Kernel.<>("\n") end - - defp serialize({key, value}, %{indentation: indentation} = opts) - when is_map(value) or is_list(value), - do: - "#{indentation}#{key}:\n#{ - serialize(value, Map.put(opts, :indentation, "#{indentation} ")) - }" - - defp serialize({key, value}, %{indentation: indentation} = opts), - do: "#{indentation}#{key}: #{serialize(value, opts)}" - - defp serialize(bitstring, %{quote: quoted} = _opts) when is_bitstring(bitstring) do - if Enum.member?(quoted, :string), - do: "\"#{bitstring}\"\n", - else: "#{bitstring}\n" - end - - defp serialize(number, %{quote: quoted} = _opts) when is_number(number) do - if Enum.member?(quoted, :number), - do: "\"#{number}\"\n", - else: "#{number}\n" - end - - defp serialize(boolean, %{quote: quoted} = _opts) when is_boolean(boolean) do - if Enum.member?(quoted, :boolean), - do: "\"#{boolean}\"\n", - else: "#{boolean}\n" - end - - defp serialize(atom, %{quote: quoted} = _opts) when is_atom(atom) do - if Enum.member?(quoted, :atom), - do: "\"#{atom}\"\n", - else: "#{atom}\n" - end - - defp serialize(map, opts) - when is_map(map), - do: Enum.map(map, &serialize(&1, opts)) - - defp serialize(list_or_tuple, %{indentation: indentation} = opts) - when is_list(list_or_tuple) do - Enum.map(list_or_tuple, fn - value when is_list(value) or is_map(value) or is_tuple(value) -> - "#{indentation}-\n#{serialize(value, Map.put(opts, :indentation, "#{indentation} "))}" - - value -> - "#{indentation}- #{serialize(value, opts)}" - end) - end end diff --git a/lib/yaml/encoder.ex b/lib/yaml/encoder.ex new file mode 100644 index 0000000..b1e3563 --- /dev/null +++ b/lib/yaml/encoder.ex @@ -0,0 +1,289 @@ +defmodule Yamel.EncodeError do + @type t :: %__MODULE__{message: String.t(), value: any} + + defexception message: nil, value: nil + + def message(%{message: nil, value: value}) do + "unable to encode value: #{inspect(value)}" + end + + def message(%{message: message}) do + message + end +end + +defmodule Yamel.Encoder.Helper do + defguard is_scalar(value) + when is_atom(value) or + is_binary(value) or + is_bitstring(value) or + is_boolean(value) or + is_nil(value) or + is_number(value) + + # https://yaml.org/spec/1.2.2/#61-indentation-spaces + @spec calculate_indentation(Yamel.Encoder.opts()) :: String.t() + def calculate_indentation(%{node_level: level, indent_size: indent_size}), + do: String.pad_trailing("", level * indent_size, " ") + + # when 'value' is a complex type + @spec serialize({key :: any(), value :: any()}, Yamel.Encoder.opts()) :: String.t() + def serialize({key, value}, opts) when is_map(value) or is_list(value) do + indent = calculate_indentation(opts) + opts = Map.update(opts, :node_level, 0, &(&1 + 1)) + "#{indent}#{key}:\n#{Yamel.Encoder.encode(value, opts)}" + end + + # when 'value' is an scalar + def serialize({key, value}, opts) do + indent = calculate_indentation(opts) + opts = Map.update(opts, :node_level, 0, &(&1 + 1)) + "#{indent}#{key}: #{Yamel.Encoder.encode(value, opts)}" + end +end + +defmodule Yamel.Encode do + @moduledoc false + + alias Yamel.EncodeError + + defmacro __using__(_) do + quote do + alias Yamel.EncodeError + alias String.Chars + + @compile {:inline, encode_name: 1} + + # Fast path encoding string keys + defp encode_name(value) when is_binary(value) do + value + end + + defp encode_name(value) do + case Chars.impl_for(value) do + nil -> + raise EncodeError, + value: value, + message: "expected a String.Chars encodable value, got: #{inspect(value)}" + + impl -> + impl.to_string(value) + end + end + end + end +end + +defprotocol Yamel.Encoder do + @fallback_to_any true + + @type quotable_type :: + :atom + | :string + | :number + | :integer + | :float + | :boolean + # quotes all scalars + | true + + @typedoc """ + `:node_level`: along with `:indent_size` calculates the indentation. 0 - root level; 1 - 2 spaces indent; 3 - 4 spaces indent + `:indent_size`: multiplier for indentation. Default 2 + `:quote`: which scalar types to be surrounded by quotes + `:quote_type`: use single quotes (' - :single) or double quotes (" - double) + `:empty_value`: how to deal with empty ("") values: leave blank (:blank) or use quotes (:quoted - default) + """ + @type opts :: %{ + required(:node_level) => non_neg_integer(), + required(:indent_size) => pos_integer(), + optional(:quote) => quotable_type() + } + + @spec encode(t, opts) :: iodata + def encode(value, opts \\ %{node_level: 0, indent_size: 2}) +end + +defimpl Yamel.Encoder, for: Atom do + def encode(nil, opts) do + types = opts[:quote] || [] + + if true in types or :atom in types, + do: "\"null\"\n", + else: "null\n" + end + + def encode(false, opts) do + types = opts[:quote] || [] + + if true in types or :boolean in types, + do: "\"false\"\n", + else: "false\n" + end + + def encode(true, opts) do + types = opts[:quote] || [] + + if true in types or :boolean in types, + do: "\"true\"\n", + else: "true\n" + end + + def encode(value, opts) do + types = opts[:quote] || [] + + if true in types or :atom in types, + do: "\"#{value}\"\n", + else: "#{value}\n" + end +end + +defimpl Yamel.Encoder, for: BitString do + use Bitwise + + def encode("", %{empty: :blank}), do: "" + def encode("", %{quote_type: :single}), do: "''" + def encode("", _opts), do: "\"\"" + + def encode(bitstring, opts) do + types = opts[:quote] || [] + + if true in types or :string in types, + do: "\"#{bitstring}\"\n", + else: "#{bitstring}\n" + end +end + +defimpl Yamel.Encoder, for: Integer do + def encode(integer, opts) do + types = opts[:quote] || [] + + if true in types or :number in types or :integer in types, + do: "\"#{integer}\"\n", + else: "#{integer}\n" + end +end + +defimpl Yamel.Encoder, for: Float do + def encode(float, opts) do + types = opts[:quote] || [] + + if true in types or :number in types or :float in types, + do: "\"#{float}\"\n", + else: "#{float}\n" + end +end + +defimpl Yamel.Encoder, for: Map do + import Yamel.Encoder.Helper + + def encode(map, opts), + do: + map + |> Enum.map(&serialize(&1, opts)) + |> Enum.join() +end + +defimpl Yamel.Encoder, for: [List, Tuple] do + import Yamel.Encoder.Helper + + def encode(tuple, opts) when is_tuple(tuple), + do: + tuple + |> Tuple.to_list() + |> encode(opts) + + def encode(list, opts) do + indent = calculate_indentation(opts) + opts = Map.update(opts, :node_level, 0, &(&1 + 1)) + + list + |> Enum.map(fn + value when is_scalar(value) -> + "#{indent}- #{Yamel.Encoder.encode(value, opts)}" + + %{__struct__: strct} = value when strct in [Date, DateTime, NaiveDateTime, Time] -> + "#{indent}- #{Yamel.Encoder.encode(value, opts)}" + + value when is_list(value) or is_map(value) -> + "#{indent}-\n#{Yamel.Encoder.encode(value, opts)}" + + {_key, _value} = value -> + "#{indent}-\n#{serialize(value, opts)}" + + value when is_tuple(value) -> + "#{indent}-\n#{Yamel.Encoder.encode(Tuple.to_list(value), opts)}" + end) + |> Enum.join() + end +end + +defimpl Yamel.Encoder, for: [Range, Stream, MapSet, HashSet, Date.Range] do + def encode(collection, opts) do + Yamel.Encoder.List.encode(collection, opts) + end +end + +defimpl Yamel.Encoder, for: [Date, DateTime, NaiveDateTime, Time] do + def encode(value, opts) do + Yamel.Encoder.encode(@for.to_iso8601(value), opts) + end +end + +defimpl Yamel.Encoder, for: URI do + def encode(value, opts) do + Yamel.Encoder.encode(@for.to_string(value), opts) + end +end + +if Code.ensure_loaded?(Decimal) do + defimpl Yamel.Encoder, for: Decimal do + def encode(value, _opts) do + Decimal.to_string(value) + end + end +end + +defimpl Yamel.Encoder, for: Any do + alias Yamel.EncodeError + + defmacro __deriving__(module, struct, opts) do + deriving(module, struct, opts) + end + + def deriving(module, _struct, opts) do + only = opts[:only] + except = opts[:except] + + extractor = + cond do + only -> + quote(do: Map.take(struct, unquote(only))) + + except -> + except = [:__struct__ | except] + quote(do: Map.drop(struct, unquote(except))) + + true -> + quote(do: Map.delete(struct, :__struct__)) + end + + quote do + defimpl Yamel.Encoder, for: unquote(module) do + def encode(struct, opts) do + Yamel.Encoder.Map.encode(unquote(extractor), opts) + end + end + end + end + + def encode(%{__struct__: _} = struct, opts) do + struct + |> Map.from_struct() + |> Yamel.Encoder.Map.encode(opts) + end + + def encode(value, _opts) do + raise EncodeError, value: value + end +end diff --git a/mix.exs b/mix.exs index 75e690c..4a55979 100644 --- a/mix.exs +++ b/mix.exs @@ -12,6 +12,7 @@ defmodule Yamel.MixProject do app: :yamel, version: @version, elixir: "~> 1.8", + elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, package: package(), deps: deps(), @@ -31,6 +32,9 @@ defmodule Yamel.MixProject do ] end + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] + # Run "mix help compile.app" to learn about applications. def application do [ diff --git a/test/support/test_encode_struct.ex b/test/support/test_encode_struct.ex new file mode 100644 index 0000000..ea478a8 --- /dev/null +++ b/test/support/test_encode_struct.ex @@ -0,0 +1,13 @@ +defmodule Yamel.EncoderTest.EncodeStructTest do + defstruct [:field1, :field2, :field3, field4: true] +end + +defmodule Yamel.EncoderTest.EncodeDerivedStructTest do + @derive {Yamel.Encoder, only: [:field4]} + defstruct [:field1, :field2, :field3, field4: true] +end + +defmodule Yamel.EncoderTest.EncodeDerivedExceptStructTest do + @derive {Yamel.Encoder, except: [:field4]} + defstruct [:field1, :field2, :field3, field4: true] +end diff --git a/test/yamel/encoder_test.exs b/test/yamel/encoder_test.exs new file mode 100644 index 0000000..58ac385 --- /dev/null +++ b/test/yamel/encoder_test.exs @@ -0,0 +1,182 @@ +defmodule Yamel.EncoderTest do + use ExUnit.Case + + alias Yamel.Encoder + alias Yamel.EncoderTest.{EncodeStructTest, EncodeDerivedExceptStructTest, EncodeDerivedStructTest} + + describe "encode/1" do + test "default opts" do + map_set = MapSet.new([1, :two, "three", {:four}, five: :six]) + # => #MapSet<[1, :two, {:four}, {:five, :six}, "three"]> + + expected_yaml = ~S""" + - 1 + - two + - + - four + - + five: six + - three + """ + + encoded_map_set_yaml = Encoder.encode(map_set) + assert encoded_map_set_yaml == expected_yaml + end + + test "with custom required opts" do + map_set = MapSet.new([1, :two, "three", {:four}, five: :six]) + # => #MapSet<[1, :two, {:four}, {:five, :six}, "three"]> + + expected_yaml = ~S""" + - 1 + - two + - + - four + - + five: six + - three + """ + + encoded_map_set_yaml = Encoder.encode(map_set, %{node_level: 1, indent_size: 4}) + assert encoded_map_set_yaml == expected_yaml + end + + test "type Range" do + range = 1..10 + + expected_yaml = ~S""" + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + """ + + encoded_range_yaml = Encoder.encode(range) + assert encoded_range_yaml == expected_yaml + end + + test "type Date.Range" do + date_range = Date.range(~D[1999-01-01], ~D[1999-01-10]) + + expected_yaml = ~S""" + - 1999-01-01 + - 1999-01-02 + - 1999-01-03 + - 1999-01-04 + - 1999-01-05 + - 1999-01-06 + - 1999-01-07 + - 1999-01-08 + - 1999-01-09 + - 1999-01-10 + """ + + encoded_date_range_yaml = Encoder.encode(date_range) + assert encoded_date_range_yaml == expected_yaml + end + + test "type MapSet" do + map_set = MapSet.new([1, :two, "three", {:four}, five: :six]) + # => #MapSet<[1, :two, {:four}, {:five, :six}, "three"]> + + expected_yaml = ~S""" + - 1 + - two + - + - four + - + five: six + - three + """ + + encoded_map_set_yaml = Encoder.encode(map_set) + assert encoded_map_set_yaml == expected_yaml + end + + test "type Tuple" do + tuple = {1, :two, "three", [:four], nil, true, %{foo: :bar}} + + expected_yaml = ~S""" + - 1 + - two + - three + - + - four + - null + - true + - + foo: bar + """ + + encoded_tuple_yaml = Encoder.encode(tuple) + assert encoded_tuple_yaml == expected_yaml + end + + test "type Stream" do + stream = + {1, :two, "three", [:four], nil, true, %{foo: :bar}} + |> Tuple.to_list() + |> Stream.cycle() + |> Stream.take(7) + + expected_yaml = ~S""" + - 1 + - two + - three + - + - four + - null + - true + - + foo: bar + """ + + encoded_stream_yaml = Encoder.encode(stream) + assert encoded_stream_yaml == expected_yaml + end + + test "type ordinary struct" do + strct = %EncodeStructTest{field1: "first field", field3: "the third"} + + expected_yaml = ~S""" + field1: first field + field2: null + field3: the third + field4: true + """ + + encoded_strct_yaml = Encoder.encode(strct) + assert encoded_strct_yaml == expected_yaml + end + + test "type derived struct" do + strct = %EncodeDerivedStructTest{field1: "first field", field3: "the third"} + + expected_yaml = ~S""" + field4: true + """ + + encoded_strct_yaml = Encoder.encode(strct) + assert encoded_strct_yaml == expected_yaml + end + + test "type derived with except fields struct" do + strct = %EncodeDerivedExceptStructTest{field1: "first field", field3: "the third"} + + expected_yaml = ~S""" + field1: first field + field2: null + field3: the third + """ + + encoded_strct_yaml = Encoder.encode(strct) + assert encoded_strct_yaml == expected_yaml + end + end +end diff --git a/test/yamel_test.exs b/test/yamel_test.exs index 61b892d..391a793 100644 --- a/test/yamel_test.exs +++ b/test/yamel_test.exs @@ -1,6 +1,5 @@ defmodule YamelTest do use ExUnit.Case - doctest Yamel @doc """ From cd8493351c2a8f09dc6a164ec1d824f8cc08464d Mon Sep 17 00:00:00 2001 From: Gio Lucas Torres Date: Mon, 19 Feb 2024 16:56:16 +0100 Subject: [PATCH 2/5] Fixes mix.exs attr module description --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 4a55979..89417d0 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Yamel.MixProject do @source_url "https://github.com/GPrimola/yamel" @logo_path "priv/img/yamel-logo.png" @licenses ["Apache-2.0"] - @description: "YAML parser and serializer lib for Elixir." + @description "YAML parser and serializer lib for Elixir." def project do [ From b666139150cd28fb665672ee3ab9b8626ae428a2 Mon Sep 17 00:00:00 2001 From: Gio Lucas Torres Date: Mon, 19 Feb 2024 16:57:54 +0100 Subject: [PATCH 3/5] Fixed format on test/yamel/encoder_test.exs --- test/yamel/encoder_test.exs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/yamel/encoder_test.exs b/test/yamel/encoder_test.exs index 58ac385..5777852 100644 --- a/test/yamel/encoder_test.exs +++ b/test/yamel/encoder_test.exs @@ -2,7 +2,12 @@ defmodule Yamel.EncoderTest do use ExUnit.Case alias Yamel.Encoder - alias Yamel.EncoderTest.{EncodeStructTest, EncodeDerivedExceptStructTest, EncodeDerivedStructTest} + + alias Yamel.EncoderTest.{ + EncodeStructTest, + EncodeDerivedExceptStructTest, + EncodeDerivedStructTest + } describe "encode/1" do test "default opts" do From 67778339fc84e79f1cc5beaaba897bce27411083 Mon Sep 17 00:00:00 2001 From: Gio Lucas Torres Date: Mon, 19 Feb 2024 17:26:25 +0100 Subject: [PATCH 4/5] Fixed dialyzer complains --- lib/yamel.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/yamel.ex b/lib/yamel.ex index 8c946f7..6e150be 100644 --- a/lib/yamel.ex +++ b/lib/yamel.ex @@ -3,7 +3,7 @@ defmodule Yamel do Defines functions to work with YAML in Elixir. """ - @type t :: map() | list() + @type t :: map() | list(any()) @type yaml :: String.t() @type keys :: :atom | :string | :atoms | :strings @type decode_opts :: [keys: keys] | [] @@ -97,7 +97,7 @@ defmodule Yamel do """ @spec encode!(Yamel.t(), Yamel.Encoder.opts()) :: yaml() - def encode!(map_or_list_or_tuple, opts \\ []) + def encode!(map_or_list_or_tuple, opts \\ %{node_level: 0, indent_size: 2}) def encode!(map_or_list_or_tuple, opts) when is_map(map_or_list_or_tuple) or is_list(map_or_list_or_tuple) or @@ -130,7 +130,7 @@ defmodule Yamel do """ @spec encode(Yamel.t(), Yamel.Encoder.opts()) :: {:ok, yaml()} | {:error, reason :: String.t()} - def encode(map_or_list, opts \\ []) + def encode(map_or_list, opts \\ %{node_level: 0, indent_size: 2}) def encode(map_or_list, opts) when is_map(map_or_list) or is_list(map_or_list), do: {:ok, to_yaml!(map_or_list, opts)} @@ -161,7 +161,7 @@ defmodule Yamel do defp maybe_atom(yaml, _keys), do: yaml - @spec to_yaml!(Yamel.t(), opts :: keyword()) :: yaml() + @spec to_yaml!(Yamel.t(), opts :: Yamel.Encoder.opts()) :: yaml() defp to_yaml!(map_or_list, opts) defp to_yaml!(map_or_list, opts) do From 134856611ac6af273b893fe811937323460b758e Mon Sep 17 00:00:00 2001 From: Giorgio Torres Date: Tue, 20 Feb 2024 09:28:43 +0100 Subject: [PATCH 5/5] New type scalar for Yamel.Encoder.Helper --- lib/yaml/encoder.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/yaml/encoder.ex b/lib/yaml/encoder.ex index b1e3563..e0bb983 100644 --- a/lib/yaml/encoder.ex +++ b/lib/yaml/encoder.ex @@ -13,6 +13,17 @@ defmodule Yamel.EncodeError do end defmodule Yamel.Encoder.Helper do + @type scalar :: + atom() + | binary() + | bitstring() + | boolean() + | number() + | Date.t() + | DateTime.t() + | NaiveDateTime.t() + | Time.t() + defguard is_scalar(value) when is_atom(value) or is_binary(value) or @@ -27,7 +38,7 @@ defmodule Yamel.Encoder.Helper do do: String.pad_trailing("", level * indent_size, " ") # when 'value' is a complex type - @spec serialize({key :: any(), value :: any()}, Yamel.Encoder.opts()) :: String.t() + @spec serialize({key :: scalar(), value :: any()}, Yamel.Encoder.opts()) :: String.t() def serialize({key, value}, opts) when is_map(value) or is_list(value) do indent = calculate_indentation(opts) opts = Map.update(opts, :node_level, 0, &(&1 + 1))