diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a96addf..942a967f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -184,4 +184,4 @@ jobs: mix local.rebar --force mix local.hex --force mix deps.get - - run: mix test + - run: mix do cmd mix test, testi diff --git a/config/config.exs b/config/config.exs index 8e1cc952..85dbc972 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,3 +10,4 @@ config :avrora, names_cache_ttl: :infinity config :logger, :console, format: "$time $metadata[$level] $levelpad$message\n" +config :elixir, :time_zone_database, Tz.TimeZoneDatabase diff --git a/lib/avrora/avro_decoder_options.ex b/lib/avrora/avro_decoder_options.ex index f141222a..607abe0f 100644 --- a/lib/avrora/avro_decoder_options.ex +++ b/lib/avrora/avro_decoder_options.ex @@ -4,6 +4,7 @@ defmodule Avrora.AvroDecoderOptions do `:avro_ocf` decoder options. """ + alias Avrora.AvroTypeConverter alias Avrora.Config @options %{ @@ -13,7 +14,8 @@ defmodule Avrora.AvroDecoderOptions do map_type: :map, record_type: :map } - @null_type_name "null" + # TODO Rename avro_type_converter into something better + @type_converters [AvroTypeConverter.NullIntoNil, AvroTypeConverter.PrimitiveIntoLogical] @doc """ A unified erlavro decoder options compatible for both binary and OCF decoders. @@ -25,17 +27,17 @@ defmodule Avrora.AvroDecoderOptions do # NOTE: This is internal module function and should never be used directly @doc false def __hook__(type, sub_name_or_idx, data, decode_fun) do - convert = convert_null_values() - decoder_hook = decoder_hook() + result = decoder_hook().(type, sub_name_or_idx, data, decode_fun) - result = decoder_hook.(type, sub_name_or_idx, data, decode_fun) - - if convert == true && :avro.get_type_name(type) == @null_type_name, - do: {nil, data}, - else: result + @type_converters + |> List.foldl(result, fn type_converter, value -> + case type_converter.convert(value, type) do + {:ok, result} -> result + {:error, reason} -> raise(reason) + end + end) end - defp convert_null_values, do: Config.self().convert_null_values() defp convert_map_to_proplist, do: Config.self().convert_map_to_proplist() defp decoder_hook, do: Config.self().decoder_hook() end diff --git a/lib/avrora/avro_logical_type_caster.ex b/lib/avrora/avro_logical_type_caster.ex new file mode 100644 index 00000000..ec633c02 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster.ex @@ -0,0 +1,14 @@ +defmodule Avrora.AvroLogicalTypeCaster do + @moduledoc """ + TODO Write AvroLogicalTypeCaster moduledoc + """ + + @doc """ + TODO Write convert callback doc + + NOTE that type is an erlavro type + and we are converting erlang/avro types into Elixir + """ + @callback cast(value :: term(), type :: term()) :: + {:ok, result :: term()} | {:error, reason :: Exception.t() | term()} +end diff --git a/lib/avrora/avro_logical_type_caster/date.ex b/lib/avrora/avro_logical_type_caster/date.ex new file mode 100644 index 00000000..449872f6 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/date.ex @@ -0,0 +1,15 @@ +defmodule Avrora.AvroLogicalTypeCaster.Date do + @moduledoc """ + The `date` logical type represents a date within the calendar, + with no reference to a particular time zone or time of day. + + The `date` logical type annotates an Avro `int`, where the `int` stores + the number of days from the unix epoch, 1 January 1970 (ISO calendar). + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @unix_epoch ~D[1970-01-01] + + @impl true + def cast(value, _type), do: {:ok, Date.add(@unix_epoch, value)} +end diff --git a/lib/avrora/avro_logical_type_caster/decimal.ex b/lib/avrora/avro_logical_type_caster/decimal.ex new file mode 100644 index 00000000..a81b59ed --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/decimal.ex @@ -0,0 +1,40 @@ +defmodule Avrora.AvroLogicalTypeCaster.Decimal do + @moduledoc """ + The `decimal` logical type represents an arbitrary-precision signed decimal + number of the form `unscaled × 10-scale`. + + The `decimal` logical type annotates Avro bytes or fixed types. + The byte array must contain the two’s-complement representation + of the unscaled integer value in big-endian byte order. + + NOTE: This module is NOT INCLUDED into defaults of `Avrora.Config` and must + be added manually, like this + + config :avrora, logical_types_casting: %{ + "decimal" => Avrora.AvroLogicalTypeCaster.Decimal + ... + } + + NOTE: This module REQUIRES presence of the Decimal library, for details see + https://hex.pm/packages/decimal + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @default_scale_prop {"scale", 0} + + @impl true + def cast(value, type) do + <> = value + + scale = + :avro.get_custom_props(type) + |> List.keyfind("scale", 0, @default_scale_prop) + |> elem(1) + + {:ok, decimal(value, scale)} + end + + defp decimal(value, 0), do: Decimal.new(value) + defp decimal(value, scale) when value > 0, do: Decimal.new(1, value, -scale) + defp decimal(value, scale) when value < 0, do: Decimal.new(-1, -value, -scale) +end diff --git a/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex b/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex new file mode 100644 index 00000000..6145a552 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex @@ -0,0 +1,21 @@ +defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMicros do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.LocalTimestampMicros moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + # @timezone "Etc/UTC" + @timezone "Japan" + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:ok, date_time} <- DateTime.from_unix(value, :microsecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end + end +end diff --git a/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex b/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex new file mode 100644 index 00000000..2531e082 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex @@ -0,0 +1,21 @@ +defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMillis do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.LocalTimestampMillis moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + # @timezone "Etc/UTC" + @timezone "Japan" + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:ok, date_time} <- DateTime.from_unix(value, :millisecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end + end +end diff --git a/lib/avrora/avro_logical_type_caster/noop.ex b/lib/avrora/avro_logical_type_caster/noop.ex new file mode 100644 index 00000000..9927cb54 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/noop.ex @@ -0,0 +1,11 @@ +defmodule Avrora.AvroLogicalTypeCaster.Noop do + @moduledoc """ + This is no-op module used for unsupported logical types. + It keeps the original value untouched and does not generate any warning. + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + @impl true + def cast(value, _type), do: {:ok, value} +end diff --git a/lib/avrora/avro_logical_type_caster/noop_warning.ex b/lib/avrora/avro_logical_type_caster/noop_warning.ex new file mode 100644 index 00000000..ea0f95d1 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/noop_warning.ex @@ -0,0 +1,18 @@ +defmodule Avrora.AvroLogicalTypeCaster.NoopWarning do + @moduledoc """ + This is no-op module used for unsupported logical types. + It keeps the original value untouched, but generated the warning. + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + require Logger + + @impl true + def cast(value, type) do + {_, logical_type} = :avro.get_custom_props(type) |> List.keyfind("logicalType", 0) + Logger.warning("unsupported logical type `#{logical_type}', its value was not type casted") + + {:ok, value} + end +end diff --git a/lib/avrora/avro_logical_type_caster/time_micros.ex b/lib/avrora/avro_logical_type_caster/time_micros.ex new file mode 100644 index 00000000..9408d85c --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/time_micros.ex @@ -0,0 +1,22 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimeMicros do + @moduledoc """ + The `time-micros` logical type represents a time of day, with no reference to + a particular calendar, time zone or date, with a precision of one microsecond. + + The `time-micros` logical type annotates an Avro `long`, where the `long` + stores the number of microseconds after midnight, 00:00:00.000000. + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @microseconds 1_000_000 + @precision 6 + + @impl true + def cast(value, _type) do + time = + div(value, @microseconds) + |> Time.from_seconds_after_midnight({rem(value, @microseconds), @precision}) + + {:ok, time} + end +end diff --git a/lib/avrora/avro_logical_type_caster/time_millis.ex b/lib/avrora/avro_logical_type_caster/time_millis.ex new file mode 100644 index 00000000..70d46549 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/time_millis.ex @@ -0,0 +1,23 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimeMillis do + @moduledoc """ + The `time-millis` logical type represents a time of day, with no reference to + a particular calendar, time zone or date, with a precision of one millisecond. + + The `time-millis` logical type annotates an Avro `int`, where the `int` stores + the number of milliseconds after midnight, 00:00:00.000. + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @milliseconds 1_000 + @precision 3 + + @impl true + def cast(value, _type) do + time = + div(value, @milliseconds) + |> Time.from_seconds_after_midnight({rem(value, @milliseconds) * 1_000, @precision}) + |> Time.truncate(:millisecond) + + {:ok, time} + end +end diff --git a/lib/avrora/avro_logical_type_caster/timestamp_micros.ex b/lib/avrora/avro_logical_type_caster/timestamp_micros.ex new file mode 100644 index 00000000..85e36c2c --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/timestamp_micros.ex @@ -0,0 +1,15 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimestampMicros do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimestampMicros moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:error, reason} <- DateTime.from_unix(value, :microsecond), + do: {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end +end diff --git a/lib/avrora/avro_logical_type_caster/timestamp_millis.ex b/lib/avrora/avro_logical_type_caster/timestamp_millis.ex new file mode 100644 index 00000000..7eab8d74 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/timestamp_millis.ex @@ -0,0 +1,15 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimestampMillis do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimestampMillis moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:error, reason} <- DateTime.from_unix(value, :millisecond), + do: {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end +end diff --git a/lib/avrora/avro_type_converter.ex b/lib/avrora/avro_type_converter.ex new file mode 100644 index 00000000..bef3e45f --- /dev/null +++ b/lib/avrora/avro_type_converter.ex @@ -0,0 +1,14 @@ +defmodule Avrora.AvroTypeConverter do + @moduledoc """ + TODO Write AvroTypeConverter moduledoc + """ + + @doc """ + TODO Write convert callback doc + + NOTE that type is an erlavro type + and we are converting erlang/avro types into Elixir + """ + @callback convert(value :: term(), type :: term()) :: + {:ok, result :: {term(), binary()}} | {:error, reason :: Exception.t() | term()} +end diff --git a/lib/avrora/avro_type_converter/null_into_nil.ex b/lib/avrora/avro_type_converter/null_into_nil.ex new file mode 100644 index 00000000..b94be189 --- /dev/null +++ b/lib/avrora/avro_type_converter/null_into_nil.ex @@ -0,0 +1,21 @@ +defmodule Avrora.AvroTypeConverter.NullIntoNil do + @moduledoc """ + TODO Write NullIntoNil moduledoc + """ + + @behaviour Avrora.AvroTypeConverter + @null_type_name "null" + + alias Avrora.Config + + @impl true + def convert(value, type) do + if enabled() && :avro.get_type_name(type) == @null_type_name do + {:ok, {nil, elem(value, 1)}} + else + {:ok, value} + end + end + + defp enabled, do: Config.self().convert_null_values() == true +end diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex new file mode 100644 index 00000000..6b0a74b9 --- /dev/null +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -0,0 +1,48 @@ +# TODO Merge into type caster and remove list handling in decoder options +defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do + @moduledoc """ + TODO Write PrimitiveIntoLogical moduledoc + """ + + @behaviour Avrora.AvroTypeConverter + + @logical_type "logicalType" + + alias Avrora.AvroLogicalTypeCaster + alias Avrora.Config + + @impl true + def convert(value, type) do + with true <- enabled(), + {@logical_type, logical_type} <- :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0), + {value, rest} <- value, + {:ok, converted} <- do_convert(value, type, logical_type) do + {:ok, {converted, rest}} + else + {:error, reason} -> {:error, reason} + _ -> {:ok, value} + end + end + + # TODO Remove and replace with config + # TODO Add support module to test Japan timezone in local timestamp + @config %{ + "uuid" => AvroLogicalTypeCaster.Noop, + "date" => AvroLogicalTypeCaster.Date, + "decimal" => AvroLogicalTypeCaster.Decimal, + "time-millis" => AvroLogicalTypeCaster.TimeMillis, + "time-micros" => AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => AvroLogicalTypeCaster.TimestampMicros, + "local-timestamp-millis" => AvroLogicalTypeCaster.LocalTimestampMillis, + "local-timestamp-micros" => AvroLogicalTypeCaster.LocalTimestampMicros, + "_" => AvroLogicalTypeCaster.NoopWarning + } + + # TODO Replace fetch! with fetch and raise generic error of Avrora + defp do_convert(value, type, logical_type) do + Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type) + end + + defp enabled, do: Config.self().cast_logical_types() == true +end diff --git a/lib/avrora/client.ex b/lib/avrora/client.ex index 5a45f307..0e789945 100644 --- a/lib/avrora/client.ex +++ b/lib/avrora/client.ex @@ -27,15 +27,20 @@ defmodule Avrora.Client do {:ok, pid} = MyClient.start_link() """ - # NOTE: Modules below contain usage of some other modules which should be defined - # under the private client module, for instance, `Avrora.Config` could be - # defined as `MyClient.Config`. Hence they are listed together with some - # aliases. + # NOTE: Modules below contain usage of other Avrora modules which will not be + # able to resolve in private client module until we define them specificly + # for the private client. + # + # For instance, `Avrora.Config` could be defined as `MyClient.Config` + # and because `Avrora.Resolver` as `MyClient.Resolver` in private client + # is using it we list it here. @modules ~w( encoder resolver avro_schema_store avro_decoder_options + avro_type_converter/null_into_nil + avro_type_converter/primitive_into_logical schema/encoder codec/plain codec/schema_registry @@ -46,12 +51,19 @@ defmodule Avrora.Client do utils/registrar ) + # NOTE: Aliases used in Avrora modules should target correct private client + # module when generated. Because of that we list every module which was + # declared in `alias` statement. + # + # As a tradeoff, we don't use grouping in alias declarations, in other + # words you will not find constructions like `alias Avrora.{Codec, Config}` @aliases ~w( Codec Config Resolver Schema.Encoder AvroDecoderOptions + AvroTypeConverter Codec.Plain Codec.SchemaRegistry Codec.ObjectContainerFile @@ -102,6 +114,16 @@ defmodule Avrora.Client do @opts unquote(opts) @otp_app Keyword.get(@opts, :otp_app) + # TODO Add tests to check logical types resolution + @logical_types_casting %{ + "_" => Avrora.AvroLogicalTypeCaster.NoopWarning, + "uuid" => Avrora.AvroLogicalTypeCaster.Noop, + "date" => Avrora.AvroLogicalTypeCaster.Date, + "time-millis" => Avrora.AvroLogicalTypeCaster.TimeMillis, + "time-micros" => Avrora.AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => Avrora.AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => Avrora.AvroLogicalTypeCaster.TimestampMicros + } def schemas_path do path = get(@opts, :schemas_path, "./priv/schemas") @@ -115,8 +137,10 @@ defmodule Avrora.Client do def registry_schemas_autoreg, do: get(@opts, :registry_schemas_autoreg, true) def convert_null_values, do: get(@opts, :convert_null_values, true) def convert_map_to_proplist, do: get(@opts, :convert_map_to_proplist, false) + def cast_logical_types, do: get(@opts, :cast_logical_types, true) def names_cache_ttl, do: get(@opts, :names_cache_ttl, :infinity) def decoder_hook, do: get(@opts, :decoder_hook, fn _, _, data, fun -> fun.(data) end) + def logical_types_casting, do: get(@opts, :logical_types_casting, @logical_types_casting) def file_storage, do: unquote(:"Elixir.#{module}.Storage.File") def memory_storage, do: unquote(:"Elixir.#{module}.Storage.Memory") def registry_storage, do: unquote(:"Elixir.#{module}.Storage.Registry") diff --git a/lib/avrora/codec.ex b/lib/avrora/codec.ex index e78c9d09..04e57bc7 100644 --- a/lib/avrora/codec.ex +++ b/lib/avrora/codec.ex @@ -45,7 +45,7 @@ defmodule Avrora.Codec do {:ok, %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99}} """ - @callback decode(payloadd :: binary()) :: {:ok, result :: map() | list(map())} | {:error, reason :: term()} + @callback decode(payload :: binary()) :: {:ok, result :: map() | list(map())} | {:error, reason :: term()} @doc """ Decode a binary Avro message into the Elixir data with given schema. @@ -61,7 +61,7 @@ defmodule Avrora.Codec do """ @callback decode(payload :: binary(), options :: keyword(Avrora.Schema.t())) :: - {:ok, result :: map() | list(map())} | {:error, reason :: term()} + {:ok, result :: map() | list(map())} | {:error, reason :: Exception.t() | term()} @doc """ Encode the Elixir data into a binary Avro message. diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 8c120ed1..9e9b2dcd 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,8 +12,10 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` + * `cast_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` + * `logical_types_casting` mapping between logical type and casting logic, default `uuid, date, time-millis, time-micros, timestamp-millis, timestamp-micros` ## Internal use interface: @@ -31,6 +33,7 @@ defmodule Avrora.Config do @callback registry_schemas_autoreg :: boolean() @callback convert_null_values :: boolean() @callback convert_map_to_proplist :: boolean() + @callback cast_logical_types :: boolean() @callback names_cache_ttl :: integer() | atom() @callback decoder_hook :: (any(), any(), any(), any() -> any()) @callback file_storage :: module() @@ -39,6 +42,16 @@ defmodule Avrora.Config do @callback http_client :: module() @callback ets_lib :: module() | atom() + @logical_types_casting %{ + "_" => Avrora.AvroLogicalTypeCaster.NoopWarning, + "uuid" => Avrora.AvroLogicalTypeCaster.Noop, + "date" => Avrora.AvroLogicalTypeCaster.Date, + "time-millis" => Avrora.AvroLogicalTypeCaster.TimeMillis, + "time-micros" => Avrora.AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => Avrora.AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => Avrora.AvroLogicalTypeCaster.TimestampMicros + } + @doc false def schemas_path do path = get_env(:schemas_path, "./priv/schemas") @@ -65,12 +78,18 @@ defmodule Avrora.Config do @doc false def convert_map_to_proplist, do: get_env(:convert_map_to_proplist, false) + @doc false + def cast_logical_types, do: get_env(:cast_logical_types, true) + @doc false def names_cache_ttl, do: get_env(:names_cache_ttl, :infinity) @doc false def decoder_hook, do: get_env(:decoder_hook, fn _, _, data, fun -> fun.(data) end) + @doc false + def logical_types_casting, do: get_env(:logical_types_casting, @logical_types_casting) + @doc false def file_storage, do: Avrora.Storage.File diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex new file mode 100644 index 00000000..90066650 --- /dev/null +++ b/lib/avrora/errors.ex @@ -0,0 +1,31 @@ +defmodule Avrora.Errors do + @moduledoc """ + TODO Write Errors moduledoc + """ + + defmodule LogicalTypeDecodingError do + @moduledoc """ + TODO Write LogicalTypeDecodingError moduledoc + """ + + defexception [:code] + + @type t :: %__MODULE__{code: atom()} + @messages %{ + invalid_unix_time: "given value is an invalid UNIX time", + missing_decimal_lib: "missing `Decimal' library, see https://hex.pm/packages/decimal", + time_zone_not_found: "configured local timezone not found in timezone database", + utc_only_time_zone_database: "default timezone database does not support configured local timezone" + } + + @impl true + def exception(code) when is_atom(code), do: %__MODULE__{code: code} + def exception(_), do: %__MODULE__{} + + @impl true + def message(%{code: code}) when is_atom(code) and code != nil, + do: "logical type decoding error, #{Map.get(@messages, code, inspect(code))}" + + def message(_), do: "logical type decoding error" + end +end diff --git a/mix.exs b/mix.exs index 5700be89..4064c7b3 100644 --- a/mix.exs +++ b/mix.exs @@ -115,6 +115,8 @@ defmodule Avrora.MixProject do {:jason, "~> 1.0"}, {:erlavro, "~> 2.9.3"}, {:credo, "~> 1.5", only: :dev, runtime: false}, + {:tz, "~> 0.26.2", only: [:dev, :test], runtime: false}, + {:decimal, "~> 2.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.24", only: :dev, runtime: false}, {:dialyxir, "~> 1.1", only: :dev, runtime: false}, {:mox, "~> 1.0", only: :test}, @@ -124,7 +126,10 @@ defmodule Avrora.MixProject do defp aliases do [ - docso: ["docs", "cmd open doc/index.html"], + test: ["test --exclude integration --color"], + testi: ["cmd --cd test/integration mix test --color"], + testall: ["do cmd mix test, testi"], + showdocs: ["docs", "cmd open doc/index.html"], check: ["cmd mix coveralls", "dialyzer", "credo"], release: [ "check", diff --git a/mix.lock b/mix.lock index e5c64d0c..fab8d34e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,28 +1,31 @@ %{ "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, - "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, + "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "dialyxir": {:hex, :dialyxir, "1.4.2", "764a6e8e7a354f0ba95d58418178d486065ead1f69ad89782817c296d0d746a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "516603d8067b2fd585319e4b13d3674ad4f314a5902ba8130cd97dc902ce6bbd"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlavro": {:hex, :erlavro, "2.9.8", "9b9c0eff6dc1c708a277b4143c0020659c42bcd634d0d7237c6435fb0c2f3266", [:make, :rebar3], [{:jsone, "1.4.6", [hex: :jsone, repo: "hexpm", optional: false]}, {:snappyer, "1.2.8", [hex: :snappyer, repo: "hexpm", optional: false]}], "hexpm", "7182c539f408633927b30380aa6123ea3e4b9a04c2bc752f0fe227ef5e9c3a70"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.30.2", "7a3e63ddb387746925bbbbcf6e9cb00e43c757cc60359a2b40059aea573e3e57", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5ba8cb61d069012f16b50e575b0e3e6cf4083935f7444fab0d92c9314ce86bb6"}, - "excoveralls": {:hex, :excoveralls, "0.16.1", "0bd42ed05c7d2f4d180331a20113ec537be509da31fed5c8f7047ce59ee5a7c5", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dae763468e2008cf7075a64cb1249c97cb4bc71e236c5c2b5e5cdf1cfa2bf138"}, + "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"}, + "excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~> 2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "jsone": {:hex, :jsone, "1.4.6", "644d6d57befb22c8e19b324dee19d73b1c004565009861a8f64c68b7b9e64dbf", [:rebar3], [], "hexpm", "78eee8bb38f0bee2e73673d71bc75fc6fb01f56f0d23e769a26eee3655487a38"}, - "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "tz": {:hex, :tz, "0.26.2", "a40e4bb223344c6fc7b74dda25df1f26b88a30db23fa6e55de843bd79148ccdb", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.5", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "224b0618dd1e032778a094040bc710ef9aff6e2fa8fffc2716299486f27b9e68"}, + "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } diff --git a/test/avrora/avro_type_converter/null_into_nil_test.exs b/test/avrora/avro_type_converter/null_into_nil_test.exs new file mode 100644 index 00000000..09f2443c --- /dev/null +++ b/test/avrora/avro_type_converter/null_into_nil_test.exs @@ -0,0 +1,39 @@ +defmodule Avrora.AvroTypeConverter.NullIntoNilTest do + use ExUnit.Case, async: true + doctest Avrora.AvroTypeConverter.NullIntoNil + + import Mox + import Support.Config + + alias Avrora.{Codec, Schema} + + setup :verify_on_exit! + setup :support_config + + describe "convert/2" do + test "when null values must be kept as is" do + stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"key" => "user-1", "value" => :null} + end + + test "when null values must be converted" do + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"key" => "user-1", "value" => nil} + end + end + + defp message, do: <<12, 117, 115, 101, 114, 45, 49, 0>> + + defp schema do + {:ok, schema} = Schema.Encoder.from_json(json_schema()) + %{schema | id: nil, version: nil} + end + + defp json_schema do + ~s({"namespace":"io.confluent","name":"Null_Value","type":"record","fields":[{"name":"key","type":"string"},{"name":"value","type":["null","int"]}]}) + end +end diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs new file mode 100644 index 00000000..8480aecb --- /dev/null +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -0,0 +1,166 @@ +defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do + use ExUnit.Case, async: true + doctest Avrora.AvroTypeConverter.PrimitiveIntoLogical + + import Mox + import Support.Config + import ExUnit.CaptureLog + + alias Avrora.{Codec, Schema} + + setup :verify_on_exit! + setup :support_config + + describe "convert/2" do + test "when logical types must be kept as is" do + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(date_message(), schema: schema(date_json())) + + assert decoded == %{"birthday" => 17100, "number" => 17100} + end + + test "when logical types must be converted" do + {:ok, decoded} = Codec.Plain.decode(date_message(), schema: schema(date_json())) + + assert decoded == %{"birthday" => ~D[2016-10-26], "number" => 17100} + end + + test "when logical types must be converted, but it is unknown logical type" do + output = + capture_log(fn -> + decoded = %{"birthday" => 17100, "number" => 17100} + assert {:ok, decoded} == Codec.Plain.decode(date_message(), schema: schema(unknown_json())) + end) + + assert output =~ "unsupported logical type `unknown', its value was not type casted" + end + + test "when logical type is uuid" do + {:ok, decoded} = Codec.Plain.decode(uuid_message(), schema: schema(uuid_json())) + + assert decoded == %{"uuid" => "016c25fd-70e0-56fe-9d1a-56e80fa20b82"} + end + + test "when logical type is decimal without scale" do + {:ok, decoded} = Codec.Plain.decode(decimal_fixed_message(), schema: schema(decimal_fixed_json())) + + assert decoded == %{"decimal" => Decimal.new("123456")} + end + + test "when logical type is decimal with scale" do + {:ok, decoded} = Codec.Plain.decode(decimal_bytes_message(), schema: schema(decimal_bytes_json())) + + assert decoded == %{"decimal" => Decimal.new("-1234.56")} + end + + test "when logical type is time with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(time_millis_message(), schema: schema(time_millis_json())) + + assert decoded == %{"time" => ~T[04:28:07.123]} + end + + test "when logical type is time with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(time_micros_message(), schema: schema(time_micros_json())) + + assert decoded == %{"time" => ~T[04:28:07.000000]} + end + + test "when logical type is timestamp with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_millis_message(), schema: schema(timestamp_millis_json())) + + assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.123Z]} + end + + test "when logical type is timestamp with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_micros_message(), schema: schema(timestamp_micros_json())) + + assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.000000Z]} + end + + test "when logical type is timestamp and its value is incorrect" do + {:error, error} = Codec.Plain.decode(timestamp_invalid_message(), schema: schema(timestamp_millis_json())) + + assert error.code == :invalid_unix_time + assert Exception.message(error) =~ "invalid UNIX time" + end + + test "when logical type is local timestamp with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_millis_message(), schema: schema(local_timestamp_millis_json())) + + assert DateTime.to_string(decoded["timestamp"]) == "2016-10-26 13:28:07.123+09:00 JST Japan" + end + + test "when logical type is local timestamp with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_micros_message(), schema: schema(local_timestamp_micros_json())) + + assert DateTime.to_string(decoded["timestamp"]) == "2016-10-26 13:28:07.000000+09:00 JST Japan" + end + + test "when logical type is local timestamp and its value is incorrect" do + {:error, error} = Codec.Plain.decode(timestamp_invalid_message(), schema: schema(local_timestamp_millis_json())) + + assert error.code == :invalid_unix_time + assert Exception.message(error) =~ "invalid UNIX time" + end + end + + defp date_message, do: <<152, 139, 2, 152, 139, 2>> + defp uuid_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" + defp decimal_fixed_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> + defp decimal_bytes_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + defp time_millis_message, do: <<166, 225, 171, 15>> + defp time_micros_message, do: <<128, 143, 225, 237, 119>> + defp timestamp_millis_message, do: <<166, 161, 246, 243, 255, 85>> + defp timestamp_micros_message, do: <<128, 143, 229, 211, 161, 239, 159, 5>> + defp timestamp_invalid_message, do: <<128, 128, 155, 199, 153, 131, 162, 132, 7>> + + defp schema(json) do + {:ok, schema} = Schema.Encoder.from_json(json) + %{schema | id: nil, version: nil} + end + + defp date_json do + ~s({"namespace":"io.confluent","name":"Date","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) + end + + defp unknown_json do + ~s({"namespace":"io.confluent","name":"Unknown","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"unknown"}}]}) + end + + defp uuid_json do + ~s({"namespace":"io.confluent","name":"Uuid","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"uuid"}}]}) + end + + defp decimal_fixed_json do + ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"decimal"}}]}) + end + + defp decimal_bytes_json do + ~s({"namespace":"io.confluent","name":"Decimal_With_Scale","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"decimal","scale":2}}]}) + end + + defp time_millis_json do + ~s({"namespace":"io.confluent","name":"Time_Millis","type":"record","fields":[{"name":"time","type":{"type":"int","logicalType":"time-millis"}}]}) + end + + defp time_micros_json do + ~s({"namespace":"io.confluent","name":"Time_Micros","type":"record","fields":[{"name":"time","type":{"type":"long","logicalType":"time-micros"}}]}) + end + + defp timestamp_millis_json do + ~s({"namespace":"io.confluent","name":"Timestamp_Millis","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-millis"}}]}) + end + + defp timestamp_micros_json do + ~s({"namespace":"io.confluent","name":"Timestamp_Micros","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-micros"}}]}) + end + + defp local_timestamp_millis_json do + ~s({"namespace":"io.confluent","name":"Local_Timestamp_Millis","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"local-timestamp-millis"}}]}) + end + + defp local_timestamp_micros_json do + ~s({"namespace":"io.confluent","name":"Local_Timestamp_Micros","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"local-timestamp-micros"}}]}) + end +end diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index c35549d6..1199b4dc 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -80,15 +80,15 @@ defmodule Avrora.Codec.PlainTest do test "when payload is a valid binary and null values must be as is" do stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) - {:ok, decoded} = Codec.Plain.decode(null_value_message(), schema: null_value_schema()) + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) - assert decoded == %{"key" => "user-1", "value" => :null} + assert decoded == %{"birthday" => ~D[2016-10-26], "guests" => :null} end test "when payload is a valid binary and null values must be converted" do - {:ok, decoded} = Codec.Plain.decode(null_value_message(), schema: null_value_schema()) + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) - assert decoded == %{"key" => "user-1", "value" => nil} + assert decoded == %{"birthday" => ~D[2016-10-26], "guests" => nil} end test "when payload is a valid binary and map type must be decoded as proplist" do @@ -135,6 +135,23 @@ defmodule Avrora.Codec.PlainTest do assert decoded_int == %{"union_field" => {"io.confluent.as_int", %{"value" => 42}}} assert decoded_str == %{"union_field" => {"io.confluent.as_str", %{"value" => "42"}}} end + + test "when decoding message and logical types must be as is" do + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) + + assert decoded == %{"birthday" => 17100, "guests" => nil} + end + + test "when decoding message and all types must be as is" do + stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) + + assert decoded == %{"birthday" => 17100, "guests" => :null} + end end describe "encode/2" do @@ -272,7 +289,7 @@ defmodule Avrora.Codec.PlainTest do 48, 48, 48, 48, 48, 48, 48, 48, 48, 123, 20, 174, 71, 225, 250, 47, 64>> end - defp null_value_message, do: <<12, 117, 115, 101, 114, 45, 49, 0>> + defp convertable_message, do: <<152, 139, 2, 0>> defp map_message, do: <<1, 20, 6, 107, 101, 121, 10, 118, 97, 108, 117, 101, 0>> defp payment_payload, do: %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99} @@ -281,11 +298,6 @@ defmodule Avrora.Codec.PlainTest do %{schema | id: nil, version: nil} end - defp null_value_schema do - {:ok, schema} = Schema.Encoder.from_json(null_value_json_schema()) - %{schema | id: nil, version: nil} - end - defp map_schema do {:ok, schema} = Schema.Encoder.from_json(map_json_schema()) %{schema | id: nil, version: nil} @@ -306,12 +318,13 @@ defmodule Avrora.Codec.PlainTest do %{schema | id: nil, version: nil} end - defp payment_json_schema do - ~s({"namespace":"io.confluent","name":"Payment","type":"record","fields":[{"name":"id","type":"string"},{"name":"amount","type":"double"}]}) + defp convertable_schema do + {:ok, schema} = Schema.Encoder.from_json(converterable_json_schema()) + %{schema | id: nil, version: nil} end - defp null_value_json_schema do - ~s({"namespace":"io.confluent","name":"Null_Value","type":"record","fields":[{"name":"key","type":"string"},{"name":"value","type":["null","int"]}]}) + defp payment_json_schema do + ~s({"namespace":"io.confluent","name":"Payment","type":"record","fields":[{"name":"id","type":"string"},{"name":"amount","type":"double"}]}) end defp map_json_schema do @@ -329,4 +342,8 @@ defmodule Avrora.Codec.PlainTest do defp union_json_schema do ~s({"namespace":"io.confluent","name":"Union_Value","type":"record","fields":[{"name":"union_field","type":[{"type":"record","name":"as_str","fields":[{"name":"value","type":"string"}]},{"type":"record","name":"as_int","fields":[{"name":"value","type":"int"}]}]}]}) end + + defp converterable_json_schema do + ~s({"namespace":"io.confluent","name":"Converter","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"date"}},{"name":"guests","type":["null","int"]}]}) + end end diff --git a/test/avrora/resolver_test.exs b/test/avrora/resolver_test.exs index 375e22a4..bb1fbed6 100644 --- a/test/avrora/resolver_test.exs +++ b/test/avrora/resolver_test.exs @@ -5,6 +5,7 @@ defmodule Avrora.ResolverTest do import Mox import Support.Config import ExUnit.CaptureLog + alias Avrora.{Resolver, Schema} setup :verify_on_exit! diff --git a/test/integration/mix.exs b/test/integration/mix.exs index 3cca9d8e..0b947e51 100644 --- a/test/integration/mix.exs +++ b/test/integration/mix.exs @@ -20,7 +20,7 @@ defmodule Integration.MixProject do defp deps do [ {:avrora, path: "../../"}, - {:dialyxir, "~> 1.0.0", runtime: false} + {:dialyxir, "~> 1.4.0", runtime: false} ] end end diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs new file mode 100644 index 00000000..b1f02e76 --- /dev/null +++ b/test/integration/test/decimal_logical_type_test.exs @@ -0,0 +1,39 @@ +defmodule Integration.DecimalLogicalTypeTest do + use ExUnit.Case + + alias Avrora.{Codec, Schema} + + @tag :integration + describe "decimal logical type" do + test "when decimal library is not installed" do + json = ~s( + { + "namespace": "io.confluent", + "name": "Decimal_Test", + "type": "record", + "fields": [ + { + "name": "decimal", + "type": { + "type": "bytes", + "precision": 3, + "logicalType": "decimal", + "scale": 2 + } + } + ] + } + ) + + {:ok, _} = Avrora.start_link() + {:ok, schema} = Schema.Encoder.from_json(json) + + schema = %{schema | id: nil, version: nil} + message = <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + + {:error, error} = Codec.Plain.decode(message, schema: schema) + + assert error == %UndefinedFunctionError{module: Decimal, function: :new, arity: 3} + end + end +end diff --git a/test/integration/test/test_helper.exs b/test/integration/test/test_helper.exs new file mode 100644 index 00000000..112dbe4c --- /dev/null +++ b/test/integration/test/test_helper.exs @@ -0,0 +1,2 @@ +Mix.shell(Mix.Shell.Process) +ExUnit.start(capture_log: true) diff --git a/test/support/config.ex b/test/support/config.ex index 184431f3..8f9d3815 100644 --- a/test/support/config.ex +++ b/test/support/config.ex @@ -52,6 +52,8 @@ defmodule Support.Config do @impl true def convert_map_to_proplist, do: false @impl true + def cast_logical_types, do: true + @impl true def file_storage, do: Avrora.Storage.FileMock @impl true def memory_storage, do: Avrora.Storage.MemoryMock