From 277bfa1b9d8b6c85848636d5dc3d8cf74944139f Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Sat, 19 Oct 2024 00:02:23 -0400 Subject: [PATCH 01/12] Support flint and add extension --- .formatter.exs | 8 ++++++-- lib/instructor/instruction.ex | 5 +++++ lib/instructor/json_schema.ex | 34 +++++++++++++++++++++++++++++++++- mix.exs | 2 +- mix.lock | 9 ++++++++- 5 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 lib/instructor/instruction.ex diff --git a/.formatter.exs b/.formatter.exs index dfaff38..a265664 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,6 +1,10 @@ # Used by "mix format" [ - import_deps: [:ecto, :phoenix, :phoenix_live_view], + import_deps: [:flint, :phoenix, :phoenix_live_view], plugins: [Phoenix.LiveView.HTMLFormatter], - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", "pages/cookbook/**/*.{ex,exs}"] + inputs: [ + "{mix,.formatter}.exs", + "{config,lib,test}/**/*.{ex,exs}", + "pages/cookbook/**/*.{ex,exs}" + ] ] diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex new file mode 100644 index 0000000..f44f9a2 --- /dev/null +++ b/lib/instructor/instruction.ex @@ -0,0 +1,5 @@ +defmodule Instructor.Instruction do + use Flint.Extension + + option :doc, default: "", validator: &is_binary/1, required: false +end diff --git a/lib/instructor/json_schema.ex b/lib/instructor/json_schema.ex index dbe3126..9a0fbf2 100644 --- a/lib/instructor/json_schema.ex +++ b/lib/instructor/json_schema.ex @@ -51,6 +51,9 @@ defmodule Instructor.JSONSchema do ecto_schema_struct_literal = "%#{title_for(ecto_schema)}{}" case Code.fetch_docs(ecto_schema) do + {:docs_v1, _, :elixir, _, %{"en" => module_doc}, _, _} -> + module_doc + {_, _, _, _, _, _, docs} -> docs |> Enum.find_value(fn @@ -72,12 +75,41 @@ defmodule Instructor.JSONSchema do when is_ecto_schema(ecto_schema) do seen_schemas = MapSet.put(seen_schemas, ecto_schema) + field_docs = + try do + ecto_schema.__schema__(:extra_options) + |> Enum.map(fn {field, opts} -> + {field, Keyword.get(opts, :doc, "")} + end) + rescue + _ -> + [] + end + properties = ecto_schema.__schema__(:fields) |> Enum.map(fn field -> type = ecto_schema.__schema__(:type, field) + field_doc = Keyword.get(field_docs, field, "") |> String.trim() value = for_type(type) - value = Map.merge(%{title: Atom.to_string(field)}, value) + + value = + Map.merge(%{title: Atom.to_string(field)}, value) + |> Map.update(:description, field_doc, fn desc -> + field_doc = + cond do + field_doc == "" -> + "" + + String.ends_with?(field_doc, ".") -> + field_doc <> " " + + true -> + field_doc <> ". " + end + + field_doc <> desc + end) {field, value} end) diff --git a/mix.exs b/mix.exs index b0b1d27..2257064 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,7 @@ defmodule Instructor.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ecto, "~> 3.12"}, + {:flint, "~> 0.4"}, {:jason, "~> 1.4.0"}, {:req, "~> 0.5 or ~> 1.0"}, {:jaxon, "~> 2.0"}, diff --git a/mix.lock b/mix.lock index a4cdf56..29c42ae 100644 --- a/mix.lock +++ b/mix.lock @@ -2,11 +2,14 @@ "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, - "ecto": {:hex, :ecto, "3.12.1", "626765f7066589de6fa09e0876a253ff60c3d00870dd3a1cd696e2ba67bfceea", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df0045ab9d87be947228e05a8d153f3e06e0d05ab10c3b3cc557d2f7243d1940"}, + "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "flint": {:hex, :flint, "0.4.0", "cb8f202dc7628bbecbba9f89e9c090099361402c8ed57ee85cc14600220de9b7", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:spark, "~> 2.2", [hex: :spark, repo: "hexpm", optional: false]}], "hexpm", "6dc8f1144bf0b16d4587bb1080711d8c6da4db4de5c0b1285592a4a159cce75d"}, + "glob_ex": {:hex, :glob_ex, "0.1.10", "d819a368637495a5c1962ef34f48fe4e9a09032410b96ade5758f2cd1cc5fcde", [:mix], [], "hexpm", "c75357e57d71c85ef8ef7269b6e787dce3f0ff71e585f79a90e4d5477c532b90"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, + "igniter": {:hex, :igniter, "0.3.64", "232801a7acb2de83a01033b2dba855ac1ce7cff28b28a70ed61589f56c2b14f7", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "df5e4c1a49220db1e397dd8b612656215fc187adc9e134b27b773c677be474e5"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, @@ -26,6 +29,10 @@ "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"}, + "rewrite": {:hex, :rewrite, "0.10.5", "6afadeae0b9d843b27ac6225e88e165884875e0aed333ef4ad3bf36f9c101bed", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "51cc347a4269ad3a1e7a2c4122dbac9198302b082f5615964358b4635ebf3d4f"}, + "sourceror": {:hex, :sourceror, "1.6.0", "9907884e1449a4bd7dbaabe95088ed4d9a09c3c791fb0103964e6316bc9448a7", [:mix], [], "hexpm", "e90aef8c82dacf32c89c8ef83d1416fc343cd3e5556773eeffd2c1e3f991f699"}, + "spark": {:hex, :spark, "2.2.35", "1c0bb30f340151eca24164885935de39e6ada4010555f444c813d0488990f8f3", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "f242d6385c287389034a0e146d8f025b5c9ab777f1ae5cf0fdfc9209db6ae748"}, + "spitfire": {:hex, :spitfire, "0.1.3", "7ea0f544005dfbe48e615ed90250c9a271bfe126914012023fd5e4b6b82b7ec7", [:mix], [], "hexpm", "d53b5107bcff526a05c5bb54c95e77b36834550affd5830c9f58760e8c543657"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, From 0a04ec51a201a82a8e1db7391b78cdc9002d821c Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Mon, 21 Oct 2024 15:50:14 -0400 Subject: [PATCH 02/12] implement validation callback in extension --- lib/instructor/instruction.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index f44f9a2..ed07440 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -2,4 +2,19 @@ defmodule Instructor.Instruction do use Flint.Extension option :doc, default: "", validator: &is_binary/1, required: false + + defmacro __using__(_opts) do + quote do + use Instructor.Validator + + @impl true + def validate_changeset(changeset, context \\ %{}) do + __MODULE__ + |> struct!() + |> changeset(changeset, Enum.into(context, [])) + end + + defoverridable validate_changeset: 1, validate_changeset: 2 + end + end end From 7c314e66e1704e0061cff4d7938fcd0a711d8f49 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Mon, 21 Oct 2024 18:15:52 -0400 Subject: [PATCH 03/12] Make Flint optional --- lib/instructor.ex | 6 +++--- lib/instructor/instruction.ex | 28 +++++++++++++++------------- mix.exs | 3 ++- mix.lock | 2 +- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/instructor.ex b/lib/instructor.ex index 50e99bd..975d912 100644 --- a/lib/instructor.ex +++ b/lib/instructor.ex @@ -589,12 +589,12 @@ defmodule Instructor do not is_ecto_schema(response_model) -> changeset - function_exported?(response_model, :validate_changeset, 1) -> - response_model.validate_changeset(changeset) - function_exported?(response_model, :validate_changeset, 2) -> response_model.validate_changeset(changeset, context) + function_exported?(response_model, :validate_changeset, 1) -> + response_model.validate_changeset(changeset) + true -> changeset end diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index ed07440..445fb37 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,20 +1,22 @@ -defmodule Instructor.Instruction do - use Flint.Extension +if Code.ensure_loaded?(Flint.Schema) do + defmodule Instructor.Instruction do + use Flint.Extension - option :doc, default: "", validator: &is_binary/1, required: false + option :doc, default: "", validator: &is_binary/1, required: false - defmacro __using__(_opts) do - quote do - use Instructor.Validator + defmacro __using__(_opts) do + quote do + use Instructor.Validator - @impl true - def validate_changeset(changeset, context \\ %{}) do - __MODULE__ - |> struct!() - |> changeset(changeset, Enum.into(context, [])) - end + @impl true + def validate_changeset(changeset, context \\ %{}) do + __MODULE__ + |> struct!() + |> changeset(changeset, Enum.into(context, [])) + end - defoverridable validate_changeset: 1, validate_changeset: 2 + defoverridable validate_changeset: 1, validate_changeset: 2 + end end end end diff --git a/mix.exs b/mix.exs index 2257064..071cd65 100644 --- a/mix.exs +++ b/mix.exs @@ -111,10 +111,11 @@ defmodule Instructor.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:flint, "~> 0.4"}, + {:ecto, "~> 3.12"}, {:jason, "~> 1.4.0"}, {:req, "~> 0.5 or ~> 1.0"}, {:jaxon, "~> 2.0"}, + {:flint, "~> 0.4", optional: true}, {:ex_doc, "~> 0.31", only: :dev, runtime: false}, {:mox, "~> 1.1.0", only: :test}, {:phoenix, "~> 1.7", only: :test}, diff --git a/mix.lock b/mix.lock index 29c42ae..a1d100e 100644 --- a/mix.lock +++ b/mix.lock @@ -9,7 +9,7 @@ "flint": {:hex, :flint, "0.4.0", "cb8f202dc7628bbecbba9f89e9c090099361402c8ed57ee85cc14600220de9b7", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:spark, "~> 2.2", [hex: :spark, repo: "hexpm", optional: false]}], "hexpm", "6dc8f1144bf0b16d4587bb1080711d8c6da4db4de5c0b1285592a4a159cce75d"}, "glob_ex": {:hex, :glob_ex, "0.1.10", "d819a368637495a5c1962ef34f48fe4e9a09032410b96ade5758f2cd1cc5fcde", [:mix], [], "hexpm", "c75357e57d71c85ef8ef7269b6e787dce3f0ff71e585f79a90e4d5477c532b90"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, - "igniter": {:hex, :igniter, "0.3.64", "232801a7acb2de83a01033b2dba855ac1ce7cff28b28a70ed61589f56c2b14f7", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "df5e4c1a49220db1e397dd8b612656215fc187adc9e134b27b773c677be474e5"}, + "igniter": {:hex, :igniter, "0.3.69", "13f99438f09b96e23ac4154a1c1ff91d2a9ad76ac1422496f42b84e6161d4208", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "a86bc0ae1f2c9c790fd0ad469323b99fdc897302a00ae9aab2a8ae09f39dcbb8"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, From 9a7f8d580d9924c3dfb25d7367da12e7fbcdb85f Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Mon, 21 Oct 2024 18:21:00 -0400 Subject: [PATCH 04/12] keep Ecto formatter since Flint is optional --- .formatter.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.formatter.exs b/.formatter.exs index a265664..76c6572 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,6 +1,6 @@ # Used by "mix format" [ - import_deps: [:flint, :phoenix, :phoenix_live_view], + import_deps: [:ecto, :flint, :phoenix, :phoenix_live_view], plugins: [Phoenix.LiveView.HTMLFormatter], inputs: [ "{mix,.formatter}.exs", From 2f428a003ac6f54f399b1167a4f82d75447af862 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Sat, 16 Nov 2024 01:04:34 -0500 Subject: [PATCH 05/12] update extension --- lib/instructor/instruction.ex | 34 ++++++++++++++++++++++++++++- lib/instructor/sse_stream_parser.ex | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index 445fb37..99bd1c2 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -2,12 +2,44 @@ if Code.ensure_loaded?(Flint.Schema) do defmodule Instructor.Instruction do use Flint.Extension - option :doc, default: "", validator: &is_binary/1, required: false + attribute(:stream, default: false, validator: &is_boolean/1) + # attribute(:validation_context, default: %{}, validator: &is_map/1) + attribute(:mode, default: :tools, validator: &Kernel.in(&1, [:tools, :json, :md_json])) + attribute(:max_retries, default: 0, validator: &is_integer/1) + attribute(:system_prompt, validator: &is_binary/1, default: "") + attribute(:model, validator: &is_binary/1, default: "") + attribute(:array, default: false, validator: &is_boolean/1) + + option(:doc, default: "", validator: &is_binary/1, required: false) defmacro __using__(_opts) do quote do use Instructor.Validator + def chat_completion(messages, opts \\ []) do + opts = + Keyword.validate!(opts, + stream: __MODULE__.__schema__(:stream), + # validation_context: __MODULE__.__schema__(:validation_context, + mode: __MODULE__.__schema__(:mode), + max_retries: __MODULE__.__schema__(:max_retries), + model: __MODULE__.__schema__(:model) + ) + + messages = + if __MODULE__.__schema__(:system_prompt) do + [%{role: "system", content: __MODULE__.__schema__(:system_prompt)} | messages] + else + messages + end + + response_model = + if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ + + opts = ([messages: messages, response_model: response_model] ++ opts) |> IO.inspect() + Instructor.chat_completion(opts) + end + @impl true def validate_changeset(changeset, context \\ %{}) do __MODULE__ diff --git a/lib/instructor/sse_stream_parser.ex b/lib/instructor/sse_stream_parser.ex index a695ecb..a88fb34 100644 --- a/lib/instructor/sse_stream_parser.ex +++ b/lib/instructor/sse_stream_parser.ex @@ -1,4 +1,5 @@ defmodule Instructor.SSEStreamParser do + require Logger @moduledoc false def parse(stream) do @@ -16,6 +17,5 @@ defmodule Instructor.SSEStreamParser do Jason.decode!(json_string) end) end) - # |> Stream.each(&IO.inspect/1) end end From 0db5f984b71b2cc7bb052294736195357bc6a44b Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Wed, 20 Nov 2024 20:23:12 -0500 Subject: [PATCH 06/12] add template attribute --- lib/instructor/instruction.ex | 36 ++++++++++++++++++++++++++++++----- mix.exs | 2 +- mix.lock | 13 +++++++------ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index 99bd1c2..f26ef40 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -3,29 +3,55 @@ if Code.ensure_loaded?(Flint.Schema) do use Flint.Extension attribute(:stream, default: false, validator: &is_boolean/1) - # attribute(:validation_context, default: %{}, validator: &is_map/1) + attribute(:validation_context, default: %{}, validator: &is_map/1) attribute(:mode, default: :tools, validator: &Kernel.in(&1, [:tools, :json, :md_json])) attribute(:max_retries, default: 0, validator: &is_integer/1) - attribute(:system_prompt, validator: &is_binary/1, default: "") - attribute(:model, validator: &is_binary/1, default: "") + attribute(:system_prompt, validator: &is_binary/1) + attribute(:model, validator: &is_binary/1) attribute(:array, default: false, validator: &is_boolean/1) + attribute(:template) option(:doc, default: "", validator: &is_binary/1, required: false) defmacro __using__(_opts) do quote do use Instructor.Validator + require EEx + + def render_template(assigns) do + EEx.eval_string(__MODULE__.__schema__(:template), assigns: assigns) + end def chat_completion(messages, opts \\ []) do opts = Keyword.validate!(opts, stream: __MODULE__.__schema__(:stream), - # validation_context: __MODULE__.__schema__(:validation_context, + validation_context: __MODULE__.__schema__(:validation_context), mode: __MODULE__.__schema__(:mode), max_retries: __MODULE__.__schema__(:max_retries), model: __MODULE__.__schema__(:model) ) + messages = if Keyword.keyword?(messages), do: [messages], else: messages + + messages = + for message <- messages do + case message do + %{role: _role, content: _content} -> + message + + _ -> + %{ + role: "user", + content: + if(__MODULE__.__schema__(:template), + do: render_template(message), + else: message + ) + } + end + end + messages = if __MODULE__.__schema__(:system_prompt) do [%{role: "system", content: __MODULE__.__schema__(:system_prompt)} | messages] @@ -36,7 +62,7 @@ if Code.ensure_loaded?(Flint.Schema) do response_model = if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ - opts = ([messages: messages, response_model: response_model] ++ opts) |> IO.inspect() + opts = ([messages: messages, response_model: response_model] ++ opts) Instructor.chat_completion(opts) end diff --git a/mix.exs b/mix.exs index 071cd65..540cf79 100644 --- a/mix.exs +++ b/mix.exs @@ -115,7 +115,7 @@ defmodule Instructor.MixProject do {:jason, "~> 1.4.0"}, {:req, "~> 0.5 or ~> 1.0"}, {:jaxon, "~> 2.0"}, - {:flint, "~> 0.4", optional: true}, + {:flint, "~> 0.6", optional: true}, {:ex_doc, "~> 0.31", only: :dev, runtime: false}, {:mox, "~> 1.1.0", only: :test}, {:phoenix, "~> 1.7", only: :test}, diff --git a/mix.lock b/mix.lock index a1d100e..9777a8a 100644 --- a/mix.lock +++ b/mix.lock @@ -1,15 +1,15 @@ %{ "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, - "flint": {:hex, :flint, "0.4.0", "cb8f202dc7628bbecbba9f89e9c090099361402c8ed57ee85cc14600220de9b7", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:spark, "~> 2.2", [hex: :spark, repo: "hexpm", optional: false]}], "hexpm", "6dc8f1144bf0b16d4587bb1080711d8c6da4db4de5c0b1285592a4a159cce75d"}, - "glob_ex": {:hex, :glob_ex, "0.1.10", "d819a368637495a5c1962ef34f48fe4e9a09032410b96ade5758f2cd1cc5fcde", [:mix], [], "hexpm", "c75357e57d71c85ef8ef7269b6e787dce3f0ff71e585f79a90e4d5477c532b90"}, + "flint": {:hex, :flint, "0.6.0", "7c67980d90faa7a0d0b62da49c9e1ba8760d08282b883963abdc4816cad44e37", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:spark, "~> 2.2", [hex: :spark, repo: "hexpm", optional: false]}, {:typed_ecto_schema, "~> 0.4", [hex: :typed_ecto_schema, repo: "hexpm", optional: true]}], "hexpm", "712bcb44f3a67889a986769b7070b7605524d72cdd589a5806e344cf1a5231a2"}, + "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, - "igniter": {:hex, :igniter, "0.3.69", "13f99438f09b96e23ac4154a1c1ff91d2a9ad76ac1422496f42b84e6161d4208", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "a86bc0ae1f2c9c790fd0ad469323b99fdc897302a00ae9aab2a8ae09f39dcbb8"}, + "igniter": {:hex, :igniter, "0.4.7", "a91ab006e400f82ae93c1ada7d112cc7e7c1684455428c0de98d6b2e66025b43", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 1.0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "89bc7a093d60c5c70e46014ffd83d993593c1151c8301e3a16f2846695ff22ec"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, @@ -29,11 +29,12 @@ "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"}, - "rewrite": {:hex, :rewrite, "0.10.5", "6afadeae0b9d843b27ac6225e88e165884875e0aed333ef4ad3bf36f9c101bed", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "51cc347a4269ad3a1e7a2c4122dbac9198302b082f5615964358b4635ebf3d4f"}, - "sourceror": {:hex, :sourceror, "1.6.0", "9907884e1449a4bd7dbaabe95088ed4d9a09c3c791fb0103964e6316bc9448a7", [:mix], [], "hexpm", "e90aef8c82dacf32c89c8ef83d1416fc343cd3e5556773eeffd2c1e3f991f699"}, + "rewrite": {:hex, :rewrite, "1.1.1", "0e6674eb5f8cb11aabe5ad6207151b4156bf173aa9b43133a68f8cc882364570", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "fcd688b3ca543c3a1f1f4615ccc054ec37cfcde91133a27a683ec09b35ae1496"}, + "sourceror": {:hex, :sourceror, "1.7.1", "599d78f4cc2be7d55c9c4fd0a8d772fd0478e3a50e726697c20d13d02aa056d4", [:mix], [], "hexpm", "cd6f268fe29fa00afbc535e215158680a0662b357dc784646d7dff28ac65a0fc"}, "spark": {:hex, :spark, "2.2.35", "1c0bb30f340151eca24164885935de39e6ada4010555f444c813d0488990f8f3", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "f242d6385c287389034a0e146d8f025b5c9ab777f1ae5cf0fdfc9209db6ae748"}, "spitfire": {:hex, :spitfire, "0.1.3", "7ea0f544005dfbe48e615ed90250c9a271bfe126914012023fd5e4b6b82b7ec7", [:mix], [], "hexpm", "d53b5107bcff526a05c5bb54c95e77b36834550affd5830c9f58760e8c543657"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, + "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, } From 800df402af8956e6bab60faf22a9667bf30a1d9e Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Mon, 25 Nov 2024 18:48:41 -0500 Subject: [PATCH 07/12] update EctoType --- lib/instructor/ecto_type.ex | 169 +++++++++++++++++++++++++++++++++- lib/instructor/gbnf.ex | 2 + lib/instructor/instruction.ex | 47 ++++++++-- lib/instructor/json_schema.ex | 153 +----------------------------- 4 files changed, 208 insertions(+), 163 deletions(-) diff --git a/lib/instructor/ecto_type.ex b/lib/instructor/ecto_type.ex index 070b200..6c9aa0f 100644 --- a/lib/instructor/ecto_type.ex +++ b/lib/instructor/ecto_type.ex @@ -4,7 +4,7 @@ defmodule Instructor.EctoType do that works natively with Instructor. ## Example - + ```elixir defmodule MyCustomType do use Ecto.Type @@ -22,10 +22,177 @@ defmodule Instructor.EctoType do ``` """ @callback to_json_schema() :: map() + @callback to_json_schema(tuple()) :: map() + + @optional_callbacks to_json_schema: 0, to_json_schema: 1 + + defguard is_ecto_schema(mod) when is_atom(mod) + defguard is_ecto_types(types) when is_map(types) + + def title_for(ecto_schema) when is_ecto_schema(ecto_schema) do + to_string(ecto_schema) |> String.trim_leading("Elixir.") + end + + def for_type(:any), do: %{} + def for_type(:id), do: %{type: "integer", description: "Integer, e.g. 1"} + def for_type(:binary_id), do: %{type: "string"} + def for_type(:integer), do: %{type: "integer", description: "Integer, e.g. 1"} + def for_type(:float), do: %{type: "number", description: "Float, e.g. 1.27", format: "float"} + def for_type(:boolean), do: %{type: "boolean", description: "Boolean, e.g. true"} + def for_type(:string), do: %{type: "string", description: "String, e.g. 'hello'"} + # def for_type(:binary), do: %{type: "unsupported"} + def for_type({:array, type}), do: %{type: "array", items: for_type(type)} + + def for_type(:map), + do: %{ + type: "object", + properties: %{}, + additionalProperties: false, + description: "An object with arbitrary keys and values, e.g. { key: value }" + } + + def for_type({:map, type}), + do: %{ + type: "object", + properties: %{}, + additionalProperties: for_type(type), + description: "An object with values of a type #{inspect(type)}, e.g. { key: value }" + } + + def for_type(:decimal), do: %{type: "number", format: "float"} + + def for_type(:date), + do: %{type: "string", description: "ISO8601 Date, e.g. \"2024-07-20\"", format: "date"} + + def for_type(:time), + do: %{ + type: "string", + description: "ISO8601 Time, e.g. \"12:00:00\"", + pattern: "^[0-9]{2}:?[0-9]{2}:?[0-9]{2}$" + } + + def for_type(:time_usec), + do: %{ + type: "string", + description: "ISO8601 Time with microseconds, e.g. \"12:00:00.000000\"", + pattern: "^[0-9]{2}:?[0-9]{2}:?[0-9]{2}.[0-9]{6}$" + } + + def for_type(:naive_datetime), + do: %{ + type: "string", + description: "ISO8601 DateTime, e.g. \"2024-07-20T12:00:00\"", + format: "date-time" + } + + def for_type(:naive_datetime_usec), + do: %{ + type: "string", + description: "ISO8601 DateTime with microseconds, e.g. \"2024-07-20T12:00:00.000000\"", + format: "date-time" + } + + def for_type(:utc_datetime), + do: %{ + type: "string", + description: "ISO8601 DateTime, e.g. \"2024-07-20T12:00:00Z\"", + format: "date-time" + } + + def for_type(:utc_datetime_usec), + do: %{ + type: "string", + description: "ISO8601 DateTime with microseconds, e.g. \"2024-07-20T12:00:00.000000Z\"", + format: "date-time" + } + + def for_type( + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} + ) + when is_ecto_schema(related) do + title = title_for(related) + + %{ + items: %{"$ref": "#/$defs/#{title}"}, + title: title, + type: "array" + } + end + + def for_type( + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} + ) + when is_ecto_types(related) do + properties = + for {field, type} <- related, into: %{} do + {field, for_type(type)} + end + + required = Map.keys(properties) |> Enum.sort() + + %{ + items: %{ + type: "object", + required: required, + properties: properties + }, + type: "array" + } + end + + def for_type( + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} + ) + when is_ecto_schema(related) do + %{"$ref": "#/$defs/#{title_for(related)}"} + end + + def for_type( + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} + ) + when is_ecto_types(related) do + properties = + for {field, type} <- related, into: %{} do + {field, for_type(type)} + end + + required = Map.keys(properties) |> Enum.sort() + + %{ + type: "object", + required: required, + properties: properties, + additionalProperties: false + } + end + + def for_type({:parameterized, {Ecto.Enum, %{mappings: mappings}}}) do + %{ + type: "string", + enum: Keyword.keys(mappings) + } + end + + def for_type({:parameterized, {mod, params}}) do + if function_exported?(mod, :to_json_schema, 1) do + mod.to_json_schema(params) + else + raise "Unsupported type: #{inspect(mod)}, please implement `to_json_schema/1` via `use Instructor.EctoType`" + end + end + + def for_type(mod) do + if function_exported?(mod, :to_json_schema, 0) do + mod.to_json_schema() + else + raise "Unsupported type: #{inspect(mod)}, please implement `to_json_schema/0` via `use Instructor.EctoType`" + end + end def __using__(_) do quote do @behaviour Instructor.EctoType + import Instructor.EctoType end end end diff --git a/lib/instructor/gbnf.ex b/lib/instructor/gbnf.ex index d5e6aa9..9b073ec 100644 --- a/lib/instructor/gbnf.ex +++ b/lib/instructor/gbnf.ex @@ -39,6 +39,8 @@ defmodule Instructor.GBNF do ws01 ::= ([ \\t\\n])? """ + import Instructor.EctoType + @doc """ Convert a JSONSchema to a GBNF grammar to be used with llama.cpp diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index f26ef40..a286ec8 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,4 +1,17 @@ if Code.ensure_loaded?(Flint.Schema) do + defmodule Instructor.Union do + use Flint.Type, extends: Flint.Types.Union + @behaviour Instructor.EctoType + import Instructor.EctoType + + @impl true + def to_json_schema(%{types: types}) when is_list(types) do + %{ + "oneOf" => Enum.map(types, &Instructor.EctoType.for_type/1) + } + end + end + defmodule Instructor.Instruction do use Flint.Extension @@ -16,21 +29,34 @@ if Code.ensure_loaded?(Flint.Schema) do defmacro __using__(_opts) do quote do use Instructor.Validator - require EEx + alias Instructor.Union def render_template(assigns) do EEx.eval_string(__MODULE__.__schema__(:template), assigns: assigns) end def chat_completion(messages, opts \\ []) do - opts = - Keyword.validate!(opts, - stream: __MODULE__.__schema__(:stream), - validation_context: __MODULE__.__schema__(:validation_context), - mode: __MODULE__.__schema__(:mode), - max_retries: __MODULE__.__schema__(:max_retries), - model: __MODULE__.__schema__(:model) - ) + {stream, opts} = Keyword.pop(opts, :stream, __MODULE__.__schema__(:stream)) + + {validation_context, opts} = + Keyword.pop(opts, :validation_context, __MODULE__.__schema__(:validation_context)) + + {mode, opts} = Keyword.pop(opts, :mode, __MODULE__.__schema__(:mode)) + + {max_retries, opts} = + Keyword.pop(opts, :max_retries, __MODULE__.__schema__(:max_retries)) + + {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) + + settings = + [ + stream: stream, + validation_context: validation_context, + mode: mode, + max_retries: max_retries, + model: model + ] + |> Enum.reject(fn {_k, v} -> is_nil(v) end) messages = if Keyword.keyword?(messages), do: [messages], else: messages @@ -62,7 +88,8 @@ if Code.ensure_loaded?(Flint.Schema) do response_model = if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ - opts = ([messages: messages, response_model: response_model] ++ opts) + opts = [messages: messages, response_model: response_model] ++ settings ++ opts + Instructor.chat_completion(opts) end diff --git a/lib/instructor/json_schema.ex b/lib/instructor/json_schema.ex index 9a0fbf2..c63382e 100644 --- a/lib/instructor/json_schema.ex +++ b/lib/instructor/json_schema.ex @@ -1,6 +1,5 @@ defmodule Instructor.JSONSchema do - defguardp is_ecto_schema(mod) when is_atom(mod) - defguardp is_ecto_types(types) when is_map(types) + import Instructor.EctoType @doc """ Generates a JSON Schema from an Ecto schema. @@ -205,9 +204,6 @@ defmodule Instructor.JSONSchema do [schema | bfs_from_ecto_schema(rest, seen_schemas)] end - defp title_for(ecto_schema) when is_ecto_schema(ecto_schema) do - to_string(ecto_schema) |> String.trim_leading("Elixir.") - end # Find all values in a map or list that match a predicate defp find_all_values(map, pred) when is_map(map) do @@ -234,153 +230,6 @@ defmodule Instructor.JSONSchema do defp find_all_values(_, _pred), do: [] - defp for_type(:id), do: %{type: "integer", description: "Integer, e.g. 1"} - defp for_type(:binary_id), do: %{type: "string"} - defp for_type(:integer), do: %{type: "integer", description: "Integer, e.g. 1"} - defp for_type(:float), do: %{type: "number", description: "Float, e.g. 1.27", format: "float"} - defp for_type(:boolean), do: %{type: "boolean", description: "Boolean, e.g. true"} - defp for_type(:string), do: %{type: "string", description: "String, e.g. 'hello'"} - # defp for_type(:binary), do: %{type: "unsupported"} - defp for_type({:array, type}), do: %{type: "array", items: for_type(type)} - - defp for_type(:map), - do: %{ - type: "object", - properties: %{}, - additionalProperties: false, - description: "An object with arbitrary keys and values, e.g. { key: value }" - } - - defp for_type({:map, type}), - do: %{ - type: "object", - properties: %{}, - additionalProperties: for_type(type), - description: "An object with values of a type #{inspect(type)}, e.g. { key: value }" - } - - defp for_type(:decimal), do: %{type: "number", format: "float"} - - defp for_type(:date), - do: %{type: "string", description: "ISO8601 Date, e.g. \"2024-07-20\"", format: "date"} - - defp for_type(:time), - do: %{ - type: "string", - description: "ISO8601 Time, e.g. \"12:00:00\"", - pattern: "^[0-9]{2}:?[0-9]{2}:?[0-9]{2}$" - } - - defp for_type(:time_usec), - do: %{ - type: "string", - description: "ISO8601 Time with microseconds, e.g. \"12:00:00.000000\"", - pattern: "^[0-9]{2}:?[0-9]{2}:?[0-9]{2}.[0-9]{6}$" - } - - defp for_type(:naive_datetime), - do: %{ - type: "string", - description: "ISO8601 DateTime, e.g. \"2024-07-20T12:00:00\"", - format: "date-time" - } - - defp for_type(:naive_datetime_usec), - do: %{ - type: "string", - description: "ISO8601 DateTime with microseconds, e.g. \"2024-07-20T12:00:00.000000\"", - format: "date-time" - } - - defp for_type(:utc_datetime), - do: %{ - type: "string", - description: "ISO8601 DateTime, e.g. \"2024-07-20T12:00:00Z\"", - format: "date-time" - } - - defp for_type(:utc_datetime_usec), - do: %{ - type: "string", - description: "ISO8601 DateTime with microseconds, e.g. \"2024-07-20T12:00:00.000000Z\"", - format: "date-time" - } - - defp for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} - ) - when is_ecto_schema(related) do - title = title_for(related) - - %{ - items: %{"$ref": "#/$defs/#{title}"}, - title: title, - type: "array" - } - end - - defp for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} - ) - when is_ecto_types(related) do - properties = - for {field, type} <- related, into: %{} do - {field, for_type(type)} - end - - required = Map.keys(properties) |> Enum.sort() - - %{ - items: %{ - type: "object", - required: required, - properties: properties - }, - type: "array" - } - end - - defp for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} - ) - when is_ecto_schema(related) do - %{"$ref": "#/$defs/#{title_for(related)}"} - end - - defp for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} - ) - when is_ecto_types(related) do - properties = - for {field, type} <- related, into: %{} do - {field, for_type(type)} - end - - required = Map.keys(properties) |> Enum.sort() - - %{ - type: "object", - required: required, - properties: properties, - additionalProperties: false - } - end - - defp for_type({:parameterized, {Ecto.Enum, %{mappings: mappings}}}) do - %{ - type: "string", - enum: Keyword.keys(mappings) - } - end - - defp for_type(mod) do - if function_exported?(mod, :to_json_schema, 0) do - mod.to_json_schema() - else - raise "Unsupported type: #{inspect(mod)}, please implement `to_json_schema/0` via `use Instructor.EctoType`" - end - end - @doc """ Traverses a tree structure of maps and lists, allowing the user to update or remove elements. From 9175d04c4e344efc5433a699833b1d1f55a03104 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Tue, 26 Nov 2024 07:51:53 -0500 Subject: [PATCH 08/12] take config keys --- lib/instructor/instruction.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index a286ec8..47e7cdc 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,8 +1,7 @@ -if Code.ensure_loaded?(Flint.Schema) do +if Code.ensure_loaded!(Flint.Schema) do defmodule Instructor.Union do use Flint.Type, extends: Flint.Types.Union @behaviour Instructor.EctoType - import Instructor.EctoType @impl true def to_json_schema(%{types: types}) when is_list(types) do @@ -48,6 +47,8 @@ if Code.ensure_loaded?(Flint.Schema) do {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) + {config, opts} = Keyword.split(opts,[:api_key, :api_url, :http_options]) + settings = [ stream: stream, @@ -90,7 +91,7 @@ if Code.ensure_loaded?(Flint.Schema) do opts = [messages: messages, response_model: response_model] ++ settings ++ opts - Instructor.chat_completion(opts) + Instructor.chat_completion(opts, config) end @impl true From 33aa7152544d96128f54ce0346e843c759985856 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Tue, 26 Nov 2024 22:58:36 -0500 Subject: [PATCH 09/12] update config management --- lib/instructor/adapters/anthropic.ex | 17 ++++++++--------- lib/instructor/adapters/openai.ex | 22 +++++++++++----------- lib/instructor/ecto_type.ex | 24 ++++++++++++------------ lib/instructor/instruction.ex | 4 ++-- mix.exs | 2 +- mix.lock | 2 +- 6 files changed, 35 insertions(+), 36 deletions(-) diff --git a/lib/instructor/adapters/anthropic.ex b/lib/instructor/adapters/anthropic.ex index e21d809..725a8b2 100644 --- a/lib/instructor/adapters/anthropic.ex +++ b/lib/instructor/adapters/anthropic.ex @@ -3,6 +3,10 @@ defmodule Instructor.Adapters.Anthropic do Anthropic adapter for Instructor. """ @behaviour Instructor.Adapter + @default_config [ + api_url: "https://api.anthropic.com/", + http_options: [receive_timeout: 60_000] + ] alias Instructor.SSEStreamParser @@ -131,14 +135,9 @@ defmodule Instructor.Adapters.Anthropic do defp api_key(config), do: Keyword.fetch!(config, :api_key) defp http_options(config), do: Keyword.fetch!(config, :http_options) - defp config(nil), do: config(Application.get_env(:instructor, :anthropic, [])) - - defp config(base_config) do - default_config = [ - api_url: "https://api.anthropic.com/", - http_options: [receive_timeout: 60_000] - ] - - Keyword.merge(default_config, base_config) + defp config(base_config \\ nil) do + @default_config + |> Keyword.merge(Application.get_env(:anthropic, :openai, [])) + |> Keyword.merge(base_config) end end diff --git a/lib/instructor/adapters/openai.ex b/lib/instructor/adapters/openai.ex index 6133074..54f4b27 100644 --- a/lib/instructor/adapters/openai.ex +++ b/lib/instructor/adapters/openai.ex @@ -8,6 +8,13 @@ defmodule Instructor.Adapters.OpenAI do alias Instructor.JSONSchema alias Instructor.SSEStreamParser + @default_config [ + api_url: "https://api.openai.com", + api_path: "/v1/chat/completions", + auth_mode: :bearer, + http_options: [receive_timeout: 60_000] + ] + @impl true def chat_completion(params, user_config \\ nil) do config = config(user_config) @@ -213,16 +220,9 @@ defmodule Instructor.Adapters.OpenAI do defp http_options(config), do: Keyword.fetch!(config, :http_options) - defp config(nil), do: config(Application.get_env(:instructor, :openai, [])) - - defp config(base_config) do - default_config = [ - api_url: "https://api.openai.com", - api_path: "/v1/chat/completions", - auth_mode: :bearer, - http_options: [receive_timeout: 60_000] - ] - - Keyword.merge(default_config, base_config) + defp config(base_config \\ nil) do + @default_config + |> Keyword.merge(Application.get_env(:instructor, :openai, [])) + |> Keyword.merge(base_config) end end diff --git a/lib/instructor/ecto_type.ex b/lib/instructor/ecto_type.ex index 6c9aa0f..273ec13 100644 --- a/lib/instructor/ecto_type.ex +++ b/lib/instructor/ecto_type.ex @@ -107,9 +107,9 @@ defmodule Instructor.EctoType do } def for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} - ) - when is_ecto_schema(related) do + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} + ) + when is_ecto_schema(related) do title = title_for(related) %{ @@ -120,9 +120,9 @@ defmodule Instructor.EctoType do end def for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} - ) - when is_ecto_types(related) do + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :many, related: related}}} + ) + when is_ecto_types(related) do properties = for {field, type} <- related, into: %{} do {field, for_type(type)} @@ -141,16 +141,16 @@ defmodule Instructor.EctoType do end def for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} - ) - when is_ecto_schema(related) do + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} + ) + when is_ecto_schema(related) do %{"$ref": "#/$defs/#{title_for(related)}"} end def for_type( - {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} - ) - when is_ecto_types(related) do + {:parameterized, {Ecto.Embedded, %Ecto.Embedded{cardinality: :one, related: related}}} + ) + when is_ecto_types(related) do properties = for {field, type} <- related, into: %{} do {field, for_type(type)} diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index 47e7cdc..3fedbd7 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,4 +1,4 @@ -if Code.ensure_loaded!(Flint.Schema) do +if Code.ensure_loaded?(Flint.Schema) do defmodule Instructor.Union do use Flint.Type, extends: Flint.Types.Union @behaviour Instructor.EctoType @@ -47,7 +47,7 @@ if Code.ensure_loaded!(Flint.Schema) do {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) - {config, opts} = Keyword.split(opts,[:api_key, :api_url, :http_options]) + {config, opts} = Keyword.split(opts, [:api_key, :api_url, :http_options]) settings = [ diff --git a/mix.exs b/mix.exs index 540cf79..c042f9c 100644 --- a/mix.exs +++ b/mix.exs @@ -115,7 +115,7 @@ defmodule Instructor.MixProject do {:jason, "~> 1.4.0"}, {:req, "~> 0.5 or ~> 1.0"}, {:jaxon, "~> 2.0"}, - {:flint, "~> 0.6", optional: true}, + {:flint, github: "acalejos/flint"}, {:ex_doc, "~> 0.31", only: :dev, runtime: false}, {:mox, "~> 1.1.0", only: :test}, {:phoenix, "~> 1.7", only: :test}, diff --git a/mix.lock b/mix.lock index 9777a8a..917caf4 100644 --- a/mix.lock +++ b/mix.lock @@ -6,7 +6,7 @@ "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, - "flint": {:hex, :flint, "0.6.0", "7c67980d90faa7a0d0b62da49c9e1ba8760d08282b883963abdc4816cad44e37", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:spark, "~> 2.2", [hex: :spark, repo: "hexpm", optional: false]}, {:typed_ecto_schema, "~> 0.4", [hex: :typed_ecto_schema, repo: "hexpm", optional: true]}], "hexpm", "712bcb44f3a67889a986769b7070b7605524d72cdd589a5806e344cf1a5231a2"}, + "flint": {:git, "https://github.com/acalejos/flint.git", "8c0a01ee1aa17e4f4f0cafbcccbd8d911915b59b", []}, "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, "igniter": {:hex, :igniter, "0.4.7", "a91ab006e400f82ae93c1ada7d112cc7e7c1684455428c0de98d6b2e66025b43", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 1.0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "89bc7a093d60c5c70e46014ffd83d993593c1151c8301e3a16f2846695ff22ec"}, From 17c3bb829ba3744deac4fda648c5c01af49fad29 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Tue, 7 Jan 2025 20:35:57 -0500 Subject: [PATCH 10/12] improve Instruction extension --- .formatter.exs | 3 +- lib/instructor/adapters/anthropic.ex | 2 +- lib/instructor/adapters/openai.ex | 2 +- lib/instructor/ecto_type.ex | 1 + lib/instructor/instruction.ex | 231 ++++++++++++++++----------- lib/instructor/json_schema.ex | 1 - mix.exs | 1 - pages/cookbook/o1_cot_ui.exs | 1 + test/sse_stream_parser_test.exs | 21 ++- 9 files changed, 152 insertions(+), 111 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index 76c6572..800c4e1 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,7 +1,6 @@ # Used by "mix format" [ - import_deps: [:ecto, :flint, :phoenix, :phoenix_live_view], - plugins: [Phoenix.LiveView.HTMLFormatter], + import_deps: [:flint], inputs: [ "{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", diff --git a/lib/instructor/adapters/anthropic.ex b/lib/instructor/adapters/anthropic.ex index 725a8b2..5e6a764 100644 --- a/lib/instructor/adapters/anthropic.ex +++ b/lib/instructor/adapters/anthropic.ex @@ -138,6 +138,6 @@ defmodule Instructor.Adapters.Anthropic do defp config(base_config \\ nil) do @default_config |> Keyword.merge(Application.get_env(:anthropic, :openai, [])) - |> Keyword.merge(base_config) + |> Keyword.merge(base_config || []) end end diff --git a/lib/instructor/adapters/openai.ex b/lib/instructor/adapters/openai.ex index 54f4b27..6657773 100644 --- a/lib/instructor/adapters/openai.ex +++ b/lib/instructor/adapters/openai.ex @@ -223,6 +223,6 @@ defmodule Instructor.Adapters.OpenAI do defp config(base_config \\ nil) do @default_config |> Keyword.merge(Application.get_env(:instructor, :openai, [])) - |> Keyword.merge(base_config) + |> Keyword.merge(base_config || []) end end diff --git a/lib/instructor/ecto_type.ex b/lib/instructor/ecto_type.ex index 273ec13..c4fbb47 100644 --- a/lib/instructor/ecto_type.ex +++ b/lib/instructor/ecto_type.ex @@ -34,6 +34,7 @@ defmodule Instructor.EctoType do end def for_type(:any), do: %{} + def for_type(nil), do: %{type: "null"} def for_type(:id), do: %{type: "integer", description: "Integer, e.g. 1"} def for_type(:binary_id), do: %{type: "string"} def for_type(:integer), do: %{type: "integer", description: "Integer, e.g. 1"} diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index 3fedbd7..b61ca2f 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,108 +1,151 @@ -if Code.ensure_loaded?(Flint.Schema) do - defmodule Instructor.Union do - use Flint.Type, extends: Flint.Types.Union - @behaviour Instructor.EctoType - - @impl true - def to_json_schema(%{types: types}) when is_list(types) do - %{ - "oneOf" => Enum.map(types, &Instructor.EctoType.for_type/1) - } - end +defmodule Instructor.Union do + use Flint.Type, extends: Flint.Types.Union + @behaviour Instructor.EctoType + + @impl true + def to_json_schema(%{types: types}) when is_list(types) do + %{ + "oneOf" => Enum.map(types, &Instructor.EctoType.for_type/1) + } end +end - defmodule Instructor.Instruction do - use Flint.Extension - - attribute(:stream, default: false, validator: &is_boolean/1) - attribute(:validation_context, default: %{}, validator: &is_map/1) - attribute(:mode, default: :tools, validator: &Kernel.in(&1, [:tools, :json, :md_json])) - attribute(:max_retries, default: 0, validator: &is_integer/1) - attribute(:system_prompt, validator: &is_binary/1) - attribute(:model, validator: &is_binary/1) - attribute(:array, default: false, validator: &is_boolean/1) - attribute(:template) - - option(:doc, default: "", validator: &is_binary/1, required: false) - - defmacro __using__(_opts) do - quote do - use Instructor.Validator - alias Instructor.Union - - def render_template(assigns) do - EEx.eval_string(__MODULE__.__schema__(:template), assigns: assigns) +defmodule Instructor.Instruction do + use Flint.Extension + + attribute :stream, default: false, validator: &is_boolean/1 + attribute :validation_context, default: %{}, validator: &is_map/1 + attribute :mode, default: :tools, validator: &Kernel.in(&1, [:tools, :json, :md_json]) + attribute :max_retries, default: 0, validator: &is_integer/1 + attribute :system_prompt, validator: &is_binary/1 + attribute :model, validator: &is_binary/1 + attribute :array, default: false, validator: &is_boolean/1 + attribute :template + + option :doc, default: "", validator: &is_binary/1, required: false + option :llm_verify, required: false + + @impl true + def changeset(changeset, bindings \\ []) do + module = changeset.data.__struct__ + env = Module.concat(module, Env) |> apply(:env, []) + + quoted_statements = + module.__schema__(:extra_options) + |> Enum.map(fn {field, opts} -> {field, Keyword.get(opts, :llm_verify)} end) + |> Enum.reject(fn + {_k, nil} -> true + other -> false + end) + + for {field, quoted_statement} <- quoted_statements, reduce: changeset do + changeset -> + bindings = bindings ++ Enum.into(changeset.changes, []) + + case eval_quoted(quoted_statement, bindings, env) do + {:ok, {<>, _bindings}} -> + model = bindings[:model] || module.__schema__(:model) + opts = if model, do: [model: model], else: [] + Instructor.Validator.validate_with_llm(changeset, field, statement, opts) + + {:ok, {_other, _bindings}} -> + raise ArgumentError, + "Expression for `:llm_verify` in field #{inspect(field)} must return a binary!" + + _ -> + raise ArgumentError, + "Failed to evaluate expression for option `:llm_verify` in field #{inspect(field)}" end + end + end - def chat_completion(messages, opts \\ []) do - {stream, opts} = Keyword.pop(opts, :stream, __MODULE__.__schema__(:stream)) - - {validation_context, opts} = - Keyword.pop(opts, :validation_context, __MODULE__.__schema__(:validation_context)) - - {mode, opts} = Keyword.pop(opts, :mode, __MODULE__.__schema__(:mode)) - - {max_retries, opts} = - Keyword.pop(opts, :max_retries, __MODULE__.__schema__(:max_retries)) - - {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) - - {config, opts} = Keyword.split(opts, [:api_key, :api_url, :http_options]) - - settings = - [ - stream: stream, - validation_context: validation_context, - mode: mode, - max_retries: max_retries, - model: model - ] - |> Enum.reject(fn {_k, v} -> is_nil(v) end) - - messages = if Keyword.keyword?(messages), do: [messages], else: messages - - messages = - for message <- messages do - case message do - %{role: _role, content: _content} -> - message - - _ -> - %{ - role: "user", - content: - if(__MODULE__.__schema__(:template), - do: render_template(message), - else: message - ) - } - end - end + defmacro __using__(_opts) do + quote do + use Instructor.Validator + alias Instructor.Union - messages = - if __MODULE__.__schema__(:system_prompt) do - [%{role: "system", content: __MODULE__.__schema__(:system_prompt)} | messages] - else - messages - end + def render_template(assigns) do + EEx.eval_string(__MODULE__.__schema__(:template), assigns: assigns) + end - response_model = - if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ + def chat_completion(messages, opts \\ []) do + {stream, opts} = Keyword.pop(opts, :stream, __MODULE__.__schema__(:stream)) + + {validation_context, messages, opts} = + cond do + Keyword.has_key?(opts, :validation_context) -> + {validation_context, opts} = Keyword.pop!(opts, :validation_context) + {validation_context, messages, opts} + + Keyword.has_key?(messages, :validation_context) -> + {validation_context, messages} = Keyword.pop!(messages, :validation_context) + {validation_context, messages, opts} + + true -> + {__MODULE__.__schema__(:validation_context), messages, opts} + end + + {mode, opts} = Keyword.pop(opts, :mode, __MODULE__.__schema__(:mode)) + + {max_retries, opts} = + Keyword.pop(opts, :max_retries, __MODULE__.__schema__(:max_retries)) + + {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) + + {config, opts} = Keyword.split(opts, [:api_key, :api_url, :http_options]) + + settings = + [ + stream: stream, + validation_context: validation_context, + mode: mode, + max_retries: max_retries, + model: model + ] + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + + messages = if Keyword.keyword?(messages), do: [messages], else: messages + + messages = + for message <- messages do + case message do + %{role: _role, content: _content} -> + message + + _ -> + %{ + role: "user", + content: + if(__MODULE__.__schema__(:template), + do: render_template(message), + else: message + ) + } + end + end - opts = [messages: messages, response_model: response_model] ++ settings ++ opts + messages = + if __MODULE__.__schema__(:system_prompt) do + [%{role: "system", content: __MODULE__.__schema__(:system_prompt)} | messages] + else + messages + end - Instructor.chat_completion(opts, config) - end + response_model = + if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ - @impl true - def validate_changeset(changeset, context \\ %{}) do - __MODULE__ - |> struct!() - |> changeset(changeset, Enum.into(context, [])) - end + opts = [messages: messages, response_model: response_model] ++ settings ++ opts + Instructor.chat_completion(opts, config) + end - defoverridable validate_changeset: 1, validate_changeset: 2 + @impl true + def validate_changeset(changeset, context \\ %{}) do + __MODULE__ + |> struct!() + |> changeset(changeset, Enum.into(context, [])) end + + defoverridable validate_changeset: 1, validate_changeset: 2 end end end diff --git a/lib/instructor/json_schema.ex b/lib/instructor/json_schema.ex index c63382e..509c53b 100644 --- a/lib/instructor/json_schema.ex +++ b/lib/instructor/json_schema.ex @@ -204,7 +204,6 @@ defmodule Instructor.JSONSchema do [schema | bfs_from_ecto_schema(rest, seen_schemas)] end - # Find all values in a map or list that match a predicate defp find_all_values(map, pred) when is_map(map) do map diff --git a/mix.exs b/mix.exs index c042f9c..9d49b90 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,6 @@ defmodule Instructor.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ecto, "~> 3.12"}, {:jason, "~> 1.4.0"}, {:req, "~> 0.5 or ~> 1.0"}, {:jaxon, "~> 2.0"}, diff --git a/pages/cookbook/o1_cot_ui.exs b/pages/cookbook/o1_cot_ui.exs index 8e63228..aa15701 100644 --- a/pages/cookbook/o1_cot_ui.exs +++ b/pages/cookbook/o1_cot_ui.exs @@ -47,6 +47,7 @@ defmodule DemoLive do pid = self() selected_model = socket.assigns.selected_model + response_model = socket.assigns.output_schema |> Enum.map(fn {key, type} -> {String.to_atom(key), type} end) diff --git a/test/sse_stream_parser_test.exs b/test/sse_stream_parser_test.exs index 17b9bc5..7ba0284 100644 --- a/test/sse_stream_parser_test.exs +++ b/test/sse_stream_parser_test.exs @@ -7,15 +7,14 @@ defmodule Instructor.SSEStreamParserTest do tokens = [ "data: { \"number\": 1 }\n", "data: { \"number\": 2 }\n", - "data: { \"number\": 3 }\n", + "data: { \"number\": 3 }\n" ] - assert SSEStreamParser.parse(tokens) |> Enum.to_list() == [ - %{"number" => 1}, - %{"number" => 2}, - %{"number" => 3} - ] + %{"number" => 1}, + %{"number" => 2}, + %{"number" => 3} + ] end test "parses a stream where lines are split across chunks" do @@ -23,13 +22,13 @@ defmodule Instructor.SSEStreamParserTest do "data: { \"number\": 1 }\n", "data: { \"number\":", " 2 }\n", - "data: { \"number\": 3 }\n", + "data: { \"number\": 3 }\n" ] assert SSEStreamParser.parse(tokens) |> Enum.to_list() == [ - %{"number" => 1}, - %{"number" => 2}, - %{"number" => 3} - ] + %{"number" => 1}, + %{"number" => 2}, + %{"number" => 3} + ] end end From f12a4a181669aef1ef1dc0a23465aa4e607051b5 Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Thu, 9 Jan 2025 00:39:38 -0500 Subject: [PATCH 11/12] passthrough regex pattern to JSON schema --- lib/instructor/extras/chain_of_thought.ex | 1 - lib/instructor/json_schema.ex | 26 ++++++++++++++++++++++- pages/cookbook/streaming_ui.exs | 1 - 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/instructor/extras/chain_of_thought.ex b/lib/instructor/extras/chain_of_thought.ex index d8a47ef..800645c 100644 --- a/lib/instructor/extras/chain_of_thought.ex +++ b/lib/instructor/extras/chain_of_thought.ex @@ -98,7 +98,6 @@ defmodule Instructor.Extras.ChainOfThought do {[step], acc} {:error, reason} -> - IO.inspect(reason, label: "ERROR") {:halt, {params, step_count}} end end, diff --git a/lib/instructor/json_schema.ex b/lib/instructor/json_schema.ex index 509c53b..37eec66 100644 --- a/lib/instructor/json_schema.ex +++ b/lib/instructor/json_schema.ex @@ -85,16 +85,30 @@ defmodule Instructor.JSONSchema do [] end + field_patterns = + try do + ecto_schema.__schema__(:extra_options) + |> Enum.map(fn {field, opts} -> + {field, Keyword.get(opts, :format)} + end) + rescue + _ -> + [] + end + properties = ecto_schema.__schema__(:fields) |> Enum.map(fn field -> type = ecto_schema.__schema__(:type, field) field_doc = Keyword.get(field_docs, field, "") |> String.trim() + field_pattern = Keyword.get(field_patterns, field) value = for_type(type) value = Map.merge(%{title: Atom.to_string(field)}, value) - |> Map.update(:description, field_doc, fn desc -> + + value = if field_doc != "" do + Map.update(value, :description, field_doc, fn desc -> field_doc = cond do field_doc == "" -> @@ -109,6 +123,16 @@ defmodule Instructor.JSONSchema do field_doc <> desc end) + else + value + end + + value = + if type == :string && match?(%Regex{}, field_pattern) do + Map.put(value, :pattern, Regex.source(field_pattern)) + else + value + end {field, value} end) diff --git a/pages/cookbook/streaming_ui.exs b/pages/cookbook/streaming_ui.exs index abdb17c..2b890ea 100644 --- a/pages/cookbook/streaming_ui.exs +++ b/pages/cookbook/streaming_ui.exs @@ -114,7 +114,6 @@ defmodule StreamingUILive do image_url {:error, error} -> - IO.inspect(error) nil end end From 60f33640bd9038a1d637db12a35e790edf1a21fe Mon Sep 17 00:00:00 2001 From: Andres Alejos Date: Tue, 14 Jan 2025 22:59:40 -0500 Subject: [PATCH 12/12] lots of changes --- lib/instructor.ex | 99 ++++++++++++---- lib/instructor/adapters/anthropic.ex | 2 +- lib/instructor/adapters/openai.ex | 26 ++--- lib/instructor/ecto_type.ex | 7 +- lib/instructor/extras/chain_of_thought.ex | 2 +- lib/instructor/instruction.ex | 134 +++++++++++++--------- lib/instructor/json_schema.ex | 51 ++++---- lib/instructor/types/union.ex | 13 +++ mix.lock | 43 +++---- test/json_schema_test.exs | 3 +- 10 files changed, 234 insertions(+), 146 deletions(-) create mode 100644 lib/instructor/types/union.ex diff --git a/lib/instructor.ex b/lib/instructor.ex index 975d912..f4e7e51 100644 --- a/lib/instructor.ex +++ b/lib/instructor.ex @@ -32,6 +32,8 @@ defmodule Instructor do * `:mode` - The mode to use when parsing the response, :tools, :json, :md_json (defaults to `:tools`), generally speaking you don't need to change this unless you are not using OpenAI. * `:max_retries` - The maximum number of times to retry the LLM call if it fails, or does not pass validations. (defaults to `0`) + * `:after_request` - A callback function that will take in the current `%Ecto.Changeset{}` along with the raw `%Req.Response{}` returned + from the adapter. This can be helpful with tracking metadata or usage information, such as rate-limit headers and token usage. ## Examples @@ -130,22 +132,38 @@ defmodule Instructor do case {response_model, is_stream} do {{:partial, {:array, response_model}}, true} -> - do_streaming_partial_array_chat_completion(response_model, params, config) + do_streaming_partial_array_chat_completion( + response_model.__struct__() |> Ecto.Changeset.change(), + params, + config + ) {{:partial, response_model}, true} -> - do_streaming_partial_chat_completion(response_model, params, config) + do_streaming_partial_chat_completion( + response_model.__struct__() |> Ecto.Changeset.change(), + params, + config + ) {{:array, response_model}, true} -> - do_streaming_array_chat_completion(response_model, params, config) + do_streaming_array_chat_completion( + response_model.__struct__() |> Ecto.Changeset.change(), + params, + config + ) {{:array, response_model}, false} -> params = Keyword.put(params, :stream, true) - do_streaming_array_chat_completion(response_model, params, config) + do_streaming_array_chat_completion( + response_model.__struct__() |> Ecto.Changeset.change(), + params, + config + ) |> Enum.to_list() {response_model, false} -> - do_chat_completion(response_model, params, config) + do_chat_completion(response_model.__struct__() |> Ecto.Changeset.change(), params, config) {_, true} -> raise """ @@ -234,6 +252,38 @@ defmodule Instructor do |> Ecto.Changeset.validate_required(fields) end + def cast_all(%Ecto.Changeset{data: data} = changeset, params) do + response_model = data.__struct__ + fields = response_model.__schema__(:fields) |> MapSet.new() + embedded_fields = response_model.__schema__(:embeds) |> MapSet.new() + associated_fields = response_model.__schema__(:associations) |> MapSet.new() + + fields = + fields + |> MapSet.difference(embedded_fields) + |> MapSet.difference(associated_fields) + + changeset = + changeset + |> Ecto.Changeset.cast(params, fields |> MapSet.to_list()) + + changeset = + for field <- embedded_fields, reduce: changeset do + changeset -> + changeset + |> Ecto.Changeset.cast_embed(field, with: &cast_all/2) + end + + changeset = + for field <- associated_fields, reduce: changeset do + changeset -> + changeset + |> Ecto.Changeset.cast_assoc(field, with: &cast_all/2) + end + + changeset + end + def cast_all(schema, params) do response_model = schema.__struct__ fields = response_model.__schema__(:fields) |> MapSet.new() @@ -415,27 +465,29 @@ defmodule Instructor do end) end - defp do_chat_completion(response_model, params, config) do + defp do_chat_completion( + %Ecto.Changeset{data: %{__struct__: response_model}} = changeset, + params, + config + ) do + after_request = Keyword.get(params, :after_request, fn c, _r -> c end) validation_context = Keyword.get(params, :validation_context, %{}) max_retries = Keyword.get(params, :max_retries) mode = Keyword.get(params, :mode, :tools) params = params_for_mode(mode, response_model, params) - model = - if is_ecto_schema(response_model) do - response_model.__struct__() - else - {%{}, response_model} - end - - with {:ok, raw_response, params} <- do_adapter_chat_completion(params, config), - {%Ecto.Changeset{valid?: true} = changeset, raw_response} <- - {cast_all(model, params), raw_response}, - {%Ecto.Changeset{valid?: true} = changeset, _raw_response} <- - {call_validate(response_model, changeset, validation_context), raw_response} do + with {:ok, {_raw_request, _raw_response} = result, params} <- + do_adapter_chat_completion(Keyword.drop(params, [:after_request]), config), + changeset = after_request.(changeset, result), + {%Ecto.Changeset{valid?: true} = changeset, result} <- + {cast_all(changeset, params), result}, + {%Ecto.Changeset{valid?: true} = changeset, _result} <- + {call_validate(response_model, changeset, validation_context), result} do {:ok, changeset |> Ecto.Changeset.apply_changes()} else - {%Ecto.Changeset{} = changeset, raw_response} -> + {%Ecto.Changeset{} = changeset, {_raw_request, raw_response} = result} -> + changeset = after_request.(changeset, result) + if max_retries > 0 do errors = Instructor.ErrorFormatter.format_errors(changeset) @@ -461,7 +513,7 @@ defmodule Instructor do ] end) - do_chat_completion(response_model, params, config) + do_chat_completion(changeset, params, config) else {:error, changeset} end @@ -535,10 +587,7 @@ defmodule Instructor do :json -> [sys_message | messages] - :json_schema -> - messages - - :tools -> + m when m in [:json_schema, :tools, :structured_output] -> messages end end) @@ -553,7 +602,7 @@ defmodule Instructor do type: "json_object" }) - :json_schema -> + m when m in [:json_schema, :structured_output] -> params |> Keyword.put(:response_format, %{ type: "json_schema", diff --git a/lib/instructor/adapters/anthropic.ex b/lib/instructor/adapters/anthropic.ex index 5e6a764..bb5c50b 100644 --- a/lib/instructor/adapters/anthropic.ex +++ b/lib/instructor/adapters/anthropic.ex @@ -135,7 +135,7 @@ defmodule Instructor.Adapters.Anthropic do defp api_key(config), do: Keyword.fetch!(config, :api_key) defp http_options(config), do: Keyword.fetch!(config, :http_options) - defp config(base_config \\ nil) do + defp config(base_config) do @default_config |> Keyword.merge(Application.get_env(:anthropic, :openai, [])) |> Keyword.merge(base_config || []) diff --git a/lib/instructor/adapters/openai.ex b/lib/instructor/adapters/openai.ex index 6657773..c9f880f 100644 --- a/lib/instructor/adapters/openai.ex +++ b/lib/instructor/adapters/openai.ex @@ -3,7 +3,7 @@ defmodule Instructor.Adapters.OpenAI do Documentation for `Instructor.Adapters.OpenAI`. """ @behaviour Instructor.Adapter - @supported_modes [:tools, :json, :md_json, :json_schema] + @supported_modes [:tools, :json, :md_json, :json_schema, :structured_output] alias Instructor.JSONSchema alias Instructor.SSEStreamParser @@ -31,10 +31,11 @@ defmodule Instructor.Adapters.OpenAI do raise "Unsupported OpenAI mode #{mode}. Supported modes: #{inspect(@supported_modes)}" end + # TODO: Only do this when `strict: true` params = case params do # OpenAI's json_schema mode doesn't support format or pattern attributes - %{"response_format" => %{"json_schema" => %{"schema" => _schema}}} -> + %{response_format: %{json_schema: %{schema: _schema}}} -> update_in(params, [:response_format, :json_schema, :schema], fn schema -> JSONSchema.traverse_and_update(schema, fn %{"type" => _} = x when is_map_key(x, "format") or is_map_key(x, "pattern") -> @@ -132,12 +133,12 @@ defmodule Instructor.Adapters.OpenAI do defp do_chat_completion(mode, params, config) do options = Keyword.merge(http_options(config), [auth_header(config), json: params]) - with {:ok, %Req.Response{status: 200, body: body} = response} <- - Req.post(url(config), options), + with {%Req.Request{}, %Req.Response{status: 200, body: body}} = result <- + Req.run(url(config), [{:method, :post} | options]), {:ok, content} <- parse_response_for_mode(mode, body) do - {:ok, response, content} + {:ok, result, content} else - {:ok, %Req.Response{status: status, body: body}} -> + {%Req.Request{}, %Req.Response{status: status, body: body}} -> {:error, "Unexpected HTTP response code: #{status}\n#{inspect(body)}"} e -> @@ -152,15 +153,8 @@ defmodule Instructor.Adapters.OpenAI do }), do: Jason.decode(args) - defp parse_response_for_mode(:md_json, %{"choices" => [%{"message" => %{"content" => content}}]}), - do: Jason.decode(content) - - defp parse_response_for_mode(:json, %{"choices" => [%{"message" => %{"content" => content}}]}), - do: Jason.decode(content) - - defp parse_response_for_mode(:json_schema, %{ - "choices" => [%{"message" => %{"content" => content}}] - }), + defp parse_response_for_mode(mode, %{"choices" => [%{"message" => %{"content" => content}}]}) + when mode in [:md_json, :json, :json_schema, :structured_output], do: Jason.decode(content) defp parse_response_for_mode(mode, response) do @@ -220,7 +214,7 @@ defmodule Instructor.Adapters.OpenAI do defp http_options(config), do: Keyword.fetch!(config, :http_options) - defp config(base_config \\ nil) do + defp config(base_config) do @default_config |> Keyword.merge(Application.get_env(:instructor, :openai, [])) |> Keyword.merge(base_config || []) diff --git a/lib/instructor/ecto_type.ex b/lib/instructor/ecto_type.ex index c4fbb47..61eb40a 100644 --- a/lib/instructor/ecto_type.ex +++ b/lib/instructor/ecto_type.ex @@ -115,7 +115,8 @@ defmodule Instructor.EctoType do %{ items: %{"$ref": "#/$defs/#{title}"}, - title: title, + # From OpenAI error: $ref cannot have keywords {'title'} + # title: title, type: "array" } end @@ -175,7 +176,7 @@ defmodule Instructor.EctoType do end def for_type({:parameterized, {mod, params}}) do - if function_exported?(mod, :to_json_schema, 1) do + if Code.ensure_loaded?(mod) && function_exported?(mod, :to_json_schema, 1) do mod.to_json_schema(params) else raise "Unsupported type: #{inspect(mod)}, please implement `to_json_schema/1` via `use Instructor.EctoType`" @@ -183,7 +184,7 @@ defmodule Instructor.EctoType do end def for_type(mod) do - if function_exported?(mod, :to_json_schema, 0) do + if Code.ensure_loaded?(mod) && function_exported?(mod, :to_json_schema, 0) do mod.to_json_schema() else raise "Unsupported type: #{inspect(mod)}, please implement `to_json_schema/0` via `use Instructor.EctoType`" diff --git a/lib/instructor/extras/chain_of_thought.ex b/lib/instructor/extras/chain_of_thought.ex index 800645c..373c750 100644 --- a/lib/instructor/extras/chain_of_thought.ex +++ b/lib/instructor/extras/chain_of_thought.ex @@ -97,7 +97,7 @@ defmodule Instructor.Extras.ChainOfThought do {[step], acc} - {:error, reason} -> + {:error, _reason} -> {:halt, {params, step_count}} end end, diff --git a/lib/instructor/instruction.ex b/lib/instructor/instruction.ex index b61ca2f..641cd5e 100644 --- a/lib/instructor/instruction.ex +++ b/lib/instructor/instruction.ex @@ -1,21 +1,13 @@ -defmodule Instructor.Union do - use Flint.Type, extends: Flint.Types.Union - @behaviour Instructor.EctoType - - @impl true - def to_json_schema(%{types: types}) when is_list(types) do - %{ - "oneOf" => Enum.map(types, &Instructor.EctoType.for_type/1) - } - end -end - defmodule Instructor.Instruction do use Flint.Extension attribute :stream, default: false, validator: &is_boolean/1 attribute :validation_context, default: %{}, validator: &is_map/1 - attribute :mode, default: :tools, validator: &Kernel.in(&1, [:tools, :json, :md_json]) + + attribute :mode, + default: :structured_output, + validator: &Kernel.in(&1, [:tools, :json, :md_json, :structured_output, :json_schema]) + attribute :max_retries, default: 0, validator: &is_integer/1 attribute :system_prompt, validator: &is_binary/1 attribute :model, validator: &is_binary/1 @@ -35,7 +27,7 @@ defmodule Instructor.Instruction do |> Enum.map(fn {field, opts} -> {field, Keyword.get(opts, :llm_verify)} end) |> Enum.reject(fn {_k, nil} -> true - other -> false + _ -> false end) for {field, quoted_statement} <- quoted_statements, reduce: changeset do @@ -59,48 +51,81 @@ defmodule Instructor.Instruction do end end - defmacro __using__(_opts) do + def pop_from_any(keywords, key, default \\ nil) do + index = Enum.find_index(keywords, &Keyword.has_key?(&1, key)) + + if is_nil(index) do + {default, keywords} + else + {value, new_kw_list} = Enum.at(keywords, index) |> Keyword.pop!(key) + {value, List.replace_at(keywords, index, new_kw_list)} + end + end + + defmacro __using__(opts) do + template_engine = Keyword.get(opts, :template_engine) |> Macro.expand_literals(__CALLER__) + quote do use Instructor.Validator alias Instructor.Union - def render_template(assigns) do - EEx.eval_string(__MODULE__.__schema__(:template), assigns: assigns) + def render_template(template, assigns) do + case unquote(template_engine) do + nil -> + EEx.eval_string(template, assigns: assigns) + + {mod, fun} = engine -> + apply(mod, fun, [template, assigns]) + end end def chat_completion(messages, opts \\ []) do - {stream, opts} = Keyword.pop(opts, :stream, __MODULE__.__schema__(:stream)) - - {validation_context, messages, opts} = - cond do - Keyword.has_key?(opts, :validation_context) -> - {validation_context, opts} = Keyword.pop!(opts, :validation_context) - {validation_context, messages, opts} - - Keyword.has_key?(messages, :validation_context) -> - {validation_context, messages} = Keyword.pop!(messages, :validation_context) - {validation_context, messages, opts} - - true -> - {__MODULE__.__schema__(:validation_context), messages, opts} - end - - {mode, opts} = Keyword.pop(opts, :mode, __MODULE__.__schema__(:mode)) - - {max_retries, opts} = - Keyword.pop(opts, :max_retries, __MODULE__.__schema__(:max_retries)) - - {model, opts} = Keyword.pop(opts, :model, __MODULE__.__schema__(:model)) + {[messages, opts], params} = + Enum.reduce( + [ + :stream, + :validation_context, + :template, + :mode, + :max_retries, + :model, + :system_prompt, + :array, + :api_key, + :api_url, + :http_options, + :adapter, + :after_request + ], + {[messages, opts], []}, + fn key, {kwords, bindings} -> + default = + case __MODULE__.__schema__(key) do + {:error, _} -> + nil + + other -> + other + end + + {value, kwords} = + Instructor.Instruction.pop_from_any(kwords, key, default) + + {kwords, [{key, value} | bindings]} + end + ) - {config, opts} = Keyword.split(opts, [:api_key, :api_url, :http_options]) + config = + Keyword.take(params, [:api_key, :api_url, :http_options, :adapter]) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) settings = [ - stream: stream, - validation_context: validation_context, - mode: mode, - max_retries: max_retries, - model: model + stream: params[:stream], + validation_context: params[:validation_context], + mode: params[:mode], + max_retries: params[:max_retries], + model: params[:model] ] |> Enum.reject(fn {_k, v} -> is_nil(v) end) @@ -116,8 +141,8 @@ defmodule Instructor.Instruction do %{ role: "user", content: - if(__MODULE__.__schema__(:template), - do: render_template(message), + if(params[:template], + do: render_template(params[:template], message), else: message ) } @@ -125,16 +150,18 @@ defmodule Instructor.Instruction do end messages = - if __MODULE__.__schema__(:system_prompt) do - [%{role: "system", content: __MODULE__.__schema__(:system_prompt)} | messages] + if params[:system_prompt] do + [%{role: "system", content: params[:system_prompt]} | messages] else messages end response_model = - if __MODULE__.__schema__(:array), do: {:array, __MODULE__}, else: __MODULE__ + if params[:array], do: {:array, __MODULE__}, else: __MODULE__ + + opts = + [messages: messages, response_model: response_model, after_request: params[:after_request]] ++ settings ++ opts - opts = [messages: messages, response_model: response_model] ++ settings ++ opts Instructor.chat_completion(opts, config) end @@ -145,7 +172,10 @@ defmodule Instructor.Instruction do |> changeset(changeset, Enum.into(context, [])) end - defoverridable validate_changeset: 1, validate_changeset: 2 + defoverridable validate_changeset: 1, + validate_changeset: 2, + chat_completion: 1, + chat_completion: 2 end end end diff --git a/lib/instructor/json_schema.ex b/lib/instructor/json_schema.ex index 37eec66..147f65f 100644 --- a/lib/instructor/json_schema.ex +++ b/lib/instructor/json_schema.ex @@ -7,6 +7,8 @@ defmodule Instructor.JSONSchema do Note: This will output a correct JSON Schema for the given Ecto schema, but it will not necessarily be optimal, nor support all Ecto types. """ + def from_ecto_schema(%Ecto.Changeset{data: %{__struct__: module}}), do: from_ecto_schema(module) + def from_ecto_schema(ecto_schema) do defs = for schema <- bfs_from_ecto_schema([ecto_schema], %MapSet{}), into: %{} do @@ -105,27 +107,32 @@ defmodule Instructor.JSONSchema do value = for_type(type) value = - Map.merge(%{title: Atom.to_string(field)}, value) - - value = if field_doc != "" do - Map.update(value, :description, field_doc, fn desc -> - field_doc = - cond do - field_doc == "" -> - "" - - String.ends_with?(field_doc, ".") -> - field_doc <> " " - - true -> - field_doc <> ". " - end - - field_doc <> desc - end) - else - value - end + if field in ecto_schema.__schema__(:embeds) do + value + else + Map.merge(%{title: Atom.to_string(field)}, value) + end + + value = + if field_doc != "" do + Map.update(value, :description, field_doc, fn desc -> + field_doc = + cond do + field_doc == "" -> + "" + + String.ends_with?(field_doc, ".") -> + field_doc <> " " + + true -> + field_doc <> ". " + end + + field_doc <> desc + end) + else + value + end value = if type == :string && match?(%Regex{}, field_pattern) do @@ -150,7 +157,7 @@ defmodule Instructor.JSONSchema do if association.cardinality == :many do %{ items: %{"$ref": "#/$defs/#{title}"}, - title: title, + # title: title, type: "array" } else diff --git a/lib/instructor/types/union.ex b/lib/instructor/types/union.ex new file mode 100644 index 0000000..7ba5e2d --- /dev/null +++ b/lib/instructor/types/union.ex @@ -0,0 +1,13 @@ +defmodule Instructor.Union do + use Flint.Type, extends: Flint.Types.Union + @behaviour Instructor.EctoType + + @impl true + def to_json_schema(%{types: types}) when is_list(types) do + # "oneOf" isn't in the allowes JSON schema subset for + # structued outputs (OpenAI) + %{ + "anyOf" => Enum.map(types, &Instructor.EctoType.for_type/1) + } + end +end diff --git a/mix.lock b/mix.lock index 917caf4..39da3ff 100644 --- a/mix.lock +++ b/mix.lock @@ -1,40 +1,35 @@ %{ - "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, - "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, - "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, - "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, - "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"}, - "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, - "flint": {:git, "https://github.com/acalejos/flint.git", "8c0a01ee1aa17e4f4f0cafbcccbd8d911915b59b", []}, - "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"}, - "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, - "igniter": {:hex, :igniter, "0.4.7", "a91ab006e400f82ae93c1ada7d112cc7e7c1684455428c0de98d6b2e66025b43", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 1.0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "89bc7a093d60c5c70e46014ffd83d993593c1151c8301e3a16f2846695ff22ec"}, + "castore": {:hex, :castore, "1.0.11", "4bbd584741601eb658007339ea730b082cc61f3554cf2e8f39bf693a11b49073", [:mix], [], "hexpm", "e03990b4db988df56262852f20de0f659871c35154691427a5047f4967a16a62"}, + "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.42", "f23d856f41919f17cd06a493923a722d87a2d684f143a1e663c04a2b93100682", [:mix], [], "hexpm", "6915b6ca369b5f7346636a2f41c6a6d78b5af419d61a611079189233358b8b8b"}, + "ecto": {:hex, :ecto, "3.12.5", "4a312960ce612e17337e7cefcf9be45b95a3be6b36b6f94dfb3d8c361d631866", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6eb18e80bef8bb57e17f5a7f068a1719fbda384d40fc37acb8eb8aeca493b6ea"}, + "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, + "ex_doc": {:hex, :ex_doc, "0.36.1", "4197d034f93e0b89ec79fac56e226107824adcce8d2dd0a26f5ed3a95efc36b1", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d7d26a7cf965dacadcd48f9fa7b5953d7d0cfa3b44fa7a65514427da44eafd89"}, + "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, + "flint": {:git, "https://github.com/acalejos/flint.git", "ba78587cad8d24d1031e8e46d8bb783816c7c3f8", []}, + "hpax": {:hex, :hpax, "1.0.2", "762df951b0c399ff67cc57c3995ec3cf46d696e41f0bba17da0518d94acd4aac", [:mix], [], "hexpm", "2f09b4c1074e0abd846747329eaa26d535be0eb3d189fa69d812bfb8bfefd32f"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, - "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"}, - "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, - "mint": {:hex, :mint, "1.6.0", "88a4f91cd690508a04ff1c3e28952f322528934be541844d54e0ceb765f01d5e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "3c5ae85d90a5aca0a49c0d8b67360bbe407f3b54f1030a111047ff988e8fefaa"}, + "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, + "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"}, "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, - "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, - "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, + "phoenix": {:hex, :phoenix, "1.7.18", "5310c21443514be44ed93c422e15870aef254cf1b3619e4f91538e7529d2b2e4", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "1797fcc82108442a66f2c77a643a62980f342bfeb63d6c9a515ab8294870004e"}, + "phoenix_html": {:hex, :phoenix_html, "4.2.0", "83a4d351b66f472ebcce242e4ae48af1b781866f00ef0eb34c15030d4e2069ac", [:mix], [], "hexpm", "9713b3f238d07043583a94296cc4bbdceacd3b3a6c74667f4df13971e7866ec8"}, "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.17", "f396bbdaf4ba227b82251eb75ac0afa6b3da5e509bc0d030206374237dfc9450", [:mix], [{:floki, "~> 0.36", [hex: :floki, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a61d741ffb78c85fdbca0de084da6a48f8ceb5261a79165b5a0b59e5f65ce98b"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, - "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"}, - "rewrite": {:hex, :rewrite, "1.1.1", "0e6674eb5f8cb11aabe5ad6207151b4156bf173aa9b43133a68f8cc882364570", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "fcd688b3ca543c3a1f1f4615ccc054ec37cfcde91133a27a683ec09b35ae1496"}, + "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"}, "sourceror": {:hex, :sourceror, "1.7.1", "599d78f4cc2be7d55c9c4fd0a8d772fd0478e3a50e726697c20d13d02aa056d4", [:mix], [], "hexpm", "cd6f268fe29fa00afbc535e215158680a0662b357dc784646d7dff28ac65a0fc"}, - "spark": {:hex, :spark, "2.2.35", "1c0bb30f340151eca24164885935de39e6ada4010555f444c813d0488990f8f3", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "f242d6385c287389034a0e146d8f025b5c9ab777f1ae5cf0fdfc9209db6ae748"}, - "spitfire": {:hex, :spitfire, "0.1.3", "7ea0f544005dfbe48e615ed90250c9a271bfe126914012023fd5e4b6b82b7ec7", [:mix], [], "hexpm", "d53b5107bcff526a05c5bb54c95e77b36834550affd5830c9f58760e8c543657"}, + "spark": {:hex, :spark, "2.2.36", "07c921e5efb27f184267c3431d2f82099e24cac90748a47383dd75cbfb558268", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "e5ac56b75e5ad43da6d8302b6713277488f8e9a3abdba9aae8f0d0f9cff04538"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, - "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, } diff --git a/test/json_schema_test.exs b/test/json_schema_test.exs index 3ee4e6e..0269f83 100644 --- a/test/json_schema_test.exs +++ b/test/json_schema_test.exs @@ -216,7 +216,7 @@ defmodule JSONSchemaTest do }, "description" => "", "properties" => %{ - "embedded" => %{"$ref" => "#/$defs/JSONSchemaTest.Embedded", "title" => "embedded"} + "embedded" => %{"$ref" => "#/$defs/JSONSchemaTest.Embedded"} }, "required" => ["embedded"], "title" => "JSONSchemaTest.Demo", @@ -312,7 +312,6 @@ defmodule JSONSchemaTest do "properties" => %{ "children" => %{ "items" => %{"$ref" => "#/$defs/JSONSchemaTest.Child"}, - "title" => "JSONSchemaTest.Child", "type" => "array" }, "id" => %{"title" => "id", "type" => "integer"}