From ba21742a70808b0140650a271fe2e2945e275bd4 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Tue, 2 Jul 2024 19:14:34 -0500
Subject: [PATCH 1/8] feat(foreign): python proof of concept

---
 flake.nix                               | 19 +++++
 lib/stampede/application.ex             |  5 ++
 lib/stampede/events/response_to_post.ex | 21 +++++-
 lib/stampede/plugin_foreign/python.ex   | 99 +++++++++++++++++++++++++
 lib_py/.gitignore                       |  2 +
 lib_py/example.py                       | 14 ++++
 lib_py/requirements.txt                 |  0
 mix.exs                                 |  6 ++
 mix.lock                                |  3 +
 test/foreign_plugins_test.exs           | 77 +++++++++++++++++++
 10 files changed, 242 insertions(+), 4 deletions(-)
 create mode 100644 lib/stampede/plugin_foreign/python.ex
 create mode 100644 lib_py/.gitignore
 create mode 100644 lib_py/example.py
 create mode 100644 lib_py/requirements.txt
 create mode 100644 test/foreign_plugins_test.exs

diff --git a/flake.nix b/flake.nix
index 967064e..51e06bd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -26,6 +26,11 @@
         erl = with pkgs; beam.packages.erlang_26;
         ex = erl.elixir_1_16;
 
+        ########################
+        # Python versions
+        python = pkgs.python312;
+        mkPyPkg = name: python.withPackages (ps: [(builtins.getAttr name ps)]);
+
         ########################
         # Git pre-push checks
         pc-hooks = git-hooks.lib.${system}.run {
@@ -78,6 +83,15 @@
               require_serial = true;
               stages = ["manual" "push" "pre-merge-commit" "pre-commit"];
             };
+
+            mypy = {
+              enable = true;
+              package = mkPyPkg "mypy";
+            };
+            black = {
+              enable = true;
+              package = mkPyPkg "black";
+            };
           };
         };
       in {
@@ -94,6 +108,11 @@
 
               pkgs.libyaml
               pkgs.libyaml.dev
+
+              python
+              (mkPyPkg "python-lsp-server")
+              (mkPyPkg "pylsp-mypy")
+              (mkPyPkg "python-lsp-black")
             ]
             ++ pc-hooks.enabledPackages;
 
diff --git a/lib/stampede/application.ex b/lib/stampede/application.ex
index 3a91429..a6164c8 100644
--- a/lib/stampede/application.ex
+++ b/lib/stampede/application.ex
@@ -16,6 +16,11 @@ defmodule Stampede.Application do
         required: true,
         doc: "Services installed as part of the mix project. Passed in from mix.exs"
       ],
+      # installed_foreign_plugins: [
+      #   type: {:or, [{:in, [[]]}, {:list, {:in, Map.values(S.services())}}]},
+      #   required: true,
+      #   doc: "Foreign Plugin sources installed as part of the mix project. Passed in from mix.exs"
+      # ],
       services: [
         type: {:or, [{:in, [:all]}, {:list, {:in, Map.keys(S.services())}}]},
         default: :all,
diff --git a/lib/stampede/events/response_to_post.ex b/lib/stampede/events/response_to_post.ex
index 3f9dc63..7cfbce5 100644
--- a/lib/stampede/events/response_to_post.ex
+++ b/lib/stampede/events/response_to_post.ex
@@ -36,12 +36,25 @@ defmodule Stampede.Events.ResponseToPost do
   @doc "makes a new ResponseToPost but automatically tags the source module unless already being tagged"
   defmacro new(keys) do
     quote do
-      struct!(
-        unquote(__MODULE__),
-        Keyword.put_new(unquote(keys), :origin_plug, __MODULE__)
-      )
+      unquote(keys)
+      |> Keyword.put_new(:origin_plug, __MODULE__)
+      |> unquote(__MODULE__).new_bare()
     end
   end
 
+  def new_bare(keys) do
+    struct!(
+      __MODULE__,
+      keys
+    )
+  end
+
   # friendly reminder to admire the macros from afar, they are charming but they have teeth and a taste for blood
+
+  # # TODO: logic for importing from Python/JSON/etc to actual elixir structs. this might just be too complex a task.
+  # def foreign_import(dict, overrides = []) do
+  #   rules = [
+  #     {:confidence, &(is_number(&1) || raise())}
+  #   ]
+  # end
 end
diff --git a/lib/stampede/plugin_foreign/python.ex b/lib/stampede/plugin_foreign/python.ex
new file mode 100644
index 0000000..953469e
--- /dev/null
+++ b/lib/stampede/plugin_foreign/python.ex
@@ -0,0 +1,99 @@
+defmodule Stampede.PluginForeign.Python.Pool do
+  @moduledoc false
+  use Doumi.Port,
+    adapter: {Doumi.Port.Adapter.Python, python_path: ["./lib_py"]},
+    pool_size: 4
+end
+
+defmodule Stampede.PluginForeign.Python do
+  @moduledoc false
+  alias Stampede, as: S
+  require S.ResponseToPost
+  alias Stampede.PluginForeign.Python, as: SPy
+
+  def start_link() do
+    Supervisor.start_link([SPy.Pool], strategy: :one_for_one, name: SPy.Supervisor)
+  end
+
+  def query(py_mod, real_cfg, real_event) do
+    cfg = dumb_down_elixir_term(real_cfg)
+    event = dumb_down_elixir_term(real_event)
+
+    with {:ok, result} <- SPy.Pool.command(py_mod, :process, [cfg, event]) do
+      case result do
+        :undefined ->
+          nil
+
+        %{
+          confidence: _confidence,
+          text: _text,
+          why: _why
+        } = basic_info ->
+          basic_info
+          |> Map.update!(:confidence, fn
+            i when is_number(i) ->
+              i
+
+            str when is_binary(str) ->
+              String.to_float(str)
+
+            [h | _] = cl when is_integer(h) ->
+              List.to_float(cl)
+          end)
+          |> Map.update!(:text, fn
+            str when is_binary(str) ->
+              str
+
+            [h | _] = cl when is_integer(h) ->
+              List.to_string(cl)
+          end)
+          |> Map.update!(:why, fn
+            str when is_binary(str) ->
+              str
+
+            [h | _] = cl when is_integer(h) ->
+              List.to_string(cl)
+          end)
+          |> Map.put(:origin_plug, "Python.#{py_mod}" |> String.to_atom())
+          |> Map.put(:origin_msg_id, event.msg_id)
+          |> Map.to_list()
+          |> S.ResponseToPost.new_bare()
+
+        other ->
+          raise("""
+          Can only handle response as a dict with the keys "confidence", "text" and "why".
+          Response given: #{S.pp(other)}
+          """)
+      end
+    end
+  end
+
+  def dumb_down_elixir_term(term) when is_atom(term), do: Atom.to_string(term)
+  def dumb_down_elixir_term({k, v}), do: {dumb_down_elixir_term(k), dumb_down_elixir_term(v)}
+  def dumb_down_elixir_term([h | t]), do: [dumb_down_elixir_term(h) | dumb_down_elixir_term(t)]
+
+  def dumb_down_elixir_term(ms) when is_struct(ms, MapSet) do
+    ms
+    |> MapSet.to_list()
+    |> Enum.map(fn
+      v ->
+        dumb_down_elixir_term(v)
+    end)
+  end
+
+  def dumb_down_elixir_term(struct) when is_struct(struct) do
+    struct
+    |> Map.from_struct()
+    |> dumb_down_elixir_term()
+  end
+
+  def dumb_down_elixir_term(map) when is_map(map) do
+    map
+    |> Map.new(fn
+      {k, v} ->
+        {dumb_down_elixir_term(k), dumb_down_elixir_term(v)}
+    end)
+  end
+
+  def dumb_down_elixir_term(otherwise), do: otherwise
+end
diff --git a/lib_py/.gitignore b/lib_py/.gitignore
new file mode 100644
index 0000000..7b6fd7a
--- /dev/null
+++ b/lib_py/.gitignore
@@ -0,0 +1,2 @@
+lib
+__pycache__
diff --git a/lib_py/example.py b/lib_py/example.py
new file mode 100644
index 0000000..ffcf014
--- /dev/null
+++ b/lib_py/example.py
@@ -0,0 +1,14 @@
+from typing import (
+    Optional,
+    Dict
+)
+
+def process(cfg, event) -> Optional[Dict]:
+    if event[b"body"] == b"ping python":
+        return {
+            "confidence": 10,
+            "text": "pong!",
+            "why": ["They pinged so I ponged!"],
+        }
+    else:
+        return None
diff --git a/lib_py/requirements.txt b/lib_py/requirements.txt
new file mode 100644
index 0000000..e69de29
diff --git a/mix.exs b/mix.exs
index 1b54dcf..1dc277c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -61,6 +61,7 @@ defmodule Stampede.MixProject do
   def configure_app([], config_acc) when is_list(config_acc), do: config_acc
 
   def application do
+    # TODO: determine with config.exs
     configure_app([:discord])
   end
 
@@ -99,6 +100,11 @@ defmodule Stampede.MixProject do
       {:nostrum, "~> 0.9.1", runtime: false},
       # {:nostrum, github: "Kraigie/nostrum", runtime: false},
 
+      # EXTERNAL PLUGIN SUPPORT
+      {:doumi_port, "~> 0.6.0"},
+      # override erlport for newer version
+      {:erlport, "~> 0.11.0", override: true},
+
       # For catching Erlang errors and sending to services
       {:logger_backends, "~> 1.0"},
 
diff --git a/mix.lock b/mix.lock
index 98dd558..62bba25 100644
--- a/mix.lock
+++ b/mix.lock
@@ -17,12 +17,14 @@
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
   "doctor": {:hex, :doctor, "0.21.0", "20ef89355c67778e206225fe74913e96141c4d001cb04efdeba1a2a9704f1ab5", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "a227831daa79784eb24cdeedfa403c46a4cb7d0eab0e31232ec654314447e4e0"},
+  "doumi_port": {:hex, :doumi_port, "0.6.0", "0ca6adcc753d9c2a8489adea336a199b4e208823cc5f9368ff34db8df140a6c4", [:mix], [{:erlport, "~> 0.10", [hex: :erlport, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "1d10b5680be9538132d5fb943cf58a25dc9bd8852d1d2ddcc887afa8a05896f9"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
   "ed25519": {:hex, :ed25519, "1.4.1", "479fb83c3e31987c9cad780e6aeb8f2015fb5a482618cdf2a825c9aff809afc4", [:mix], [], "hexpm", "0dacb84f3faa3d8148e81019ca35f9d8dcee13232c32c9db5c2fb8ff48c80ec7"},
   "eflambe": {:hex, :eflambe, "0.3.1", "ef0a35084fad1f50744496730a9662782c0a9ebf449d3e03143e23295c5926ea", [:rebar3], [{:meck, "0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "58d5997be606d4e269e9e9705338e055281fdf3e4935cc902c8908e9e4516c5f"},
   "eflame": {:hex, :eflame, "1.0.1", "0664d287e39eef3c413749254b3af5f4f8b00be71c1af67d325331c4890be0fc", [:mix], [], "hexpm", "e0b08854a66f9013129de0b008488f3411ae9b69b902187837f994d7a99cf04e"},
   "equivalex": {:hex, :equivalex, "1.0.3", "170d9a82ae066e0020dfe1cf7811381669565922eb3359f6c91d7e9a1124ff74", [:mix], [], "hexpm", "46fa311adb855117d36e461b9c0ad2598f72110ad17ad73d7533c78020e045fc"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
+  "erlport": {:hex, :erlport, "0.11.0", "8bb46a520e6eb9146e655fbf9b824433d9d532194667069d9aa45696aae9684b", [:rebar3], [], "hexpm", "8eb136ccaf3948d329b8d1c3278ad2e17e2a7319801bc4cc2da6db278204eee4"},
   "ex_check": {:hex, :ex_check, "0.16.0", "07615bef493c5b8d12d5119de3914274277299c6483989e52b0f6b8358a26b5f", [:mix], [], "hexpm", "4d809b72a18d405514dda4809257d8e665ae7cf37a7aee3be6b74a34dec310f5"},
   "ex_doc": {:hex, :ex_doc, "0.34.0", "ab95e0775db3df71d30cf8d78728dd9261c355c81382bcd4cefdc74610bef13e", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "60734fb4c1353f270c3286df4a0d51e65a2c1d9fba66af3940847cc65a8066d7"},
   "expo": {:hex, :expo, "0.5.2", "beba786aab8e3c5431813d7a44b828e7b922bfa431d6bfbada0904535342efe2", [:mix], [], "hexpm", "8c9bfa06ca017c9cb4020fabe980bc7fdb1aaec059fd004c2ab3bff03b1c599c"},
@@ -49,6 +51,7 @@
   "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
+  "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "norm": {:hex, :norm, "0.13.0", "2c562113f3205e3f195ee288d3bd1ab903743e7e9f3282562c56c61c4d95dec4", [:mix], [{:stream_data, "~> 0.5", [hex: :stream_data, repo: "hexpm", optional: true]}], "hexpm", "447cc96dd2d0e19dcb37c84b5fc0d6842aad69386e846af048046f95561d46d7"},
   "nostrum": {:hex, :nostrum, "0.9.1", "52832df6adcd09389d83074bbb7f9e634eb110f178566e6df64314d981e0d0ed", [:mix], [{:castle, "~> 0.3.0", [hex: :castle, repo: "hexpm", optional: false]}, {:certifi, "~> 2.13", [hex: :certifi, repo: "hexpm", optional: false]}, {:gun, "~> 2.0", [hex: :gun, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:kcl, "~> 1.4", [hex: :kcl, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "d5697109e07bd1f747b3d2a74b69d003c12210ab12e57ac54d83dcf087de34f5"},
   "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"},
diff --git a/test/foreign_plugins_test.exs b/test/foreign_plugins_test.exs
new file mode 100644
index 0000000..3b6175e
--- /dev/null
+++ b/test/foreign_plugins_test.exs
@@ -0,0 +1,77 @@
+defmodule ForeignPluginsTest do
+  use ExUnit.Case, async: true
+  import ExUnit.CaptureLog
+  alias Services.Dummy, as: D
+  alias Stampede, as: S
+  import AssertValue
+  require S.MsgReceived
+
+  @dummy_cfg_verified %{
+    service: Services.Dummy,
+    server_id: :testing,
+    error_channel_id: :error,
+    prefix: "!",
+    plugs: MapSet.new([Plugins.Test, Plugins.Sentience, Plugins.Why, Plugins.Help]),
+    dm_handler: false,
+    filename: "test SiteConfig load_all",
+    vip_ids: MapSet.new([:server]),
+    bot_is_loud: false
+  }
+  setup_all do
+    %{
+      app_pid:
+        Stampede.Application.start(
+          :normal,
+          installed_services: [:dummy],
+          services: [:dummy],
+          log_to_file: false,
+          log_post_serious_errors: false,
+          clear_state: true
+        )
+    }
+  end
+
+  setup context do
+    id = context.test
+
+    if context[:dummy] do
+      @dummy_cfg_verified
+      |> Map.to_list()
+      |> Keyword.put(:server_id, id)
+      |> Keyword.put(:filename, id |> Atom.to_string())
+      |> Keyword.merge(context[:cfg_overrides] || [])
+      |> D.new_server()
+    end
+
+    %{id: id}
+  end
+
+  describe "Python" do
+    test "basic" do
+      {:ok, _pid} = Stampede.PluginForeign.Python.start_link()
+
+      cfg =
+        @dummy_cfg_verified
+        |> Stampede.PluginForeign.Python.dumb_down_elixir_term()
+
+      msg =
+        S.MsgReceived.new(
+          id: 0,
+          body: "!ping python",
+          channel_id: :t1,
+          author_id: :u1,
+          server_id: :none
+        )
+        |> S.MsgReceived.add_context(@dummy_cfg_verified)
+        |> Stampede.PluginForeign.Python.dumb_down_elixir_term()
+
+      assert_value Stampede.PluginForeign.Python.Pool.command(:example, :process, [cfg, msg]) ==
+                     {:ok,
+                      %{
+                        ~c"confidence" => 10,
+                        ~c"text" => ~c"pong!",
+                        ~c"why" => [~c"They pinged so I ponged!"]
+                      }}
+    end
+  end
+end

From 917812d73b643f64adc5605941b041d5600dc669 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Wed, 3 Jul 2024 16:43:34 -0500
Subject: [PATCH 2/8] feat(python): externally configurable python exe/dirs

---
 config/config.exs                     | 4 ++++
 flake.nix                             | 2 ++
 lib/stampede/plugin_foreign/python.ex | 6 +++++-
 lib_py/example.py                     | 6 ++----
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 586bbd5..9f3e5b5 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -91,6 +91,10 @@ config :ex_unit,
 # By default, Nostrum requires ffmpeg to use voice.
 config :nostrum, :ffmpeg, false
 
+config :stampede,
+  python_exe: System.fetch_env!("FLAKE_PYTHON"),
+  python_plugin_dirs: ["./lib_py"]
+
 for config <- "./*.secret.exs" |> Path.expand(__DIR__) |> Path.wildcard() do
   import_config config
 end
diff --git a/flake.nix b/flake.nix
index 51e06bd..aca8a2b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -118,6 +118,8 @@
 
           # define shell startup command
           sh-hook = ''
+            export FLAKE_PYTHON="${python}/bin/python3"
+
             # this allows mix to work on the local directory
             mkdir -p .nix-mix
             mkdir -p .nix-hex
diff --git a/lib/stampede/plugin_foreign/python.ex b/lib/stampede/plugin_foreign/python.ex
index 953469e..024eb96 100644
--- a/lib/stampede/plugin_foreign/python.ex
+++ b/lib/stampede/plugin_foreign/python.ex
@@ -1,7 +1,11 @@
 defmodule Stampede.PluginForeign.Python.Pool do
   @moduledoc false
   use Doumi.Port,
-    adapter: {Doumi.Port.Adapter.Python, python_path: ["./lib_py"]},
+    adapter: {
+      Doumi.Port.Adapter.Python,
+      python: Application.fetch_env!(:stampede, :python_exe),
+      python_path: Application.fetch_env!(:stampede, :python_plugin_dirs)
+    },
     pool_size: 4
 end
 
diff --git a/lib_py/example.py b/lib_py/example.py
index ffcf014..cef25fe 100644
--- a/lib_py/example.py
+++ b/lib_py/example.py
@@ -1,7 +1,5 @@
-from typing import (
-    Optional,
-    Dict
-)
+from typing import Optional, Dict
+
 
 def process(cfg, event) -> Optional[Dict]:
     if event[b"body"] == b"ping python":

From e4480721c18c727d98f3e0f5bfb5885401ac4e72 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Thu, 4 Jul 2024 21:51:13 -0500
Subject: [PATCH 3/8] fix(hooks): python formatters on commit

---
 flake.nix | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/flake.nix b/flake.nix
index aca8a2b..fe0544b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -73,25 +73,27 @@
               pass_filenames = false;
               require_serial = true;
             };
-            custom-mix-format = {
-              enable = true;
-              name = "mix-format";
-              entry = "${ex}/bin/mix format --check-formatted";
-              files = "\\.exs?$";
-              types = ["text"];
-              pass_filenames = false;
-              require_serial = true;
-              stages = ["manual" "push" "pre-merge-commit" "pre-commit"];
-            };
-
-            mypy = {
-              enable = true;
-              package = mkPyPkg "mypy";
-            };
-            black = {
-              enable = true;
-              package = mkPyPkg "black";
-            };
+            custom-mix-format =
+              enable_on_commit
+              // {
+                name = "mix-format";
+                entry = "${ex}/bin/mix format --check-formatted";
+                files = "\\.exs?$";
+                types = ["text"];
+                pass_filenames = false;
+                require_serial = true;
+              };
+
+            mypy =
+              enable_on_commit
+              // {
+                package = mkPyPkg "mypy";
+              };
+            black =
+              enable_on_commit
+              // {
+                package = mkPyPkg "black";
+              };
           };
         };
       in {

From 7d5ddf1489e511d9c1338b36c3b3cb3d0b32eaf1 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:11:49 -0500
Subject: [PATCH 4/8] chore: rename "plugin_foreign" to "external"

---
 lib/stampede/{plugin_foreign => external}/python.ex | 6 +++---
 test/foreign_plugins_test.exs                       | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)
 rename lib/stampede/{plugin_foreign => external}/python.ex (95%)

diff --git a/lib/stampede/plugin_foreign/python.ex b/lib/stampede/external/python.ex
similarity index 95%
rename from lib/stampede/plugin_foreign/python.ex
rename to lib/stampede/external/python.ex
index 024eb96..30e4b84 100644
--- a/lib/stampede/plugin_foreign/python.ex
+++ b/lib/stampede/external/python.ex
@@ -1,4 +1,4 @@
-defmodule Stampede.PluginForeign.Python.Pool do
+defmodule Stampede.External.Python.Pool do
   @moduledoc false
   use Doumi.Port,
     adapter: {
@@ -9,11 +9,11 @@ defmodule Stampede.PluginForeign.Python.Pool do
     pool_size: 4
 end
 
-defmodule Stampede.PluginForeign.Python do
+defmodule Stampede.External.Python do
   @moduledoc false
   alias Stampede, as: S
   require S.ResponseToPost
-  alias Stampede.PluginForeign.Python, as: SPy
+  alias Stampede.External.Python, as: SPy
 
   def start_link() do
     Supervisor.start_link([SPy.Pool], strategy: :one_for_one, name: SPy.Supervisor)
diff --git a/test/foreign_plugins_test.exs b/test/foreign_plugins_test.exs
index 3b6175e..1dcaf1c 100644
--- a/test/foreign_plugins_test.exs
+++ b/test/foreign_plugins_test.exs
@@ -48,11 +48,11 @@ defmodule ForeignPluginsTest do
 
   describe "Python" do
     test "basic" do
-      {:ok, _pid} = Stampede.PluginForeign.Python.start_link()
+      {:ok, _pid} = Stampede.External.Python.start_link()
 
       cfg =
         @dummy_cfg_verified
-        |> Stampede.PluginForeign.Python.dumb_down_elixir_term()
+        |> Stampede.External.Python.dumb_down_elixir_term()
 
       msg =
         S.MsgReceived.new(
@@ -63,9 +63,9 @@ defmodule ForeignPluginsTest do
           server_id: :none
         )
         |> S.MsgReceived.add_context(@dummy_cfg_verified)
-        |> Stampede.PluginForeign.Python.dumb_down_elixir_term()
+        |> Stampede.External.Python.dumb_down_elixir_term()
 
-      assert_value Stampede.PluginForeign.Python.Pool.command(:example, :process, [cfg, msg]) ==
+      assert_value Stampede.External.Python.Pool.command(:example, :process, [cfg, msg]) ==
                      {:ok,
                       %{
                         ~c"confidence" => 10,

From 70516c592aab71ca4e21c934028efd75a08350e9 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Thu, 18 Jul 2024 11:52:53 -0500
Subject: [PATCH 5/8] fix(mnesia): startup more reliable

---
 lib/stampede/tables.ex | 63 ++++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/lib/stampede/tables.ex b/lib/stampede/tables.ex
index 1dc0cf6..41bc8ea 100644
--- a/lib/stampede/tables.ex
+++ b/lib/stampede/tables.ex
@@ -13,15 +13,19 @@ defmodule Stampede.Tables do
   def init(args) do
     clear_state = Keyword.fetch!(args, :clear_state)
     Logger.debug("Tables: starting")
-    _ = Memento.stop()
+
+    # NOTE: issue #23: this should be done with Memento.stop() and Memento.start()
+    # However, Memento.start() hands back control before Mnesia is done starting.
+    # Will submit a fix to Memento
+    _ = Application.stop(:mnesia)
     :ok = ensure_schema_exists(S.nodes())
-    :ok = Memento.start()
+    {:ok, _} = Application.ensure_all_started(:mnesia)
     # # DEBUG
     # Memento.info()
     # Memento.Schema.info()
     :ok = ensure_tables_exist(@mnesia_tables)
 
-    if clear_state == true do
+    if clear_state do
       :ok = clear_all_tables()
     end
 
@@ -68,28 +72,45 @@ defmodule Stampede.Tables do
     end
   end
 
-  @spec! ensure_tables_exist(list(atom())) :: :ok
-  def ensure_tables_exist(tables) when is_list(tables) do
-    Enum.each(tables, fn t ->
-      case Memento.Table.create(t) do
-        :ok ->
-          :ok
-
-        {:error, {:already_exists, ^t}} ->
-          :ok
-
-        other ->
-          raise "Memento table creation error: #{S.pp(other)}"
-      end
-
-      # DEBUG
-      Memento.Table.info(t)
-    end)
+  @spec! ensure_tables_exist(nonempty_list(atom())) :: :ok
+  def ensure_tables_exist(ll) do
+    do_ensure_tables_exist(ll, [])
+  end
 
+  defp do_ensure_tables_exist([], done) do
     :ok =
       Memento.wait(
-        tables,
+        done,
         :timer.seconds(5)
       )
   end
+
+  defp do_ensure_tables_exist(ll = [t | rest], done) do
+    case Memento.Table.create(t) do
+      :ok ->
+        :ok
+
+      {:error, {:already_exists, ^t}} ->
+        :ok
+
+      {:error, {:node_not_running, _}} ->
+        :retry
+
+      {:error, :mmesia_stopped} ->
+        :mnesia_stopped
+
+      other ->
+        raise "Memento table creation error: #{S.pp(other)}"
+    end
+    |> case do
+      :ok ->
+        # DEBUG
+        Memento.Table.info(t)
+
+        do_ensure_tables_exist(rest, [t | done])
+
+      :retry ->
+        do_ensure_tables_exist(ll, done)
+    end
+  end
 end

From 32a1c8f12584c0389ffac6fb372035b0ceefdf91 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Thu, 18 Jul 2024 11:58:27 -0500
Subject: [PATCH 6/8] refactor: various docs and organization

---
 lib/stampede.ex                 | 36 ++++++++++++----------
 lib/stampede/external/python.ex | 53 +++++++++++++++++++++++++++------
 2 files changed, 65 insertions(+), 24 deletions(-)

diff --git a/lib/stampede.ex b/lib/stampede.ex
index d3517b8..3c5f22d 100644
--- a/lib/stampede.ex
+++ b/lib/stampede.ex
@@ -4,6 +4,8 @@ defmodule Stampede do
   """
   @compile [:bin_opt_info, :recv_opt_info]
   use TypeCheck
+
+  # Types used across Stampede
   @type! service_name :: module()
   @type! channel_id :: any()
   @typedoc """
@@ -13,21 +15,7 @@ defmodule Stampede do
   @type! server_id :: integer() | atom() | dm_tuple()
   @type! user_id :: any()
   @type! msg_id :: any()
-  @type! log_level ::
-           :emergency | :alert | :critical | :error | :warning | :warn | :notice | :info | :debug
-  @type! log_msg ::
-           {log_level(), identifier(), {Logger, String.t() | maybe_improper_list(), any(), any()}}
   @type! prefix :: String.t() | Regex.t()
-  @type! module_function_args :: {module(), atom(), tuple() | list()}
-  # BUG: type_check issue #189, iolist()
-  #      this stand-in isn't type complete but it'll do
-  @type! str_list ::
-           String.t()
-           | []
-           | nonempty_list(lazy(Stampede.str_list()))
-  @type! mapset(t) :: map(any(), t)
-  @type! mapset() :: mapset(any())
-
   @type! enabled_plugs :: :all | [] | nonempty_list(module())
   @type! channel_lock_action ::
            false | {:lock, channel_id(), module_function_args()} | {:unlock, channel_id()}
@@ -35,12 +23,30 @@ defmodule Stampede do
            false | {module_function_args(), atom(), integer()}
   @type! timestamp :: DateTime.t()
   @type! interaction_id :: non_neg_integer()
-
   @type! bot_invoked_status ::
            nil
            | :mentioned_from_service
            | :prefixed
 
+  # Elixir-generic stuff that could/should be builtin types
+  @type! module_function_args :: {module(), atom(), tuple() | list()}
+  @type! log_level ::
+           :emergency | :alert | :critical | :error | :warning | :warn | :notice | :info | :debug
+  @type! log_msg ::
+           {log_level(), identifier(), {Logger, String.t() | maybe_improper_list(), any(), any()}}
+  @type! mapset(t) :: map(any(), t)
+  @type! mapset() :: mapset(any())
+  @type! kwlist(t) :: list({atom(), t})
+  @type! kwlist() :: kwlist(any())
+  # BUG: type_check issue #189, iolist()
+  #      this stand-in isn't type complete but it'll do
+  #      No improper lists allowed
+  #      also VERY SLOW to check.
+  @type! str_list ::
+           String.t()
+           | []
+           | nonempty_list(lazy(Stampede.str_list()))
+
   def confused_response(),
     do: {:italics, "confused beeping"}
 
diff --git a/lib/stampede/external/python.ex b/lib/stampede/external/python.ex
index 30e4b84..d948576 100644
--- a/lib/stampede/external/python.ex
+++ b/lib/stampede/external/python.ex
@@ -10,7 +10,11 @@ defmodule Stampede.External.Python.Pool do
 end
 
 defmodule Stampede.External.Python do
-  @moduledoc false
+  @moduledoc """
+  Run Python functions with [Doumi](https://hexdocs.pm/doumi_port/readme.html).
+  When working on the Python side, you should use the Python helpers that Doumi makes available from the [ErlPort](http://erlport.org/docs/python.html) project.
+  """
+  use TypeCheck
   alias Stampede, as: S
   require S.ResponseToPost
   alias Stampede.External.Python, as: SPy
@@ -19,6 +23,18 @@ defmodule Stampede.External.Python do
     Supervisor.start_link([SPy.Pool], strategy: :one_for_one, name: SPy.Supervisor)
   end
 
+  @doc """
+  Run a Python function and return a result.
+  """
+  def exec(py_mod, func_atom, args, opts \\ []) do
+    SPy.Pool.command(py_mod, func_atom, args, opts)
+  end
+
+  @doc """
+  An example way to make a ResponseToPost from a Python module response, trying to minimize complications with cross-environment communications.
+  Expects the Python module to respond with a Dict containing only keys "confidence", "text", and "why".
+  Don't forget you can also make an Elixir plugin that only calls Python as needed, which will be better for tracebacks etc..
+  """
   def query(py_mod, real_cfg, real_event) do
     cfg = dumb_down_elixir_term(real_cfg)
     event = dumb_down_elixir_term(real_event)
@@ -72,11 +88,30 @@ defmodule Stampede.External.Python do
     end
   end
 
-  def dumb_down_elixir_term(term) when is_atom(term), do: Atom.to_string(term)
-  def dumb_down_elixir_term({k, v}), do: {dumb_down_elixir_term(k), dumb_down_elixir_term(v)}
-  def dumb_down_elixir_term([h | t]), do: [dumb_down_elixir_term(h) | dumb_down_elixir_term(t)]
+  @doc """
+  A brute-force way to make Elixir objects more generic for easier Python use. Start by checking if the object is a keyword list, which should really be a Dict in Python.
+  """
+  def dumb_down_elixir_term(term) do
+    if TypeCheck.conforms?(term, S.kwlist()) do
+      Map.new(term, fn {k, v} -> {Atom.to_string(k), dumb_down_elixir_term(v)} end)
+    else
+      do_dumb_down_elixir_term(term)
+    end
+  end
+
+  defp do_dumb_down_elixir_term(term) when is_atom(term), do: Atom.to_string(term)
+
+  defp do_dumb_down_elixir_term(tup) when is_tuple(tup) do
+    tup
+    |> Tuple.to_list()
+    |> Enum.map(&dumb_down_elixir_term/1)
+    |> List.to_tuple()
+  end
+
+  defp do_dumb_down_elixir_term([h | t]),
+    do: [dumb_down_elixir_term(h) | do_dumb_down_elixir_term(t)]
 
-  def dumb_down_elixir_term(ms) when is_struct(ms, MapSet) do
+  defp do_dumb_down_elixir_term(ms) when is_struct(ms, MapSet) do
     ms
     |> MapSet.to_list()
     |> Enum.map(fn
@@ -85,13 +120,13 @@ defmodule Stampede.External.Python do
     end)
   end
 
-  def dumb_down_elixir_term(struct) when is_struct(struct) do
+  defp do_dumb_down_elixir_term(struct) when is_struct(struct) do
     struct
     |> Map.from_struct()
-    |> dumb_down_elixir_term()
+    |> do_dumb_down_elixir_term()
   end
 
-  def dumb_down_elixir_term(map) when is_map(map) do
+  defp do_dumb_down_elixir_term(map) when is_map(map) do
     map
     |> Map.new(fn
       {k, v} ->
@@ -99,5 +134,5 @@ defmodule Stampede.External.Python do
     end)
   end
 
-  def dumb_down_elixir_term(otherwise), do: otherwise
+  defp do_dumb_down_elixir_term(otherwise), do: otherwise
 end

From 70ff87693730aa31e113cba6bd31a809faffa913 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Thu, 25 Jul 2024 15:32:50 -0500
Subject: [PATCH 7/8] chore: update ignores

---
 .dialyzer_ignore.exs            | 19 +++++++++++++------
 lib/stampede/external/python.ex |  7 +++++--
 test/foreign_plugins_test.exs   |  6 +++---
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs
index 3481cb3..9eb6de9 100644
--- a/.dialyzer_ignore.exs
+++ b/.dialyzer_ignore.exs
@@ -1,11 +1,13 @@
 [
-  {"deps/type_check/lib/type_check/spec.ex", "The pattern can never match the type {:ok, [], _}."},
+  {"deps/type_check/lib/type_check/spec.ex",
+   "The pattern can never match the type {:ok, [], _}."},
   {"lib/plugin.ex", "Function usage_tuples/0 has no local return."},
   {"lib/plugin.ex", "Function job_result/0 has no local return."},
   {"lib/plugin.ex", "Function plugin_job_result/0 has no local return."},
   {"lib/plugin.ex", "The guard clause can never succeed."},
   {"lib/services/discord.ex", "Function vips/0 has no local return."},
-  {"lib/services/dummy.ex", "@spec for ask_bot has more types than are returned by the function."},
+  {"lib/services/dummy.ex",
+   "@spec for ask_bot has more types than are returned by the function."},
   {"lib/services/dummy.ex", "Function dummy_channel_id/0 has no local return."},
   {"lib/services/dummy.ex", "Function msg_content/0 has no local return."},
   {"lib/services/dummy.ex", "Function msg_reference/0 has no local return."},
@@ -18,20 +20,25 @@
   {"lib/site_config.ex", "Function cfg_list/0 has no local return."},
   {"lib/stampede.ex", "Function dm_tuple/0 has no local return."},
   {"lib/stampede.ex", "Function server_id/0 has no local return."},
-  {"lib/stampede.ex", "Function log_level/0 has no local return."},
-  {"lib/stampede.ex", "Function log_msg/0 has no local return."},
   {"lib/stampede.ex", "Function prefix/0 has no local return."},
-  {"lib/stampede.ex", "Function module_function_args/0 has no local return."},
-  {"lib/stampede.ex", "Function str_list/0 has no local return."},
   {"lib/stampede.ex", "Function enabled_plugs/0 has no local return."},
   {"lib/stampede.ex", "Function channel_lock_action/0 has no local return."},
   {"lib/stampede.ex", "Function channel_lock_status/0 has no local return."},
   {"lib/stampede.ex", "Function timestamp/0 has no local return."},
   {"lib/stampede.ex", "Function bot_invoked_status/0 has no local return."},
+  {"lib/stampede.ex", "Function module_function_args/0 has no local return."},
+  {"lib/stampede.ex", "Function log_level/0 has no local return."},
+  {"lib/stampede.ex", "Function log_msg/0 has no local return."},
+  {"lib/stampede.ex", "Function kwlist/1 has no local return."},
+  {"lib/stampede.ex", "Function kwlist/0 has no local return."},
+  {"lib/stampede.ex", "Function str_list/0 has no local return."},
   {"lib/stampede.ex", "Function throw_internal_error/0 has no local return."},
   {"lib/stampede.ex", "Function throw_internal_error/1 only terminates with explicit exception."},
   {"lib/stampede/cfg_table.ex", "Function vips/0 has no local return."},
   {"lib/stampede/cfg_table.ex", "Function table_object/0 has no local return."},
+  {"lib/stampede/external/python.ex", "The pattern can never match the type true."},
+  {"lib/stampede/external/python.ex",
+   "Function do_dumb_down_elixir_term/1 will never be called."},
   {"lib/stampede/logger.ex", "Function logger_state/0 has no local return."},
   {"lib/stampede/tables/channel_locks.ex", "The guard clause can never succeed."},
   {"lib/stampede/tables/interactions.ex", "The guard clause can never succeed."},
diff --git a/lib/stampede/external/python.ex b/lib/stampede/external/python.ex
index d948576..e20cdfb 100644
--- a/lib/stampede/external/python.ex
+++ b/lib/stampede/external/python.ex
@@ -16,7 +16,7 @@ defmodule Stampede.External.Python do
   """
   use TypeCheck
   alias Stampede, as: S
-  require S.ResponseToPost
+  require S.Events.ResponseToPost
   alias Stampede.External.Python, as: SPy
 
   def start_link() do
@@ -77,7 +77,7 @@ defmodule Stampede.External.Python do
           |> Map.put(:origin_plug, "Python.#{py_mod}" |> String.to_atom())
           |> Map.put(:origin_msg_id, event.msg_id)
           |> Map.to_list()
-          |> S.ResponseToPost.new_bare()
+          |> S.Events.ResponseToPost.new_bare()
 
         other ->
           raise("""
@@ -108,6 +108,9 @@ defmodule Stampede.External.Python do
     |> List.to_tuple()
   end
 
+  defp do_dumb_down_elixir_term([h]),
+    do: [dumb_down_elixir_term(h)]
+
   defp do_dumb_down_elixir_term([h | t]),
     do: [dumb_down_elixir_term(h) | do_dumb_down_elixir_term(t)]
 
diff --git a/test/foreign_plugins_test.exs b/test/foreign_plugins_test.exs
index 1dcaf1c..f504bee 100644
--- a/test/foreign_plugins_test.exs
+++ b/test/foreign_plugins_test.exs
@@ -4,7 +4,7 @@ defmodule ForeignPluginsTest do
   alias Services.Dummy, as: D
   alias Stampede, as: S
   import AssertValue
-  require S.MsgReceived
+  require S.Events.MsgReceived
 
   @dummy_cfg_verified %{
     service: Services.Dummy,
@@ -55,14 +55,14 @@ defmodule ForeignPluginsTest do
         |> Stampede.External.Python.dumb_down_elixir_term()
 
       msg =
-        S.MsgReceived.new(
+        S.Events.MsgReceived.new(
           id: 0,
           body: "!ping python",
           channel_id: :t1,
           author_id: :u1,
           server_id: :none
         )
-        |> S.MsgReceived.add_context(@dummy_cfg_verified)
+        |> S.Events.MsgReceived.add_context(@dummy_cfg_verified)
         |> Stampede.External.Python.dumb_down_elixir_term()
 
       assert_value Stampede.External.Python.Pool.command(:example, :process, [cfg, msg]) ==

From c9da4f3c230d63296784463d7088bf38d6325d73 Mon Sep 17 00:00:00 2001
From: Producer Matt <58014742+ProducerMatt@users.noreply.github.com>
Date: Thu, 25 Jul 2024 15:48:24 -0500
Subject: [PATCH 8/8] chore(flake): update lock

---
 flake.lock | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/flake.lock b/flake.lock
index ca55a37..d023ff0 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1718879355,
-        "narHash": "sha256-RTyqP4fBX2MdhNuMP+fnR3lIwbdtXhyj7w7fwtvgspc=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "8cd35b9496d21a6c55164d8547d9d5280162b07a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -78,11 +78,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1718318537,
-        "narHash": "sha256-4Zu0RYRcAY/VWuu6awwq4opuiD//ahpc2aFHg2CWqFY=",
+        "lastModified": 1721743106,
+        "narHash": "sha256-adRZhFpBTnHiK3XIELA3IBaApz70HwCYfv7xNrHjebA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e9ee548d90ff586a6471b4ae80ae9cfcbceb3420",
+        "rev": "dc14ed91132ee3a26255d01d8fd0c1f5bff27b2f",
         "type": "github"
       },
       "original": {
@@ -94,16 +94,16 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1718447546,
-        "narHash": "sha256-JHuXsrC9pr4kA4n7LuuPfWFJUVlDBVJ1TXDVpHEuUgM=",
+        "lastModified": 1720386169,
+        "narHash": "sha256-NGKVY4PjzwAa4upkGtAMz1npHGoRzWotlSnVlqI40mo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "842253bf992c3a7157b67600c2857193f126563a",
+        "rev": "194846768975b7ad2c4988bdb82572c00222c0d7",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }