Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Facilitate overall system debugging when using the Kubernetes.DNS strategy #132

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 56 additions & 9 deletions lib/strategy/kubernetes_dns.ex
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@ defmodule Cluster.Strategy.Kubernetes.DNS do
It will fetch the addresses of all pods under a shared headless service and attempt to connect.
It will continually monitor and update its connections every 5s.

It assumes that all Erlang nodes were launched under a base name, are using longnames, and are unique
based on their FQDN, rather than the base hostname. In other words, in the following
longname, `<basename>@<ip>`, `basename` would be the value configured through
`application_name`.
It assumes that all Erlang nodes were launched under a base name, are using longnames,
and are unique based on their FQDN, rather than the base hostname.
In other words, by default it uses node names given by the following function:

fn application_name, ip ->
:"\#{application_name}@\#{ip}"
end

An example configuration is below:

Expand All @@ -20,8 +23,38 @@ defmodule Cluster.Strategy.Kubernetes.DNS do
config: [
service: "myapp-headless",
application_name: "myapp",
polling_interval: 10_000]]]
polling_interval: 10_000, # optional
node_naming: [MyModule, :my_node_naming, [extra_arg]] # optional
]
]
]

You can also use DNS based naming by passing your own custom function in the
`node_naming` option under `config`.
For example, to be able to establish a remote shell and run observer in a running
system, some people might think in a few tricks involving forwarding BEAM ports
and changing the dev machine's `/etc/hosts` (a workaround the fact the dev machine
is usually not connected to the internal Kubernetes network).
Assuming that they are using regular Deployment objects (no StatefulSet or
hostname configuration), that would require a custom naming compatible to Kubernetes DNS,
similar to the following:

@spec my_node_naming(String.t(), String.t()) :: node()
def my_node_naming(application_name, ip) do
:"\#{application_name}@\#{String.replace(ip, ".", "-")}.default.pod.cluster.local"
end

Of course, to use a custom naming schema, please make sure to change the
BEAM arguments accordingly on the release configuration
(See `Cluster.Strategy.Kubernetes` for an example).

Please notice that when using configuration files the `node_naming` option is
better given as `[module(), function_name :: atom(), extra_args :: [any()]]`,
since this kind of file is compiled into plain Erlang terms and therefore
don't support anonymous functions. In the case a list is provided, it will be
invoked via `Kernel.apply/3`, and the `extra_args` will be appended to the
application name and IP. Two-argument anonymous functions can be used
normally when passing options inline, directly to the supervisor.
"""
use GenServer
use Cluster.Strategy
Expand Down Expand Up @@ -109,14 +142,15 @@ defmodule Cluster.Strategy.Kubernetes.DNS do
app_name = Keyword.fetch!(config, :application_name)
service = Keyword.fetch!(config, :service)
resolver = Keyword.get(config, :resolver, &:inet_res.getbyname(&1, :a))
node_naming = Keyword.get(config, :node_naming, &default_node_naming/2)

cond do
app_name != nil and service != nil ->
headless_service = to_charlist(service)

case resolver.(headless_service) do
{:ok, {:hostent, _fqdn, [], :inet, _value, addresses}} ->
parse_response(addresses, app_name)
parse_response(addresses, app_name, node_naming)

{:error, reason} ->
error(topology, "lookup against #{service} failed: #{inspect(reason)}")
Expand Down Expand Up @@ -145,10 +179,23 @@ defmodule Cluster.Strategy.Kubernetes.DNS do
Keyword.get(config, :polling_interval, @default_polling_interval)
end

defp parse_response(addresses, app_name) do
@doc "Assumes the BEAM node uses a long name composed by the app name and the IP."
@spec default_node_naming(String.t(), String.t()) :: node()
def default_node_naming(app_name, ip) do
:"#{app_name}@#{ip}"
end

defp parse_response(addresses, app_name, [module, function, extra_args])
when is_atom(module)
when is_atom(function)
when is_list(extra_args) do
parse_response(addresses, app_name, &apply(module, function, [&1, &2 | extra_args]))
end

defp parse_response(addresses, app_name, node_naming) do
addresses
|> Enum.map(&:inet_parse.ntoa(&1))
|> Enum.map(&"#{app_name}@#{&1}")
|> Enum.map(&String.to_atom(&1))
|> Enum.map(&to_string/1)
|> Enum.map(&node_naming.(app_name, &1))
end
end
58 changes: 58 additions & 0 deletions test/kubernetes_dns_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,62 @@ defmodule Cluster.Strategy.KubernetesDNSTest do
end)
end
end

describe "KubernetesDNS configs" do
test "allow custom node naming" do
node_naming = fn app_name, ip ->
:"#{app_name}@#{String.replace(ip, ".", "-")}.default.pod.cluster.local"
end

capture_log(fn ->
[%State{
topology: :k8s_dns_example,
config: [
polling_interval: 100,
service: "app",
application_name: "node",
node_naming: node_naming,
resolver: fn _query ->
{:ok, {:hostent, 'app', [], :inet, 4, [{10, 0, 0, 1}, {10, 0, 0, 2}]}}
end
],
connect: {Nodes, :connect, [self()]},
disconnect: {Nodes, :disconnect, [self()]},
list_nodes: {Nodes, :list_nodes, [[]]}
}]
|> DNS.start_link()

assert_receive {:connect, :"[email protected]"}, 100
assert_receive {:connect, :"[email protected]"}, 100
end)
end

def dummy_node_naming(app_name, ip, arg1, arg2) do
:"#{app_name}@#{arg1}.#{arg2}.#{ip}"
end

test "allow node naming as [module, function, [args]]" do
capture_log(fn ->
[%State{
topology: :k8s_dns_example,
config: [
polling_interval: 100,
service: "app",
application_name: "node",
node_naming: [__MODULE__, :dummy_node_naming, ['extra', 'args']],
resolver: fn _query ->
{:ok, {:hostent, 'app', [], :inet, 4, [{10, 0, 0, 1}, {10, 0, 0, 2}]}}
end
],
connect: {Nodes, :connect, [self()]},
disconnect: {Nodes, :disconnect, [self()]},
list_nodes: {Nodes, :list_nodes, [[]]}
}]
|> DNS.start_link()

assert_receive {:connect, :"[email protected]"}, 100
assert_receive {:connect, :"[email protected]"}, 100
end)
end
end
end