merging everything back to main

merging mst branch changes (not finished), updated rustler, removed unnecessary hex dep, removed unnecessary closure in dagcbor crate

lib/hipdster/blockstore.ex

@@ -0,0 +1,129 @@
+defmodule Hipdster.BlockStore do
+  @callback put_block(key :: binary(), value :: binary()) :: :ok | {:error, term()}
+  @callback get_block(key :: binary()) :: {:ok, binary()} | {:error, term()}
+  @callback del_block(key :: binary()) :: :ok | {:error, term()}
+end
+
+defmodule Hipdster.BlockStoreServer do
+  use GenServer
+  @behaviour Hipdster.BlockStore
+
+  def init(init_arg) do
+    {:ok, init_arg}
+  end
+
+  def start_link(inital_state \\ %{}) do
+    GenServer.start_link(__MODULE__, inital_state, name: __MODULE__)
+  end
+
+  def put_block(key, value) do
+    GenServer.call(__MODULE__, {:put_block, key, value})
+  end
+
+  def get_block(key) do
+    GenServer.call(__MODULE__, {:get_block, key})
+  end
+
+  def del_block(key) do
+    GenServer.cast(__MODULE__, {:del_block, key})
+  end
+
+  def init() do
+    {:ok, :state}
+  end
+
+  def handle_call({:put_block, key, value}, _from, state) do
+    case Map.fetch(state, key) do
+      {:ok, ^value} -> {:reply, :ok, state}
+      {:ok, _} -> {:reply, {:error, :immutable_value}, state}
+      :error -> {:reply, :ok, Map.put(state, key, value)}
+    end
+  end
+
+  def handle_call({:get_block, key}, _from, state) do
+    case Map.fetch(state, key) do
+      {:ok, value} -> {:reply, {:ok, value}, state}
+      :error -> {:reply, {:error, :not_found}, state}
+    end
+  end
+
+  def handle_cast({:del_block, key}, state) do
+    {:noreply, Map.delete(state, key)}
+  end
+end
+
+defmodule BlockStore.Repo do
+  use Ecto.Repo,
+    otp_app: :blockstore_app,
+    adapter: Ecto.Adapters.SQLite3
+end
+
+defmodule BlocksTable do
+  use Ecto.Schema
+
+  schema "blocks" do
+    field(:key, :string)
+    field(:value, :string)
+  end
+end
+
+defmodule Hipdster.EctoBlockStore do
+  use Ecto.Repo, otp_app: :hipdster, adapter: Ecto.Adapters.SQLite3
+  import Ecto.Query
+  @behaviour Hipdster.BlockStore
+
+  def init(_type, config) do
+    {:ok, Keyword.put(config, :database, :memory)}
+  end
+
+  def put_block(key, value) do
+    case get_by(BlocksTable, key: key) do
+      nil ->
+        case insert!(%BlocksTable{key: key, value: value}) do
+          {:ok, _} -> :ok
+          {:error, _} -> {:error, :insert_failed}
+        end
+
+      {:ok, res} when res == value ->
+        :ok
+
+      {:ok, _} ->
+        {:error, :different_value}
+
+      {:error, _} ->
+        {:error, :get_failed}
+    end
+  end
+
+  def get_block(key) do
+    case get_by(BlocksTable, key: key) do
+      nil -> {:error, :not_found}
+      block -> {:ok, block.value}
+    end
+  end
+
+  def del_block(key) do
+    delete_all(from(b in BlocksTable, where: b.key == ^key))
+  end
+end
+
+defmodule Hipdster.OverlayBlockStore do
+  @behaviour Hipdster.BlockStore
+
+  defstruct upper: nil, lower: nil
+
+  def put_block(%{upper: upper}, key, value) do
+    Hipdster.BlockStore.put_block(upper, key, value)
+  end
+
+  def get_block(%{upper: upper, lower: lower}, key) do
+    case BlockStore.get_block(upper, key) do
+      {:error, :not_found} -> BlockStore.get_block(lower, key)
+      result -> result
+    end
+  end
+
+  def del_block(%{upper: upper}, key) do
+    BlockStore.del_block(upper, key)
+  end
+end

lib/hipdster/mst.ex

@@ -1 +1,130 @@
+defmodule Hipdster.MST do
+  alias Hipdster.MST
+  alias Hipdster.MST.MSTNode, as: MSTNode
+  @hash_fun :sha256
 
+  def terminal_node?(%MSTNode{subtrees: subtrees}) do
+    Enum.empty?(subtrees)
+  end
+
+  def build_tree(data_list) when is_list(data_list) do
+    leaves = Enum.map(data_list, &create_node(&1))
+    build_nodes(leaves, nil)
+  end
+
+  defp create_node(data) do
+    # hash = :crypto.hash(@hash_fun, data)
+    # %MSTNode{hash: hash, subtrees: []}
+  end
+
+  defp build_nodes([single_node], _parent) do
+    single_node
+  end
+
+  # defp build_nodes(nodes, _parent) do
+  #   paired_nodes = Enum.chunk_every(nodes, 2, 2, :undefined)
+
+  #   parent_nodes =
+  #     Enum.map(paired_nodes, fn [left = %MSTNode{hash: left_hash}, right] ->
+  #       right_hash =
+  #         case right do
+  #           # %MSTNode{hash: hash} -> hash
+  #           :undefined -> ""
+  #           _ -> :crypto.hash(@hash_fun, "")
+  #         end
+  #     end)
+
+  #   build_nodes(parent_nodes, nil)
+  # end
+
+  defp build_nodes([single_hash], _parent) do
+    # This is a single node tree (or a root for its subtree), no parent is needed.
+    single_hash
+  end
+
+  defp build_nodes(hashes, parent) do
+    paired_hashes = Enum.chunk_every(hashes, 2, 2, :undefined)
+
+    parent_hashes =
+      Enum.map(paired_hashes, fn [left, right] ->
+        {left_hash, right_hash} = {left || :undefined, right || :undefined}
+        combined = if right_hash == :undefined, do: left_hash, else: left_hash <> right_hash
+        hash = :crypto.hash(@hash_fun, combined)
+        # Insert each parent node into the ETS table. Each parent node contains
+        # its hash, the hashes of its left and right children, and its own parent.
+        :ets.insert(:merkle_tree, {hash, left_hash, right_hash, parent})
+        hash
+      end)
+
+    build_nodes(parent_hashes, nil)
+  end
+
+  # defp leaf_hash(leaf) when is_nil(leaf), do: ""
+  # defp leaf_hash(%Hipdster.MST.MSTNode{hash: hash}), do: hash
+
+  def verify(tree_hash, data) do
+    # Hmmm
+  end
+
+  def depth(key), do: hash_depth(:crypto.hash(:sha256, key))
+  def hash_depth(key, depth \\ 0)
+
+  def hash_depth(<<0::2, rest::bitstring>>, depth), do: hash_depth(rest, depth + 1)
+
+  def hash_depth(<<_::2, _rest::bitstring>>, depth), do: depth
+end
+
+defmodule Hipdster.MSTServer do
+  use GenServer
+  alias Hipdster.MST.MSTNode, as: MSTNode
+
+  # Starting the server with an empty root
+  @spec start_link(any()) :: :ignore | {:error, any()} | {:ok, pid()}
+  def start_link(%MSTNode{subtrees: [nil], keys: [], vals: []} = args) do
+    GenServer.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  # GenServer callback for initialization
+  def init(_) do
+    # Initialize the ETS table with the name :merkle_tree, making it a set
+    # where each object is unique and public so all processes can access it.
+    # The :keypos option indicates the position of the key in the tuple (2nd position here).
+    table = :ets.new(:merkle_tree, [:set, :public, {:keypos, 2}])
+    {:ok, table}
+  end
+
+  # Other callbacks, get, put, etc.
+  def handle_call({:get, key}, _from, state) do
+    # To-do
+    # Pattern match on the state
+    # Recursively search the tree for the key
+    # {:reply, value, state}
+  end
+
+  def handle_call({:add_data, data}, from, state), do: {:reply, add_data(data, state)}
+
+  def handle_cast({:put, key, value}, state) do
+    # To-do
+    # Insert the key-value pair into the tree
+    # Recursively adjust the tree structure
+    # {:noreply, new_state}
+  end
+
+  def handle_cast({:delete, key}, state) do
+    # Implement logic to remove a key-value pair from the tree
+    # Adjust the tree structure accordingly
+    # {:noreply, new_state}
+  end
+
+  def add_data(data, table) do
+    hash = :crypto.hash(@hash_fun, data)
+    # Insert a tuple into the ETS table with the structure {hash, data}.
+    :ets.insert(table, {hash, data})
+    hash
+  end
+
+  # def build_tree(data_list) when is_list(data_list) do
+  #   hashed_data = Enum.map(data_list, &add_data())
+  #   build_nodes(hashed_data, nil)
+  # end
+end

lib/hipdster/mstnode.ex

@@ -0,0 +1,17 @@
+defmodule Hipdster.MST.MSTNode do
+  @moduledoc """
+  Represents a node in the merkle tree structure. It can function as both a
+  leaf (is_terminal_node) and an internal node.
+
+  - 'keys' - collection of rkeys
+  - 'vals' - CID references to data
+  - 'subtrees' - optional CID references to child nodes
+  """
+  @type t :: %__MODULE__{
+          keys: [String.t()],
+          vals: [Hipdster.CID.t()],
+          subtrees: [Hipdster.CID.t() | nil]
+        }
+
+  defstruct keys: [], vals: [], subtrees: []
+end

native/hipdster_auth_jwt_internal/Cargo.toml

@@ -12,4 +12,5 @@ crate-type = ["cdylib"]
 [dependencies]
 jwt-simple = "0.12.9"
 serde_json = "1.0"
-rustler = "0.32.1"
+rustler = "0.33.0"
+cmake = "0.1.50"

native/hipdster_dagcbor_internal/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "hipdster_dagcbor_internal"
 version = "0.1.0"
-authors = [ "NetWatchInc" ]
+authors = ["NetWatchInc"]
 edition = "2021"
 
 [lib]
@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 libipld = "0.16.0"
-rustler = "0.32.0"
+rustler = "0.33.0"
 data-encoding = "2.5.0"
 serde_json = "1.0.113"
 [features]