APIs for metadata

backend/lib/azimutt/projects.ex

@@ -147,6 +147,16 @@ defmodule Azimutt.Projects do
     end
   end
 
+  def update_project_content(project_id, current_user, now, f) do
+    with {:ok, %Project{} = project} <- get_project(project_id, current_user),
+         {:ok, content} <- get_project_content(project),
+         {:ok, json} <- Jason.decode(content),
+         json_updated = f.(json),
+         {:ok, content_updated} <- Jason.encode(json_updated),
+         {:ok, %Project{} = _project_updated} <- update_project_file(project, content_updated, current_user, now),
+         do: {:ok, json_updated}
+  end
+
   def list_project_tokens(project_id, %User{} = current_user, now) do
     project_query()
     |> where([p, _, om], p.id == ^project_id and p.storage_kind == :remote and om.user_id == ^current_user.id)

backend/lib/azimutt/utils/mapx.ex

@@ -1,5 +1,6 @@
 defmodule Azimutt.Utils.Mapx do
   @moduledoc "Helper functions on Map."
+  alias Azimutt.Utils.Mapx
   alias Azimutt.Utils.Result
   alias Azimutt.Utils.Stringx
 
@@ -56,6 +57,36 @@ defmodule Azimutt.Utils.Mapx do
     end
   end
 
+  @doc """
+  Similar to https://hexdocs.pm/elixir/Kernel.html#put_in/3 but does not raise.
+  """
+  def put_in(enumerable, keys, value) do
+    case keys do
+      [key | rest] ->
+        default = rest |> Enum.reverse() |> Enum.reduce(value, fn key, acc -> %{} |> Map.put(key, acc) end)
+        enumerable |> Map.update(key, default, fn v -> Mapx.put_in(v || %{}, rest, value) end)
+
+      [] ->
+        value
+    end
+  end
+
+  def update_in(enumerable, keys, f) do
+    case keys do
+      [key | rest] ->
+        value = Map.get(enumerable, key)
+
+        cond do
+          value == nil -> enumerable |> Mapx.put_in(keys, f.(nil))
+          Enum.empty?(rest) -> enumerable |> Map.put(key, f.(value))
+          true -> enumerable |> Map.update(key, nil, fn v -> Mapx.update_in(v || %{}, rest, f) end)
+        end
+
+      [] ->
+        f.(nil)
+    end
+  end
+
   @doc """
   Remove a key if it's present with the expected value, or set it
   ## Examples

backend/lib/azimutt_web/controllers/api/gallery_controller.ex

@@ -5,7 +5,7 @@ defmodule AzimuttWeb.Api.GalleryController do
   action_fallback AzimuttWeb.Api.FallbackController
 
   swagger_path :index do
-    get("/api/v1/gallery")
+    get("/gallery")
     summary("List sample projects")
     description("List sample projects")
     produces("application/json")

backend/lib/azimutt_web/controllers/api/metadata_controller.ex

@@ -0,0 +1,240 @@
+defmodule AzimuttWeb.Api.MetadataController do
+  use AzimuttWeb, :controller
+  use PhoenixSwagger
+  alias Azimutt.Projects
+  alias Azimutt.Projects.Project
+  alias Azimutt.Utils.Mapx
+  alias Azimutt.Utils.Result
+  alias AzimuttWeb.Utils.JsonSchema
+  alias AzimuttWeb.Utils.ProjectSchema
+  alias AzimuttWeb.Utils.SwaggerCommon
+  action_fallback AzimuttWeb.Api.FallbackController
+
+  swagger_path :index do
+    tag("Metadata")
+    summary("Get project metadata")
+    description("Get all metadata for the project, ie all notes and tags for all tables and columns.")
+    produces("application/json")
+    get("#{SwaggerCommon.project_path()}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    response(200, "OK", Schema.ref(:ProjectMetadata))
+    response(400, "Client Error")
+  end
+
+  def index(conn, %{"organization_id" => _organization_id, "project_id" => project_id}) do
+    with {:ok, %Project{} = project} <- Projects.get_project(project_id, conn.assigns.current_user),
+         {:ok, content} <- Projects.get_project_content(project) |> Result.flat_map(fn c -> Jason.decode(c) end),
+         do: conn |> render("index.json", metadata: content["metadata"] || %{})
+  end
+
+  swagger_path :update do
+    tag("Metadata")
+    summary("Update project metadata")
+    description("Set the whole project metadata at once. Fetch it, update it then update it. Beware to not override changes made by others.")
+    produces("application/json")
+    put("#{SwaggerCommon.project_path()}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    parameters do
+      payload(:body, :object, "Project Metadata", required: true, schema: Schema.ref(:ProjectMetadata))
+    end
+
+    response(200, "OK", Schema.ref(:ProjectMetadata))
+    response(400, "Client Error")
+  end
+
+  def update(conn, %{"organization_id" => _organization_id, "project_id" => project_id}) do
+    with {:ok, body} <- conn.body_params |> JsonSchema.validate(ProjectSchema.metadata()) |> Result.zip_error_left(:bad_request),
+         {:ok, updated} <-
+           Projects.update_project_content(project_id, conn.assigns.current_user, DateTime.utc_now(), fn project ->
+             project |> Map.put("metadata", body)
+           end),
+         do: conn |> render("index.json", metadata: updated["metadata"] || %{})
+  end
+
+  swagger_path :table do
+    tag("Metadata")
+    summary("Get table metadata")
+    description("Get all metadata for the table, notes and tags. You can include columns metadata too with the `expand` query param.")
+    produces("application/json")
+    get("#{SwaggerCommon.project_path()}/tables/{table_id}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    parameters do
+      table_id(:path, :string, "Id of the table (ex: public.users)", required: true)
+      expand(:query, :array, "Expand columns metadata", collectionFormat: "csv", items: %{type: "string", enum: ["columns"]})
+    end
+
+    response(200, "OK", Schema.ref(:TableMetadata))
+    response(400, "Client Error")
+  end
+
+  def table(conn, %{"organization_id" => _organization_id, "project_id" => project_id, "table_id" => table_id} = params) do
+    with {:ok, %Project{} = project} <- Projects.get_project(project_id, conn.assigns.current_user),
+         {:ok, content} <- Projects.get_project_content(project) |> Result.flat_map(fn c -> Jason.decode(c) end),
+         do: conn |> render("table.json", metadata: content["metadata"][table_id] || %{}, show_columns: params["expand"] == "columns")
+  end
+
+  swagger_path :table_update do
+    tag("Metadata")
+    summary("Update table metadata")
+    description("Set table metadata. If you include columns, they will be replaced, otherwise they will stay the same.")
+    produces("application/json")
+    put("#{SwaggerCommon.project_path()}/tables/{table_id}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    parameters do
+      table_id(:path, :string, "Id of the table (ex: public.users)", required: true)
+      payload(:body, :object, "Table Metadata", required: true, schema: Schema.ref(:TableMetadata))
+    end
+
+    response(200, "OK", Schema.ref(:TableMetadata))
+    response(400, "Client Error")
+  end
+
+  def table_update(conn, %{"organization_id" => _organization_id, "project_id" => project_id, "table_id" => table_id}) do
+    with {:ok, body} <- conn.body_params |> JsonSchema.validate(ProjectSchema.table_meta()) |> Result.zip_error_left(:bad_request),
+         {:ok, updated} <-
+           Projects.update_project_content(project_id, conn.assigns.current_user, DateTime.utc_now(), fn project ->
+             project
+             |> Mapx.update_in(["metadata", table_id], fn table ->
+               cond do
+                 body["columns"] -> body
+                 table["columns"] -> body |> Map.put("columns", table["columns"])
+                 true -> body
+               end
+             end)
+           end),
+         do: conn |> render("table.json", metadata: updated["metadata"][table_id] || %{}, show_columns: not is_nil(body["columns"]))
+  end
+
+  swagger_path :column do
+    tag("Metadata")
+    summary("Get column metadata")
+    description("Get all metadata for the column, notes and tags. For nested columns, use the column path (ex: details:address:street).")
+    produces("application/json")
+    get("#{SwaggerCommon.project_path()}/tables/{table_id}/columns/{column_path}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    parameters do
+      table_id(:path, :string, "Id of the table (ex: public.users)", required: true)
+      column_path(:path, :string, "Path of the column (ex: id, name or details:location)", required: true)
+    end
+
+    response(200, "OK", Schema.ref(:ColumnMetadata))
+    response(400, "Client Error")
+  end
+
+  def column(conn, %{"organization_id" => _organization_id, "project_id" => project_id, "table_id" => table_id, "column_path" => column_path}) do
+    with {:ok, %Project{} = project} <- Projects.get_project(project_id, conn.assigns.current_user),
+         {:ok, content} <- Projects.get_project_content(project) |> Result.flat_map(fn c -> Jason.decode(c) end),
+         do: conn |> render("column.json", metadata: content["metadata"][table_id]["columns"][column_path] || %{})
+  end
+
+  swagger_path :column_update do
+    tag("Metadata")
+    summary("Update column metadata")
+    description("Set column metadata. For nested columns, use the column path (ex: details:address:street).")
+    produces("application/json")
+    put("#{SwaggerCommon.project_path()}/tables/{table_id}/columns/{column_path}/metadata")
+    SwaggerCommon.authorization()
+    SwaggerCommon.project_params()
+
+    parameters do
+      table_id(:path, :string, "Id of the table (ex: public.users)", required: true)
+      column_path(:path, :string, "Path of the column (ex: id, name or details:location)", required: true)
+      payload(:body, :object, "Column Metadata", required: true, schema: Schema.ref(:ColumnMetadata))
+    end
+
+    response(200, "OK", Schema.ref(:ColumnMetadata))
+    response(400, "Client Error")
+  end
+
+  def column_update(conn, %{"organization_id" => _organization_id, "project_id" => project_id, "table_id" => table_id, "column_path" => column_path}) do
+    with {:ok, body} <- conn.body_params |> JsonSchema.validate(ProjectSchema.column_meta()) |> Result.zip_error_left(:bad_request),
+         {:ok, updated} <-
+           Projects.update_project_content(project_id, conn.assigns.current_user, DateTime.utc_now(), fn project ->
+             project |> Mapx.put_in(["metadata", table_id, "columns", column_path], body)
+           end),
+         do: conn |> render("column.json", metadata: updated["metadata"][table_id]["columns"][column_path] || %{})
+  end
+
+  def swagger_definitions do
+    %{
+      ProjectMetadata:
+        swagger_schema do
+          title("ProjectMetadata")
+          description("All Metadata of the project")
+          type(:object)
+          additional_properties(Schema.ref(:TableMetadata))
+
+          example(%{
+            "public.users": %{
+              notes: "Table notes",
+              tags: ["table-tag"],
+              columns: %{
+                id: %{
+                  notes: "Column notes",
+                  tags: ["column-tag"]
+                },
+                "settings:theme": %{
+                  notes: "Nested column notes",
+                  tags: ["nested-column-tag"]
+                }
+              }
+            },
+            ".test": %{
+              notes: "Table with empty schema"
+            }
+          })
+        end,
+      TableMetadata:
+        swagger_schema do
+          title("TableMetadata")
+          description("The Metadata used to document tables")
+
+          properties do
+            notes(:string, "Markdown text to document the table", example: "*Table* notes")
+            tags(:array, "Tags to categorize the table", items: %{type: :string}, example: ["table-tag"])
+            columns(:object, "Columns metadata", additionalProperties: Schema.ref(:ColumnMetadata))
+          end
+
+          example(%{
+            notes: "Table notes",
+            tags: ["table-tag"],
+            columns: %{
+              id: %{
+                notes: "Column notes",
+                tags: ["column-tag"]
+              },
+              "settings:theme": %{
+                notes: "Nested column notes",
+                tags: ["nested-column-tag"]
+              }
+            }
+          })
+        end,
+      ColumnMetadata:
+        swagger_schema do
+          title("ColumnMetadata")
+          description("The Metadata used to document columns")
+
+          properties do
+            notes(:string, "Markdown text to document the column", example: "*Column* notes")
+            tags(:array, "Tags to categorize the column", items: %{type: :string}, example: ["column-tag"])
+          end
+
+          example(%{
+            notes: "Column notes",
+            tags: ["column-tag"]
+          })
+        end
+    }
+  end
+end

backend/lib/azimutt_web/controllers/api/project_controller.ex

@@ -9,7 +9,7 @@ defmodule AzimuttWeb.Api.ProjectController do
   action_fallback AzimuttWeb.Api.FallbackController
 
   swagger_path :index do
-    get("/api/v1/projects")
+    get("/projects")
     summary("Query for projects")
     description("Query for projects. This operation supports with paging and filtering")
     produces("application/json")
@@ -36,7 +36,7 @@ defmodule AzimuttWeb.Api.ProjectController do
   end
 
   swagger_path :create do
-    post("/api/v1/organizations/:organization_id/projects")
+    post("/organizations/:organization_id/projects")
     summary("Create a project")
     description("TODO")
     produces("application/json")

backend/lib/azimutt_web/controllers/api/source_controller.ex

@@ -6,6 +6,7 @@ defmodule AzimuttWeb.Api.SourceController do
   alias Azimutt.Utils.Mapx
   alias Azimutt.Utils.Result
   alias AzimuttWeb.Utils.CtxParams
+  alias AzimuttWeb.Utils.JsonSchema
   alias AzimuttWeb.Utils.ProjectSchema
   action_fallback AzimuttWeb.Api.FallbackController
 
@@ -48,7 +49,7 @@ defmodule AzimuttWeb.Api.SourceController do
       "definitions" => %{"column" => ProjectSchema.column()}
     }
 
-    with {:ok, body} <- validate_json_schema(create_schema, conn.body_params) |> Result.zip_error_left(:bad_request),
+    with {:ok, body} <- conn.body_params |> JsonSchema.validate(create_schema) |> Result.zip_error_left(:bad_request),
          {:ok, %Project{} = project} <- Projects.get_project(project_id, current_user),
          {:ok, content} <- Projects.get_project_content(project),
          {:ok, json} <- Jason.decode(content),
@@ -77,7 +78,7 @@ defmodule AzimuttWeb.Api.SourceController do
       "definitions" => %{"column" => ProjectSchema.column()}
     }
 
-    with {:ok, body} <- validate_json_schema(update_schema, conn.body_params) |> Result.zip_error_left(:bad_request),
+    with {:ok, body} <- conn.body_params |> JsonSchema.validate(update_schema) |> Result.zip_error_left(:bad_request),
          {:ok, %Project{} = project} <- Projects.get_project(project_id, current_user),
          {:ok, content} <- Projects.get_project_content(project),
          {:ok, json} <- Jason.decode(content),
@@ -118,13 +119,4 @@ defmodule AzimuttWeb.Api.SourceController do
     |> Map.put("relations", params["relations"])
     |> Mapx.put_no_nil("types", params["types"])
   end
-
-  defp validate_json_schema(schema, json) do
-    # TODO: add the string uuid format validation
-    ExJsonSchema.Validator.validate(schema, json)
-    |> Result.map_both(
-      fn errors -> %{errors: errors |> Enum.map(fn {error, path} -> %{path: path, error: error} end)} end,
-      fn _ -> json end
-    )
-  end
 end

backend/lib/azimutt_web/router.ex

@@ -194,7 +194,7 @@ defmodule AzimuttWeb.Router do
     resources("/events", Admin.EventController, param: "event_id", only: [:index, :show])
   end
 
-  scope "/api/v1/swagger" do
+  scope "/api/v1" do
     forward("/", PhoenixSwagger.Plug.SwaggerUI, otp_app: :azimutt, swagger_file: "swagger.json")
   end
 
@@ -204,6 +204,12 @@ defmodule AzimuttWeb.Router do
     get("/gallery", Api.GalleryController, :index)
     get("/organizations/:organization_id/projects/:project_id", Api.ProjectController, :show)
     resources("/organizations/:organization_id/projects/:project_id/sources", Api.SourceController, param: "source_id", only: [:index, :show, :create, :update, :delete])
+    get("/organizations/:organization_id/projects/:project_id/metadata", Api.MetadataController, :index)
+    put("/organizations/:organization_id/projects/:project_id/metadata", Api.MetadataController, :update)
+    get("/organizations/:organization_id/projects/:project_id/tables/:table_id/metadata", Api.MetadataController, :table)
+    put("/organizations/:organization_id/projects/:project_id/tables/:table_id/metadata", Api.MetadataController, :table_update)
+    get("/organizations/:organization_id/projects/:project_id/tables/:table_id/columns/:column_path/metadata", Api.MetadataController, :column)
+    put("/organizations/:organization_id/projects/:project_id/tables/:table_id/columns/:column_path/metadata", Api.MetadataController, :column_update)
     post("/events", Api.TrackingController, :create)
   end
 
@@ -260,6 +266,7 @@ defmodule AzimuttWeb.Router do
           email: Azimutt.config(:azimutt_email)
         }
       },
+      basePath: "/api/v1",
       consumes: ["application/json"],
       produces: ["application/json"]
     }