Skip to content

Commit

Permalink
support sub-schemas in mermaid flowchart
Browse files Browse the repository at this point in the history
  • Loading branch information
@LeonardoNatale
LeonardoNatale committed Nov 15, 2023
1 parent 2c1f9d1 commit 4e815ad
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 40 deletions.
21 changes: 14 additions & 7 deletions examples/jaffle_shop/docs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,27 @@ flowchart TB
```mermaid
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB
subgraph analytics
analytics.finance.kpis(finance.kpis)
analytics.kpis(kpis)
subgraph finance
analytics.finance.kpis(kpis)
end
analytics.kpis(kpis)
end
subgraph core
core.customers(customers)
core.orders(orders)
core.customers(customers)
core.orders(orders)
end
subgraph staging
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end
core.orders --> analytics.finance.kpis
Expand Down
12 changes: 8 additions & 4 deletions examples/jaffle_shop/docs/analytics/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ SELECT *
FROM analytics.finance__kpis
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:--------------------|:---------|:--------------|:---------|
| average_order_value | `DOUBLE` | | |
| total_order_value | `DOUBLE` | | |

### analytics.kpis

Expand All @@ -24,6 +26,8 @@ SELECT *
FROM analytics.kpis
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------|:----------|:--------------|:---------|
| metric | `VARCHAR` | | |
| value | `BIGINT` | | |

24 changes: 20 additions & 4 deletions examples/jaffle_shop/docs/core/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,15 @@ SELECT *
FROM core.customers
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | ✅ |
| customer_lifetime_value | `DOUBLE` | | |
| first_name | `VARCHAR` | | |
| first_order | `VARCHAR` | | |
| last_name | `VARCHAR` | | |
| most_recent_order | `VARCHAR` | | |
| number_of_orders | `BIGINT` | | |

### core.orders

Expand All @@ -24,6 +31,15 @@ SELECT *
FROM core.orders
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------------------|:----------|:--------------|:---------|
| amount | `DOUBLE` | | |
| bank_transfer_amount | `DOUBLE` | | |
| coupon_amount | `DOUBLE` | | |
| credit_card_amount | `DOUBLE` | | |
| customer_id | `BIGINT` | | |
| gift_card_amount | `DOUBLE` | | |
| order_date | `VARCHAR` | | |
| order_id | `BIGINT` | | |
| status | `VARCHAR` | | |

23 changes: 17 additions & 6 deletions examples/jaffle_shop/docs/staging/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ SELECT *
FROM staging.customers
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | |
| first_name | `VARCHAR` | | |
| last_name | `VARCHAR` | | |

### staging.orders

Expand All @@ -29,8 +32,12 @@ SELECT *
FROM staging.orders
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | |
| order_date | `VARCHAR` | | |
| order_id | `BIGINT` | | |
| status | `VARCHAR` | | |

### staging.payments

Expand All @@ -39,6 +46,10 @@ SELECT *
FROM staging.payments
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------------|:----------|:--------------|:---------|
| amount | `DOUBLE` | | |
| order_id | `BIGINT` | | |
| payment_id | `BIGINT` | | |
| payment_method | `VARCHAR` | | |

72 changes: 53 additions & 19 deletions lea/views/dag.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
import collections
import graphlib
import io
import itertools

import lea

FOUR_SPACES = " "


class DAGOfViews(graphlib.TopologicalSorter, collections.UserDict):
def __init__(self, views: list[lea.views.View]):
Expand Down Expand Up @@ -47,25 +48,51 @@ def _list_descendants(node):

return list(_list_descendants(node))

def _build_nested_schema(self, deps):
nested_schema = {}

for path in deps:
current_level = nested_schema
for part in path:
if part not in current_level:
current_level[part] = {}
current_level = current_level[part]

return nested_schema

def _to_mermaid_subgraphs(self, out, nested_schema: dict):
def output_subgraph(schema: str, values: dict, prefix: str = ""):
out.write(f"\n{FOUR_SPACES}subgraph {schema}\n")
for value in sorted(values.keys()):
sub_values = values[value]
path = f"{prefix}.{schema}" if prefix else schema
full_path = f"{path}.{value}"
if sub_values:
output_subgraph(value, sub_values, path)
else:
out.write(f"{FOUR_SPACES*2}{full_path}({value})\n")
out.write(f"{FOUR_SPACES}end\n\n")

for schema in sorted(nested_schema.keys()):
values = nested_schema[schema]
output_subgraph(schema, values)

def _to_mermaid_views(self):
out = io.StringIO()
out.write('%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%\n')
out.write("flowchart TB\n")
nodes = set(node for deps in self.dependencies.values() for node in deps) | set(
self.dependencies.keys()
)
schema_nodes = itertools.groupby(sorted(nodes), lambda node: node[0])
for schema, nodes in schema_nodes:
out.write(f" subgraph {schema}\n")
for _, *node in sorted(nodes):
node = ".".join(node)
out.write(f" {schema}.{node}({node})\n")
out.write(" end\n\n")

nested_schema = self._build_nested_schema(nodes)
self._to_mermaid_subgraphs(out, nested_schema)

for dst, srcs in sorted(self.dependencies.items()):
dst = ".".join(dst)
for src in sorted(srcs):
src = ".".join(src)
out.write(f" {src} --> {dst}\n")
out.write(f"{FOUR_SPACES}{src} --> {dst}\n")
return out.getvalue()

def _to_mermaid_schemas(self):
Expand All @@ -77,10 +104,10 @@ def _to_mermaid_schemas(self):
schema_dependencies.keys()
)
for node in sorted(nodes):
out.write(f" {node}({node})\n")
out.write(f"{FOUR_SPACES}{node}({node})\n")
for dst, srcs in sorted(schema_dependencies.items()):
for src in sorted(srcs):
out.write(f" {src} --> {dst}\n")
out.write(f"{FOUR_SPACES}{src} --> {dst}\n")
return out.getvalue()

def to_mermaid(self, schemas_only=False):
Expand All @@ -107,21 +134,28 @@ def to_mermaid(self, schemas_only=False):
>>> print(dag.to_mermaid())
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB
<BLANKLINE>
subgraph analytics
analytics.finance.kpis(finance.kpis)
analytics.kpis(kpis)
<BLANKLINE>
subgraph finance
analytics.finance.kpis(kpis)
end
<BLANKLINE>
analytics.kpis(kpis)
end
<BLANKLINE>
<BLANKLINE>
subgraph core
core.customers(customers)
core.orders(orders)
core.customers(customers)
core.orders(orders)
end
<BLANKLINE>
<BLANKLINE>
subgraph staging
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end
<BLANKLINE>
core.orders --> analytics.finance.kpis
core.customers --> analytics.kpis
Expand Down

0 comments on commit 4e815ad

Please sign in to comment.