Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

mermaid flowchart - sub-schemas #13

Merged
merged 2 commits into from
Nov 16, 2023
Merged

mermaid flowchart - sub-schemas #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e815ad
support sub-schemas in mermaid flowchart
LeonardoNatale Nov 15, 2023
b284f3e
fix doctest
LeonardoNatale Nov 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions examples/jaffle_shop/docs/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,27 @@ flowchart TB
```mermaid
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB

subgraph analytics
analytics.finance.kpis(finance.kpis)
analytics.kpis(kpis)

subgraph finance
analytics.finance.kpis(kpis)
end

analytics.kpis(kpis)
end


subgraph core
core.customers(customers)
core.orders(orders)
core.customers(customers)
core.orders(orders)
end


subgraph staging
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end

core.orders --> analytics.finance.kpis
Expand Down
12 changes: 8 additions & 4 deletions examples/jaffle_shop/docs/analytics/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ SELECT *
FROM analytics.finance__kpis
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:--------------------|:---------|:--------------|:---------|
| average_order_value | `DOUBLE` | | |
| total_order_value | `DOUBLE` | | |

### analytics.kpis

Expand All @@ -24,6 +26,8 @@ SELECT *
FROM analytics.kpis
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------|:----------|:--------------|:---------|
| metric | `VARCHAR` | | |
| value | `BIGINT` | | |

24 changes: 20 additions & 4 deletions examples/jaffle_shop/docs/core/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,15 @@ SELECT *
FROM core.customers
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | ✅ |
| customer_lifetime_value | `DOUBLE` | | |
| first_name | `VARCHAR` | | |
| first_order | `VARCHAR` | | |
| last_name | `VARCHAR` | | |
| most_recent_order | `VARCHAR` | | |
| number_of_orders | `BIGINT` | | |

### core.orders

Expand All @@ -24,6 +31,15 @@ SELECT *
FROM core.orders
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------------------|:----------|:--------------|:---------|
| amount | `DOUBLE` | | |
| bank_transfer_amount | `DOUBLE` | | |
| coupon_amount | `DOUBLE` | | |
| credit_card_amount | `DOUBLE` | | |
| customer_id | `BIGINT` | | |
| gift_card_amount | `DOUBLE` | | |
| order_date | `VARCHAR` | | |
| order_id | `BIGINT` | | |
| status | `VARCHAR` | | |

23 changes: 17 additions & 6 deletions examples/jaffle_shop/docs/staging/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ SELECT *
FROM staging.customers
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | |
| first_name | `VARCHAR` | | |
| last_name | `VARCHAR` | | |

### staging.orders

Expand All @@ -29,8 +32,12 @@ SELECT *
FROM staging.orders
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:------------|:----------|:--------------|:---------|
| customer_id | `BIGINT` | | |
| order_date | `VARCHAR` | | |
| order_id | `BIGINT` | | |
| status | `VARCHAR` | | |

### staging.payments

Expand All @@ -39,6 +46,10 @@ SELECT *
FROM staging.payments
```

| Column | Type | Description | Unique |
|----------|--------|---------------|----------|
| Column | Type | Description | Unique |
|:---------------|:----------|:--------------|:---------|
| amount | `DOUBLE` | | |
| order_id | `BIGINT` | | |
| payment_id | `BIGINT` | | |
| payment_method | `VARCHAR` | | |

70 changes: 52 additions & 18 deletions lea/views/dag.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
import collections
import graphlib
import io
import itertools

import lea

FOUR_SPACES = " "


class DAGOfViews(graphlib.TopologicalSorter, collections.UserDict):
def __init__(self, views: list[lea.views.View]):
Expand Down Expand Up @@ -47,25 +48,51 @@ def _list_descendants(node):

return list(_list_descendants(node))

def _build_nested_schema(self, deps):
nested_schema = {}

for path in deps:
current_level = nested_schema
for part in path:
if part not in current_level:
current_level[part] = {}
current_level = current_level[part]

return nested_schema

def _to_mermaid_subgraphs(self, out, nested_schema: dict):
def output_subgraph(schema: str, values: dict, prefix: str = ""):
out.write(f"\n{FOUR_SPACES}subgraph {schema}\n")
for value in sorted(values.keys()):
sub_values = values[value]
path = f"{prefix}.{schema}" if prefix else schema
full_path = f"{path}.{value}"
if sub_values:
output_subgraph(value, sub_values, path)
else:
out.write(f"{FOUR_SPACES*2}{full_path}({value})\n")
out.write(f"{FOUR_SPACES}end\n\n")

for schema in sorted(nested_schema.keys()):
values = nested_schema[schema]
output_subgraph(schema, values)

def _to_mermaid_views(self):
out = io.StringIO()
out.write('%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%\n')
out.write("flowchart TB\n")
nodes = set(node for deps in self.dependencies.values() for node in deps) | set(
self.dependencies.keys()
)
schema_nodes = itertools.groupby(sorted(nodes), lambda node: node[0])
for schema, nodes in schema_nodes:
out.write(f" subgraph {schema}\n")
for _, *node in sorted(nodes):
node = ".".join(node)
out.write(f" {schema}.{node}({node})\n")
out.write(" end\n\n")

nested_schema = self._build_nested_schema(nodes)
self._to_mermaid_subgraphs(out, nested_schema)

for dst, srcs in sorted(self.dependencies.items()):
dst = ".".join(dst)
for src in sorted(srcs):
src = ".".join(src)
out.write(f" {src} --> {dst}\n")
out.write(f"{FOUR_SPACES}{src} --> {dst}\n")
return out.getvalue()

def _to_mermaid_schemas(self):
Expand All @@ -77,10 +104,10 @@ def _to_mermaid_schemas(self):
schema_dependencies.keys()
)
for node in sorted(nodes):
out.write(f" {node}({node})\n")
out.write(f"{FOUR_SPACES}{node}({node})\n")
for dst, srcs in sorted(schema_dependencies.items()):
for src in sorted(srcs):
out.write(f" {src} --> {dst}\n")
out.write(f"{FOUR_SPACES}{src} --> {dst}\n")
return out.getvalue()

def to_mermaid(self, schemas_only=False):
Expand All @@ -107,20 +134,27 @@ def to_mermaid(self, schemas_only=False):
>>> print(dag.to_mermaid())
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB
<BLANKLINE>
subgraph analytics
analytics.finance.kpis(finance.kpis)
analytics.kpis(kpis)
<BLANKLINE>
subgraph finance
analytics.finance.kpis(kpis)
end
<BLANKLINE>
analytics.kpis(kpis)
end
<BLANKLINE>
<BLANKLINE>
subgraph core
core.customers(customers)
core.orders(orders)
core.customers(customers)
core.orders(orders)
end
<BLANKLINE>
<BLANKLINE>
subgraph staging
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end
<BLANKLINE>
core.orders --> analytics.finance.kpis
Expand Down
Loading