Skip to content

Commit

Permalink
remove blacklist func
Browse files Browse the repository at this point in the history
  • Loading branch information
MaxHalford committed Oct 17, 2023
1 parent b30483a commit 50921fb
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 47 deletions.
14 changes: 4 additions & 10 deletions examples/jaffle_shop/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB
analytics(analytics)
staging(staging)
core(core)
staging --> core
staging(staging)
core --> analytics
staging --> core
```

## Flowchart
Expand All @@ -24,26 +24,20 @@ flowchart TB
%%{init: {"flowchart": {"defaultRenderer": "elk"}} }%%
flowchart TB
subgraph analytics
analytics.kpis(kpis)
end
subgraph core
core.customers(customers)
core.orders(orders)
end
subgraph staging
staging.customers(customers)
staging.orders(orders)
staging.payments(payments)
end
core.customers --> analytics.kpis
core.orders --> analytics.kpis
staging.customers --> core.customers
staging.orders --> core.customers
staging.payments --> core.customers
staging.orders --> core.orders
staging.payments --> core.orders
core.customers --> analytics.kpis
core.orders --> analytics.kpis
```

3 changes: 0 additions & 3 deletions lea/app/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ def run(
):
from lea.app.run import run

# Massage CLI inputs
only = [tuple(v.split(".")) for v in only] if only else None

# The client determines where the views will be written
client = _make_client(production)

Expand Down
39 changes: 5 additions & 34 deletions lea/app/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,37 +33,6 @@ def pretty_print_view(view: lea.views.View, console: rich.console.Console) -> st
console.print(syntax)


def make_blacklist(dag: lea.views.DAGOfViews, only: list) -> set:
"""
Basic implementation of dbt-like graph operators.
References
----------
https://docs.getdbt.com/reference/node-selection/graph-operators
"""

# Start by assuming that all views are blacklisted
blacklist = set(dag.keys())

for schema, table in only:
# Ancestors
if schema.startswith("+"):
blacklist.difference_update(dag.list_ancestors((schema[1:], table)))
schema = schema[1:]

# Descendants
if table.endswith("+"):
blacklist.difference_update(dag.list_descendants((schema, table[:-1])))
table = table[:-1]

# The node itself
blacklist.remove((schema, table))

return blacklist


def make_whitelist(query: str, dag: lea.views.DAGOfViews) -> set:
"""Make a whitelist of tables given a query.
Expand Down Expand Up @@ -213,8 +182,10 @@ def run(
dag = lea.views.DAGOfViews(views)

# Determine which views need to be run
blacklist = make_blacklist(dag, only) if only else set()
console_log(f"{len(views) - len(blacklist):,d} view(s) selected")
whitelist = (
set.union(*(make_whitelist(query, dag) for query in only)) if only else set(dag.keys())
)
console_log(f"{len(whitelist):,d} view(s) selected")

# Remove orphan views
for schema, table in client.list_existing_view_names():
Expand Down Expand Up @@ -280,7 +251,7 @@ def display_progress() -> rich.table.Table:
# they're external dependencies which can be ignored
node not in dag
# Some nodes are blacklisted, so we skip them
or node in blacklist
or node not in whitelist
):
dag.done(node)
continue
Expand Down

0 comments on commit 50921fb

Please sign in to comment.