Skip to content

Commit

Permalink
Refactor NbdevLookup to use cached _build_lookup_table
Browse files Browse the repository at this point in the history
  • Loading branch information
ncoop57 committed Nov 21, 2024
1 parent ab193c6 commit e3e19e2
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 52 deletions.
1 change: 1 addition & 0 deletions nbdev/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
'nbdev.doclinks.NbdevLookup.link_line': ('api/doclinks.html#nbdevlookup.link_line', 'nbdev/doclinks.py'),
'nbdev.doclinks.NbdevLookup.linkify': ('api/doclinks.html#nbdevlookup.linkify', 'nbdev/doclinks.py'),
'nbdev.doclinks._binop_leafs': ('api/doclinks.html#_binop_leafs', 'nbdev/doclinks.py'),
'nbdev.doclinks._build_lookup_table': ('api/doclinks.html#_build_lookup_table', 'nbdev/doclinks.py'),
'nbdev.doclinks._build_modidx': ('api/doclinks.html#_build_modidx', 'nbdev/doclinks.py'),
'nbdev.doclinks._find_mod': ('api/doclinks.html#_find_mod', 'nbdev/doclinks.py'),
'nbdev.doclinks._get_exps': ('api/doclinks.html#_get_exps', 'nbdev/doclinks.py'),
Expand Down
42 changes: 23 additions & 19 deletions nbdev/doclinks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,29 +200,33 @@ def _qual_syms(entries):
# %% ../nbs/api/05_doclinks.ipynb
_re_backticks = re.compile(r'`([^`\s]+?)(?:\(\))?`')

# %% ../nbs/api/05_doclinks.ipynb
@lru_cache(None)
def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None):
cfg = get_config()
if strip_libs is None:
try: strip_libs = cfg.get('strip_libs', cfg.get('lib_path', 'nbdev').name).split()
except FileNotFoundError: strip_libs = 'nbdev'
skip_mods = setify(skip_mods)
strip_libs = L(strip_libs)
if incl_libs is not None: incl_libs = (L(incl_libs)+strip_libs).unique()
entries = {o.name: _qual_syms(o.resolve()) for o in list(pkg_resources.iter_entry_points(group='nbdev'))
if incl_libs is None or o.dist.key in incl_libs}
py_syms = merge(*L(o['syms'].values() for o in entries.values()).concat())
for m in strip_libs:
if m in entries:
_d = entries[m]
stripped = {remove_prefix(k,f"{mod}."):v
for mod,dets in _d['syms'].items() if mod not in skip_mods
for k,v in dets.items()}
py_syms = merge(stripped, py_syms)
return entries,py_syms

# %% ../nbs/api/05_doclinks.ipynb
class NbdevLookup:
"Mapping from symbol names to docs and source URLs"
def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None):
cfg = get_config()
if strip_libs is None:
try: strip_libs = cfg.get('strip_libs', cfg.get('lib_path', 'nbdev').name).split()
except FileNotFoundError: strip_libs = 'nbdev'
skip_mods = setify(skip_mods)
strip_libs = L(strip_libs)
if incl_libs is not None: incl_libs = (L(incl_libs)+strip_libs).unique()
# Dict from lib name to _nbdev module for incl_libs (defaults to all)
self.entries = {o.name: _qual_syms(o.resolve()) for o in list(pkg_resources.iter_entry_points(group='nbdev'))
if incl_libs is None or o.dist.key in incl_libs}
py_syms = merge(*L(o['syms'].values() for o in self.entries.values()).concat())
for m in strip_libs:
if m in self.entries:
_d = self.entries[m]
stripped = {remove_prefix(k,f"{mod}."):v
for mod,dets in _d['syms'].items() if mod not in skip_mods
for k,v in dets.items()}
py_syms = merge(stripped, py_syms)
self.syms = py_syms
self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods)
self.aliases = {n:o.__name__ for n,o in (ns or {}).items() if isinstance(o, ModuleType)}

def __getitem__(self, s):
Expand Down
129 changes: 96 additions & 33 deletions nbs/api/05_doclinks.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,28 @@
"def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|hide\n",
"_get_exps.cache_clear()\n",
"\n",
"# Test _get_exps caching\n",
"initial = _get_exps.cache_info()\n",
"_ = _get_exps('nbdev/maker.py') # First call should miss\n",
"after_first = _get_exps.cache_info()\n",
"_ = _get_exps('nbdev/maker.py') # Second call should hit\n",
"after_second = _get_exps.cache_info()\n",
"\n",
"test_eq(after_first.misses, initial.misses + 1)\n",
"test_eq(after_first.hits, initial.hits)\n",
"test_eq(after_second.hits, after_first.hits + 1)\n",
"test_eq(after_second.misses, after_first.misses)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -496,6 +518,35 @@
"test_eq(_re_backticks.findall('consecutive `foo``bar``baz`'), ['foo', 'bar', 'baz'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@lru_cache(None)\n",
"def _build_lookup_table(strip_libs=None, incl_libs=None, skip_mods=None):\n",
" cfg = get_config()\n",
" if strip_libs is None:\n",
" try: strip_libs = cfg.get('strip_libs', cfg.get('lib_path', 'nbdev').name).split()\n",
" except FileNotFoundError: strip_libs = 'nbdev'\n",
" skip_mods = setify(skip_mods)\n",
" strip_libs = L(strip_libs)\n",
" if incl_libs is not None: incl_libs = (L(incl_libs)+strip_libs).unique()\n",
" entries = {o.name: _qual_syms(o.resolve()) for o in list(pkg_resources.iter_entry_points(group='nbdev'))\n",
" if incl_libs is None or o.dist.key in incl_libs}\n",
" py_syms = merge(*L(o['syms'].values() for o in entries.values()).concat())\n",
" for m in strip_libs:\n",
" if m in entries:\n",
" _d = entries[m]\n",
" stripped = {remove_prefix(k,f\"{mod}.\"):v\n",
" for mod,dets in _d['syms'].items() if mod not in skip_mods\n",
" for k,v in dets.items()}\n",
" py_syms = merge(stripped, py_syms)\n",
" return entries,py_syms"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -506,25 +557,7 @@
"class NbdevLookup:\n",
" \"Mapping from symbol names to docs and source URLs\"\n",
" def __init__(self, strip_libs=None, incl_libs=None, skip_mods=None, ns=None):\n",
" cfg = get_config()\n",
" if strip_libs is None:\n",
" try: strip_libs = cfg.get('strip_libs', cfg.get('lib_path', 'nbdev').name).split()\n",
" except FileNotFoundError: strip_libs = 'nbdev'\n",
" skip_mods = setify(skip_mods)\n",
" strip_libs = L(strip_libs)\n",
" if incl_libs is not None: incl_libs = (L(incl_libs)+strip_libs).unique()\n",
" # Dict from lib name to _nbdev module for incl_libs (defaults to all)\n",
" self.entries = {o.name: _qual_syms(o.resolve()) for o in list(pkg_resources.iter_entry_points(group='nbdev'))\n",
" if incl_libs is None or o.dist.key in incl_libs}\n",
" py_syms = merge(*L(o['syms'].values() for o in self.entries.values()).concat())\n",
" for m in strip_libs:\n",
" if m in self.entries:\n",
" _d = self.entries[m]\n",
" stripped = {remove_prefix(k,f\"{mod}.\"):v\n",
" for mod,dets in _d['syms'].items() if mod not in skip_mods\n",
" for k,v in dets.items()}\n",
" py_syms = merge(stripped, py_syms)\n",
" self.syms = py_syms\n",
" self.entries,self.syms = _build_lookup_table(strip_libs, incl_libs, skip_mods)\n",
" self.aliases = {n:o.__name__ for n,o in (ns or {}).items() if isinstance(o, ModuleType)}\n",
" \n",
" def __getitem__(self, s): \n",
Expand Down Expand Up @@ -566,6 +599,37 @@
" return '\\n'.join(lines)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/nathan/miniconda3/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"#|hide\n",
"_build_lookup_table.cache_clear()\n",
"\n",
"# Test _build_lookup_table caching\n",
"initial = _build_lookup_table.cache_info()\n",
"_ = _build_lookup_table() # First call should miss\n",
"after_first = _build_lookup_table.cache_info()\n",
"_ = _build_lookup_table() # Second call should hit\n",
"after_second = _build_lookup_table.cache_info()\n",
"\n",
"test_eq(after_first.misses, initial.misses + 1)\n",
"test_eq(after_first.hits, initial.hits)\n",
"test_eq(after_second.hits, after_first.hits + 1)\n",
"test_eq(after_second.misses, after_first.misses)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -578,14 +642,6 @@
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/nathan/miniconda3/envs/main/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"data": {
"text/plain": [
Expand Down Expand Up @@ -614,7 +670,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L227){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L234){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
Expand All @@ -625,7 +681,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L227){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L234){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
Expand Down Expand Up @@ -709,7 +765,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L232){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L239){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
Expand All @@ -720,7 +776,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L232){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L239){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
Expand Down Expand Up @@ -768,7 +824,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L250){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L257){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
Expand All @@ -777,7 +833,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L250){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L257){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
Expand Down Expand Up @@ -899,6 +955,13 @@
"from nbdev._modidx import d\n",
"assert d['syms']['nbdev.doclinks']['nbdev.doclinks.NbdevLookup'][0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down

0 comments on commit e3e19e2

Please sign in to comment.