Skip to content

Commit 90ef61a

Browse files
jph00jph00
authored
Merge pull request #1478 from AnswerDotAI/index
Migrate index creation from nbdev-index
2 parents e47b6a3 + 830d462 commit 90ef61a

File tree

4 files changed

+180
-13
lines changed

4 files changed

+180
-13
lines changed

‎nbdev/_modidx.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
'nbdev.doclinks._qual_sym': ('api/doclinks.html#_qual_sym', 'nbdev/doclinks.py'),
7171
'nbdev.doclinks._qual_syms': ('api/doclinks.html#_qual_syms', 'nbdev/doclinks.py'),
7272
'nbdev.doclinks._sym_nm': ('api/doclinks.html#_sym_nm', 'nbdev/doclinks.py'),
73+
'nbdev.doclinks.create_index': ('api/doclinks.html#create_index', 'nbdev/doclinks.py'),
7374
'nbdev.doclinks.nbdev_export': ('api/doclinks.html#nbdev_export', 'nbdev/doclinks.py'),
7475
'nbdev.doclinks.nbglob': ('api/doclinks.html#nbglob', 'nbdev/doclinks.py'),
7576
'nbdev.doclinks.nbglob_cli': ('api/doclinks.html#nbglob_cli', 'nbdev/doclinks.py'),

‎nbdev/doclinks.py

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/05_doclinks.ipynb.
44

55
# %% auto 0
6-
__all__ = ['patch_name', 'nbglob', 'nbglob_cli', 'nbdev_export', 'NbdevLookup']
6+
__all__ = ['typs', 'bset', 'patch_name', 'nbglob', 'nbglob_cli', 'nbdev_export', 'create_index', 'NbdevLookup']
77

88
# %% ../nbs/api/05_doclinks.ipynb
99
from .config import *
@@ -14,11 +14,14 @@
1414
from fastcore.script import *
1515
from fastcore.utils import *
1616
from fastcore.meta import delegates
17+
from fastcore.net import urlread
1718

18-
import ast,contextlib
19+
import ast,builtins,contextlib
1920
import pkg_resources,importlib
20-
from astunparse import unparse
2121

22+
from astunparse import unparse
23+
from io import BytesIO
24+
from collections import defaultdict
2225
from pprint import pformat
2326
from urllib.parse import urljoin
2427
from functools import lru_cache
@@ -153,6 +156,28 @@ def nbdev_export(
153156
add_init(get_config().lib_path)
154157
_build_modidx()
155158

159+
# %% ../nbs/api/05_doclinks.ipynb
160+
typs = 'module','class','method','function'
161+
bset = set(dir(builtins))
162+
163+
# %% ../nbs/api/05_doclinks.ipynb
164+
def create_index(url, pre=None):
165+
"Create a documentation index from a sphinx inventory file at `url`, with optional prefix `pre`"
166+
try: from sphinx.util.inventory import InventoryFile
167+
except ImportError: raise ImportError('`sphinx` is a dependency for building indexes. Run `pip install sphinx` to use `create_index`.')
168+
pre = ifnone(pre, f"{url}/")
169+
invs = urlread(f'{url}/objects.inv', decode=False)
170+
idx = InventoryFile.load(stream=BytesIO(invs), uri=pre, joinfunc=urljoin)
171+
_get = lambda o: {k:v[2] for k,v in idx[f'py:{o}'].items() if k[0]!='_'}
172+
d = {o:_get(o) for o in typs}
173+
syms = defaultdict(dict)
174+
for o in typs:
175+
for k,v in d[o].items():
176+
if k.split('.')[0] in bset: k = 'builtins.' + k
177+
modparts = k.split(".")[:-2 if o=='method' else -1]
178+
if modparts: syms['.'.join(modparts)][k] = v
179+
return syms
180+
156181
# %% ../nbs/api/05_doclinks.ipynb
157182
import importlib,ast
158183
from functools import lru_cache

‎nbs/api/05_doclinks.ipynb

Lines changed: 150 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,14 @@
3333
"from fastcore.script import *\n",
3434
"from fastcore.utils import *\n",
3535
"from fastcore.meta import delegates\n",
36+
"from fastcore.net import urlread\n",
3637
"\n",
37-
"import ast,contextlib\n",
38+
"import ast,builtins,contextlib\n",
3839
"import pkg_resources,importlib\n",
39-
"from astunparse import unparse\n",
4040
"\n",
41+
"from astunparse import unparse\n",
42+
"from io import BytesIO\n",
43+
"from collections import defaultdict\n",
4144
"from pprint import pformat\n",
4245
"from urllib.parse import urljoin\n",
4346
"from functools import lru_cache\n",
@@ -355,7 +358,6 @@
355358
"outputs": [],
356359
"source": [
357360
"#|export\n",
358-
"\n",
359361
"@call_parse\n",
360362
"@delegates(nbglob_cli)\n",
361363
"def nbdev_export(\n",
@@ -383,6 +385,101 @@
383385
"N.B.: the `black_format` processor is passed in by default. But it is a no-op, unless `black_formatting=True` is set in your `settings.ini` configuration. You can omit it from `nbdev_export` on the command line by passing in `--procs`."
384386
]
385387
},
388+
{
389+
"cell_type": "markdown",
390+
"metadata": {},
391+
"source": [
392+
"## Construct Index"
393+
]
394+
},
395+
{
396+
"cell_type": "code",
397+
"execution_count": null,
398+
"metadata": {},
399+
"outputs": [],
400+
"source": [
401+
"#|export\n",
402+
"typs = 'module','class','method','function'\n",
403+
"bset = set(dir(builtins))"
404+
]
405+
},
406+
{
407+
"cell_type": "code",
408+
"execution_count": null,
409+
"metadata": {},
410+
"outputs": [],
411+
"source": [
412+
"#|export\n",
413+
"def create_index(url, pre=None):\n",
414+
" \"Create a documentation index from a sphinx inventory file at `url`, with optional prefix `pre`\"\n",
415+
" try: from sphinx.util.inventory import InventoryFile\n",
416+
" except ImportError: raise ImportError('`sphinx` is a dependency for building indexes. Run `pip install sphinx` to use `create_index`.')\n",
417+
" pre = ifnone(pre, f\"{url}/\")\n",
418+
" invs = urlread(f'{url}/objects.inv', decode=False)\n",
419+
" idx = InventoryFile.load(stream=BytesIO(invs), uri=pre, joinfunc=urljoin)\n",
420+
" _get = lambda o: {k:v[2] for k,v in idx[f'py:{o}'].items() if k[0]!='_'}\n",
421+
" d = {o:_get(o) for o in typs}\n",
422+
" syms = defaultdict(dict)\n",
423+
" for o in typs:\n",
424+
" for k,v in d[o].items():\n",
425+
" if k.split('.')[0] in bset: k = 'builtins.' + k\n",
426+
" modparts = k.split(\".\")[:-2 if o=='method' else -1]\n",
427+
" if modparts: syms['.'.join(modparts)][k] = v\n",
428+
" return syms"
429+
]
430+
},
431+
{
432+
"cell_type": "code",
433+
"execution_count": null,
434+
"metadata": {},
435+
"outputs": [],
436+
"source": [
437+
"url = 'https://docs.python.org/3'\n",
438+
"syms = create_index(url)"
439+
]
440+
},
441+
{
442+
"cell_type": "code",
443+
"execution_count": null,
444+
"metadata": {},
445+
"outputs": [
446+
{
447+
"data": {
448+
"text/plain": [
449+
"{'builtins.bool': 'https://docs.python.org/3/library/functions.html#bool',\n",
450+
" 'builtins.bytearray': 'https://docs.python.org/3/library/stdtypes.html#bytearray',\n",
451+
" 'builtins.bytes': 'https://docs.python.org/3/library/stdtypes.html#bytes',\n",
452+
" 'builtins.complex': 'https://docs.python.org/3/library/functions.html#complex',\n",
453+
" 'builtins.dict': 'https://docs.python.org/3/library/stdtypes.html#dict',\n",
454+
" 'builtins.float': 'https://docs.python.org/3/library/functions.html#float',\n",
455+
" 'builtins.frozenset': 'https://docs.python.org/3/library/stdtypes.html#frozenset',\n",
456+
" 'builtins.int': 'https://docs.python.org/3/library/functions.html#int',\n",
457+
" 'builtins.list': 'https://docs.python.org/3/library/stdtypes.html#list',\n",
458+
" 'builtins.memoryview': 'https://docs.python.org/3/library/stdtypes.html#memoryview'}"
459+
]
460+
},
461+
"execution_count": null,
462+
"metadata": {},
463+
"output_type": "execute_result"
464+
}
465+
],
466+
"source": [
467+
"dict(list(syms['builtins'].items())[:10])"
468+
]
469+
},
470+
{
471+
"cell_type": "code",
472+
"execution_count": null,
473+
"metadata": {},
474+
"outputs": [],
475+
"source": [
476+
"for b in syms['builtins']:\n",
477+
" b = b.split('.')\n",
478+
" if len(b) != 2: continue\n",
479+
" b = b[1]\n",
480+
" assert b in bset"
481+
]
482+
},
386483
{
387484
"cell_type": "markdown",
388485
"metadata": {},
@@ -721,7 +818,7 @@
721818
"text/markdown": [
722819
"---\n",
723820
"\n",
724-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L241){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
821+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L266){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
725822
"\n",
726823
"### NbdevLookup.doc\n",
727824
"\n",
@@ -732,7 +829,7 @@
732829
"text/plain": [
733830
"---\n",
734831
"\n",
735-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L241){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
832+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L266){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
736833
"\n",
737834
"### NbdevLookup.doc\n",
738835
"\n",
@@ -823,7 +920,7 @@
823920
"text/markdown": [
824921
"---\n",
825922
"\n",
826-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L246){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
923+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L271){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
827924
"\n",
828925
"### NbdevLookup.code\n",
829926
"\n",
@@ -834,7 +931,7 @@
834931
"text/plain": [
835932
"---\n",
836933
"\n",
837-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L246){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
934+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L271){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
838935
"\n",
839936
"### NbdevLookup.code\n",
840937
"\n",
@@ -882,7 +979,7 @@
882979
"text/markdown": [
883980
"---\n",
884981
"\n",
885-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L264){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
982+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L289){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
886983
"\n",
887984
"### NbdevLookup.linkify\n",
888985
"\n",
@@ -891,7 +988,7 @@
891988
"text/plain": [
892989
"---\n",
893990
"\n",
894-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L264){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
991+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L289){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
895992
"\n",
896993
"### NbdevLookup.linkify\n",
897994
"\n",
@@ -962,6 +1059,50 @@
9621059
"assert NbdevLookup().linkify(md) == md"
9631060
]
9641061
},
1062+
{
1063+
"cell_type": "code",
1064+
"execution_count": null,
1065+
"metadata": {},
1066+
"outputs": [
1067+
{
1068+
"data": {
1069+
"text/plain": [
1070+
"'[`builtins.str.split`](https://docs.python.org/3/library/stdtypes.html#str.split)'"
1071+
]
1072+
},
1073+
"execution_count": null,
1074+
"metadata": {},
1075+
"output_type": "execute_result"
1076+
}
1077+
],
1078+
"source": [
1079+
"# Test builtins\n",
1080+
"md = \"`builtins.str.split`\"\n",
1081+
"NbdevLookup().linkify(md)"
1082+
]
1083+
},
1084+
{
1085+
"cell_type": "code",
1086+
"execution_count": null,
1087+
"metadata": {},
1088+
"outputs": [
1089+
{
1090+
"data": {
1091+
"text/plain": [
1092+
"'[`str.split`](https://docs.python.org/3/library/stdtypes.html#str.split) and [`str`](https://docs.python.org/3/library/locale.html#locale.str)'"
1093+
]
1094+
},
1095+
"execution_count": null,
1096+
"metadata": {},
1097+
"output_type": "execute_result"
1098+
}
1099+
],
1100+
"source": [
1101+
"# ... now with stripping\n",
1102+
"md = \"`str.split` and `str`\"\n",
1103+
"NbdevLookup('nbdev_stdlib').linkify(md)"
1104+
]
1105+
},
9651106
{
9661107
"cell_type": "markdown",
9671108
"metadata": {},

‎settings.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ requirements = fastcore>=1.5.27 execnb>=0.1.4 astunparse ghapi>=1.0.3 watchdog a
1919
pip_requirements = PyYAML
2020
conda_requirements = pyyaml
2121
conda_user = fastai
22-
dev_requirements = ipywidgets nbdev-numpy nbdev-stdlib pandas matplotlib black svg.py nbclassic pysymbol_llm llms-txt
22+
dev_requirements = ipywidgets nbdev-numpy nbdev-stdlib pandas matplotlib black svg.py nbclassic pysymbol_llm llms-txt sphinx
2323
console_scripts = nbdev_create_config=nbdev.config:nbdev_create_config
2424
nbdev_update=nbdev.sync:nbdev_update
2525
nbdev_update_license=nbdev.cli:nbdev_update_license

0 commit comments

Comments
 (0)