Skip to content

exported modules can scrub_magics #1250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 17, 2023
Merged

exported modules can scrub_magics #1250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9366c7f
exported modules can scrub_magics via settings.ini
yegeniy Dec 16, 2022
f3af3fe
Merge remote-tracking branch 'origin/master' into scrub-magics-squash
Feb 10, 2023
3f7dea8
Merge remote-tracking branch 'origin/master' into scrub-magics
yegeniy Feb 16, 2023
ca1d792
Allow scrubbing magics via `nbdev_export --procs`
yegeniy Feb 16, 2023
0f4d622
--procs takes * nargs using optional_procs choices
yegeniy Feb 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion nbdev/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@
'nbdev.export.ExportModuleProc._exporti_': ('api/export.html#exportmoduleproc._exporti_', 'nbdev/export.py'),
'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py')},
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
'nbdev.export.optional_procs': ('api/export.html#optional_procs', 'nbdev/export.py'),
'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
'nbdev.extract_attachments': {},
'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),
'nbdev.frontmatter.FrontmatterProc._update': ( 'api/frontmatter.html#frontmatterproc._update',
Expand Down
16 changes: 10 additions & 6 deletions nbdev/doclinks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,19 +131,23 @@ def nbglob_cli(
@delegates(nbglob_cli)
def nbdev_export(
path:str=None, # Path or filename
procs:Param("tokens naming the export processors to use.", nargs="*", choices=optional_procs())="black_format",
**kwargs):
"Export notebooks in `path` to Python modules"
if os.environ.get('IN_TEST',0): return
if procs:
import nbdev.export
procs = [getattr(nbdev.export, p) for p in L(procs)]
files = nbglob(path=path, as_path=True, **kwargs).sorted('name')
for f in files: nb_export(f)
for f in files: nb_export(f, procs=procs)
add_init(get_config().lib_path)
_build_modidx()

# %% ../nbs/api/05_doclinks.ipynb 24
# %% ../nbs/api/05_doclinks.ipynb 25
import importlib,ast
from functools import lru_cache

# %% ../nbs/api/05_doclinks.ipynb 25
# %% ../nbs/api/05_doclinks.ipynb 26
def _find_mod(mod):
mp,_,mr = mod.partition('/')
spec = importlib.util.find_spec(mp)
Expand All @@ -166,7 +170,7 @@ def _get_exps(mod):

def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None

# %% ../nbs/api/05_doclinks.ipynb 27
# %% ../nbs/api/05_doclinks.ipynb 28
def _qual_sym(s, settings):
if not isinstance(s,tuple): return s
nb,py = s
Expand All @@ -181,10 +185,10 @@ def _qual_syms(entries):
if 'doc_host' not in settings: return entries
return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}

# %% ../nbs/api/05_doclinks.ipynb 28
# %% ../nbs/api/05_doclinks.ipynb 29
_re_backticks = re.compile(r'`([^`\s]+)`')

# %% ../nbs/api/05_doclinks.ipynb 29
# %% ../nbs/api/05_doclinks.ipynb 30
@lru_cache(None)
class NbdevLookup:
"Mapping from symbol names to docs and source URLs"
Expand Down
27 changes: 23 additions & 4 deletions nbdev/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/04_export.ipynb.

# %% auto 0
__all__ = ['ExportModuleProc', 'black_format', 'nb_export']
__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'optional_procs', 'nb_export']

# %% ../nbs/api/04_export.ipynb 2
from .config import *
Expand All @@ -26,7 +26,7 @@ def _export_(self, cell, exp_to=None):
self.in_all[ifnone(exp_to, '#')].append(cell)
_exports_=_export_

# %% ../nbs/api/04_export.ipynb 7
# %% ../nbs/api/04_export.ipynb 8
def black_format(cell, # Cell to format
force=False): # Turn black formatting on regardless of settings.ini
"Processor to format code with `black`"
Expand All @@ -40,8 +40,27 @@ def black_format(cell, # Cell to format
try: cell.source = _format_str(cell.source).strip()
except: pass

# %% ../nbs/api/04_export.ipynb 9
def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):
# %% ../nbs/api/04_export.ipynb 10
# includes the newline, because calling .strip() would affect all cells.
_magics_pattern = re.compile(r'^\s*(%%|%).*\n?', re.MULTILINE)
Comment on lines +44 to +45
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The magic pattern in clean_magics (in the processors notebook) is a little different. It doesn't include the newline and relies on calling .strip(). That's a much easier to follow, but seems to impact the whole cell instead of just the spark line.

When I used r'^\s*(%%|%).*' to test nbdev_export --procs scrub_magics, an unrelated notebook (with no magics!) was stripped of their trailing whitespace. Probably not a big deal, but I wanted to minimize the disruption here.


def scrub_magics(cell): # Cell to format
"Processor to remove cell magics from exported code"
try: cfg = get_config()
except FileNotFoundError: return
if cell.cell_type != 'code': return
try: cell.source = _magics_pattern.sub('', cell.source)
except: pass

# %% ../nbs/api/04_export.ipynb 13
import nbdev.export
def optional_procs():
"An explicit list of processors that could be used by `nb_export`"
return L([p for p in nbdev.export.__all__
if p not in ["nb_export", "ExportModuleProc", "optional_procs"]])

# %% ../nbs/api/04_export.ipynb 16
def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that I changed procs=black_format to procs=None. Refactoring that out of here seemed prudent. The code didn't seem to be used anywhere else, black_format is still passed in by default in nbdev_export, and tests passed locally. So I don't think it's a breaking change. Highlighting it, just in case though.

"Create module(s) from notebook"
if lib_path is None: lib_path = get_config().lib_path
exp = ExportModuleProc()
Expand Down
103 changes: 101 additions & 2 deletions nbs/api/04_export.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,13 @@
"assert 'h_n' in exp.in_all['some.thing'][0].source"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Optional export processors"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -122,7 +129,7 @@
"metadata": {},
"outputs": [],
"source": [
"_cell = read_nb('../../tests/black.ipynb')['cells'][0]\n",
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][0]\n",
"black_format(_cell, force=True)\n",
"test_eq(_cell.source, 'j = [1, 2, 3]')"
]
Expand All @@ -134,7 +141,99 @@
"outputs": [],
"source": [
"#|export\n",
"def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):\n",
"# includes the newline, because calling .strip() would affect all cells.\n",
"_magics_pattern = re.compile(r'^\\s*(%%|%).*\\n?', re.MULTILINE)\n",
"\n",
"def scrub_magics(cell): # Cell to format\n",
" \"Processor to remove cell magics from exported code\"\n",
" try: cfg = get_config()\n",
" except FileNotFoundError: return\n",
" if cell.cell_type != 'code': return\n",
" try: cell.source = _magics_pattern.sub('', cell.source)\n",
" except: pass"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`scrub_magics` is a processor that scrubs the jupyter \"magics\" lines out of exported cells. This can be helpful when using tools like [sparkmagic](https://github.com/jupyter-incubator/sparkmagic) or just [Jupyter's builtin magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) in an nbdev project.\n",
"\n",
"Usage: \n",
"This behavior can be enabled by passing `scrub_magics` into the `--procs` flag of the `nbdev_export` command.\n",
"- `nbdev_export --procs scrub_magics`\n",
"- `nbdev_export --procs 'scrub_magics black_format'`\n",
"\n",
"Example:\n",
"\n",
"A cell like below could export the line `\"hello nbdev\"` into the `bar` module. And the `%%spark` magic line would be omitted.\n",
"\n",
"```python\n",
"%%spark\n",
"#|export bar\n",
"\"hello nbdev\"\n",
"```\n",
"\n",
"It will export as something similar to this:\n",
"\n",
"```python\n",
"# %% ../path/to/01_bar.ipynb 1\n",
"\"hello nbdev\"\n",
"```\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][2]\n",
"scrub_magics(_cell)\n",
"test_eq(_cell.source, '''#|export bar\n",
"\"hello nbdev\"''')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"import nbdev.export\n",
"def optional_procs():\n",
" \"An explicit list of processors that could be used by `nb_export`\"\n",
" return L([p for p in nbdev.export.__all__\n",
" if p not in [\"nb_export\", \"ExportModuleProc\", \"optional_procs\"]])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# every optional processor should be explicitly listed here\n",
"test_eq(optional_procs(), ['black_format', 'scrub_magics'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### `nb_export`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):\n",
" \"Create module(s) from notebook\"\n",
" if lib_path is None: lib_path = get_config().lib_path\n",
" exp = ExportModuleProc()\n",
Expand Down
28 changes: 21 additions & 7 deletions nbs/api/05_doclinks.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,19 +347,33 @@
"outputs": [],
"source": [
"#|export\n",
"\n",
"@call_parse\n",
"@delegates(nbglob_cli)\n",
"def nbdev_export(\n",
" path:str=None, # Path or filename\n",
" procs:Param(\"tokens naming the export processors to use.\", nargs=\"*\", choices=optional_procs())=\"black_format\",\n",
" **kwargs):\n",
" \"Export notebooks in `path` to Python modules\"\n",
" if os.environ.get('IN_TEST',0): return\n",
" if procs:\n",
" import nbdev.export\n",
" procs = [getattr(nbdev.export, p) for p in L(procs)]\n",
" files = nbglob(path=path, as_path=True, **kwargs).sorted('name')\n",
" for f in files: nb_export(f)\n",
" for f in files: nb_export(f, procs=procs)\n",
" add_init(get_config().lib_path)\n",
" _build_modidx()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`procs` names the optional processors you wish to run on the exported cells of your notebook.\n",
"\n",
"N.B.: the `black_format` processor is passed in by default. But it is a no-op, unless `black_formatting=True` is set in your `settings.ini` configuration. You can omit it from `nbdev_export` on the command line by passing in `--procs`."
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -556,7 +570,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
Expand All @@ -567,7 +581,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
Expand Down Expand Up @@ -650,7 +664,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
Expand All @@ -661,7 +675,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
Expand Down Expand Up @@ -709,7 +723,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
Expand All @@ -718,7 +732,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
Expand Down
20 changes: 20 additions & 0 deletions tests/black.ipynb → tests/export_procs.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,26 @@
" 3\n",
"]"
]
},
{
"cell_type": "markdown",
"id": "40855489-6543-4f63-81c4-1127f4e09c31",
"metadata": {},
"source": [
"to test `scrub_magics`:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "22aa79e5-80ba-4b11-a655-ad8d830fa2ef",
"metadata": {},
"outputs": [],
"source": [
"%%spark\n",
"#|export bar\n",
"\"hello nbdev\""
]
}
],
"metadata": {
Expand Down