Skip to content

Commit 9366c7f

Browse files
committed
exported modules can scrub_magics via settings.ini
This scrubs the cell magic lines out in the generated code. In some sense, it is a follow-up to #905. Would y'all be interested in a PR that adds a new option to the settings.ini config? It would scrub any `%%magics` from the exported modules. Fairly similar to how `black_formatting` works today. --- The specific use-case is to simplify doing exploratory programming in PySpark. We can now use nbdev notebooks to generate valid [pyspark][0] code modules. At the same time, the Python-kernel notebook can continue to use the [`%%spark`][1] magic. We can use a local notebook to interact with a remote cluster running Livy. [0]: https://github.com/apache/spark/tree/v3.1.1-rc3/examples/src/main/python [1]: https://github.com/jupyter-incubator/sparkmagic --- Please note that the implementation is just copying `processors.clean_magics`. Not sure how much of an issue that is for this repository. Should we add a code cell like below above the scrub_magics method? ``` \#|hide \# FIXME: reusing processors.clean_magics runs into a circular import \# from .processors import clean_magics ```
1 parent 46d05d4 commit 9366c7f

File tree

5 files changed

+38
-4
lines changed

5 files changed

+38
-4
lines changed

nbdev/_modidx.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@
7676
'nbdev.export.ExportModuleProc._exporti_': ('api/export.html#exportmoduleproc._exporti_', 'nbdev/export.py'),
7777
'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
7878
'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
79-
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py')},
79+
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
80+
'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
8081
'nbdev.extract_attachments': {},
8182
'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),
8283
'nbdev.frontmatter.FrontmatterProc._update': ( 'api/frontmatter.html#frontmatterproc._update',

nbdev/config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ def _apply_defaults(
5656
language='English', # Language PyPI classifier
5757
recursive:bool_arg=True, # Include subfolders in notebook globs?
5858
black_formatting:bool_arg=False, # Format libraries with black?
59+
scrub_magics:bool_arg=False, # Remove magics from exported modules?
5960
readme_nb='index.ipynb', # Notebook to export as repo readme
6061
title='%(lib_name)s', # Quarto website title
6162
allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata

nbdev/export.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/export.ipynb.
22

33
# %% auto 0
4-
__all__ = ['ExportModuleProc', 'black_format', 'nb_export']
4+
__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'nb_export']
55

66
# %% ../nbs/api/export.ipynb 2
77
from .config import *
@@ -41,7 +41,19 @@ def black_format(cell, # Cell to format
4141
except: pass
4242

4343
# %% ../nbs/api/export.ipynb 9
44-
def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):
44+
_magics_pattern = re.compile(r'^\s*(%%|%).*', re.MULTILINE)
45+
46+
def scrub_magics(cell): # Cell to format
47+
"Processor to remove cell magics from exported code"
48+
try: cfg = get_config()
49+
except FileNotFoundError: return
50+
if (not cfg.scrub_magics) or cell.cell_type != 'code': return
51+
try:
52+
if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()
53+
except: pass
54+
55+
# %% ../nbs/api/export.ipynb 10
56+
def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):
4557
"Create module(s) from notebook"
4658
if lib_path is None: lib_path = get_config().lib_path
4759
exp = ExportModuleProc()

nbs/api/config.ipynb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@
151151
" language='English', # Language PyPI classifier\n",
152152
" recursive:bool_arg=True, # Include subfolders in notebook globs?\n",
153153
" black_formatting:bool_arg=False, # Format libraries with black?\n",
154+
" scrub_magics:bool_arg=False, # Remove magics from exported modules?\n",
154155
" readme_nb='index.ipynb', # Notebook to export as repo readme\n",
155156
" title='%(lib_name)s', # Quarto website title\n",
156157
" allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata\n",

nbs/api/export.ipynb

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,26 @@
134134
"outputs": [],
135135
"source": [
136136
"#|export\n",
137-
"def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):\n",
137+
"_magics_pattern = re.compile(r'^\\s*(%%|%).*', re.MULTILINE)\n",
138+
"\n",
139+
"def scrub_magics(cell): # Cell to format\n",
140+
" \"Processor to remove cell magics from exported code\"\n",
141+
" try: cfg = get_config()\n",
142+
" except FileNotFoundError: return\n",
143+
" if (not cfg.scrub_magics) or cell.cell_type != 'code': return\n",
144+
" try:\n",
145+
" if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()\n",
146+
" except: pass"
147+
]
148+
},
149+
{
150+
"cell_type": "code",
151+
"execution_count": null,
152+
"metadata": {},
153+
"outputs": [],
154+
"source": [
155+
"#|export\n",
156+
"def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):\n",
138157
" \"Create module(s) from notebook\"\n",
139158
" if lib_path is None: lib_path = get_config().lib_path\n",
140159
" exp = ExportModuleProc()\n",

0 commit comments

Comments
 (0)