From 4f23698e5efd9d052c0282b74ee123dc5b1b7557 Mon Sep 17 00:00:00 2001 From: Josh Barnes Date: Wed, 16 Mar 2016 00:49:27 +0000 Subject: [PATCH] New installation method based on a jupyter application calling notebook 4.2-style extension API methods Also make check for running server more robust --- scripts/jupyter-contrib-nbextension | 6 + setup.py | 12 + src/jupyter_contrib_nbextensions/__init__.py | 31 ++ .../application.py | 184 +++++++++++ src/jupyter_contrib_nbextensions/install.py | 312 +++++++++--------- 5 files changed, 398 insertions(+), 147 deletions(-) create mode 100644 scripts/jupyter-contrib-nbextension create mode 100644 src/jupyter_contrib_nbextensions/application.py diff --git a/scripts/jupyter-contrib-nbextension b/scripts/jupyter-contrib-nbextension new file mode 100644 index 0000000..cd01791 --- /dev/null +++ b/scripts/jupyter-contrib-nbextension @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +from jupyter_contrib_nbextensions.application import main + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 6c79c83..05145db 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ encounter any problems. ], install_requires=[ 'ipython_genutils', + 'jupyter_contrib_core >=0.3', 'jupyter_core', 'jupyter_nbextensions_configurator', 'nbconvert', @@ -80,6 +81,17 @@ encounter any problems. # we can't be zip safe as we require templates etc to be accessible to # jupyter server zip_safe=False, + entry_points={ + 'console_scripts': [ + 'jupyter-contrib-nbextension = jupyter_contrib_nbextensions.application:main', # noqa + ], + 'jupyter_contrib_core.app.subcommands': [ + 'nbextension = jupyter_contrib_nbextensions.application:jupyter_contrib_core_app_subcommands', # noqa + ], + }, + scripts=[os.path.join('scripts', p) for p in [ + 'jupyter-contrib-nbextension', + ]], classifiers=[ 'Development Status :: 1 - Planning', 'Intended Audience :: End Users/Desktop', diff --git a/src/jupyter_contrib_nbextensions/__init__.py b/src/jupyter_contrib_nbextensions/__init__.py index 7ef538f..17ca680 100644 --- a/src/jupyter_contrib_nbextensions/__init__.py +++ b/src/jupyter_contrib_nbextensions/__init__.py @@ -1,3 +1,34 @@ # -*- coding: utf-8 -*- +import os + +import jupyter_nbextensions_configurator + __version__ = '0.0.0' + + +def _jupyter_server_extension_paths(): + """Magically-named function for jupyter extension installations.""" + return [] + + +def _jupyter_nbextension_paths(): + """Magically-named function for jupyter extension installations.""" + nbextension_dirs = [ + os.path.join(os.path.dirname(__file__), 'nbextensions')] + specs = jupyter_nbextensions_configurator.get_configurable_nbextensions( + nbextension_dirs=nbextension_dirs) + + return [dict( + section=nbext['Section'], + # src is a directory in which we assume the require file resides. + # the path is relative to the package directory + src=os.path.join( + 'nbextensions', + os.path.dirname(nbext['require']) + ), + # directory in the `nbextension/` namespace + dest=os.path.dirname(nbext['require']), + # _also_ in the `nbextension/` namespace + require=nbext['require'], + ) for nbext in specs] diff --git a/src/jupyter_contrib_nbextensions/application.py b/src/jupyter_contrib_nbextensions/application.py new file mode 100644 index 0000000..4c8648f --- /dev/null +++ b/src/jupyter_contrib_nbextensions/application.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +"""App to install/remove jupyter_contrib_nbextensions.""" + +from __future__ import print_function, unicode_literals + +import sys + +from jupyter_contrib_core.notebook_compat.nbextensions import ArgumentConflict +from jupyter_core.application import JupyterApp +from tornado.log import LogFormatter +from traitlets import Bool, Unicode, default + +import jupyter_contrib_nbextensions +from jupyter_contrib_nbextensions.install import install, uninstall + + +class BaseContribNbextensionsApp(JupyterApp): + """Base class for jupyter_contrib_nbextensions apps.""" + + version = jupyter_contrib_nbextensions.__version__ + + _log_formatter_cls = LogFormatter + + @default('log_datefmt') + def _log_datefmt_default(self): + """Exclude date from timestamp.""" + return "%H:%M:%S" + + @default('log_format') + def _log_format_default(self): + """Override default log format to include time.""" + return ('%(color)s[' + '%(levelname)1.1s %(asctime)s %(name)s' + ']%(end_color)s ' + '%(message)s') + + +class BaseContribNbextensionsInstallApp(BaseContribNbextensionsApp): + """Base jupyter_contrib_nbextensions (un)installer app.""" + + aliases = { + 'prefix': 'BaseContribNbextensionsInstallApp.prefix', + 'nbextensions': 'BaseContribNbextensionsInstallApp.nbextensions_dir', + } + flags = { + 'debug': JupyterApp.flags['debug'], + 'user': ({ + 'BaseContribNbextensionsInstallApp': {'user': True}}, + 'Perform the operation for the current user' + ), + 'system': ({ + 'BaseContribNbextensionsInstallApp': { + 'user': False, 'sys_prefix': False}}, + 'Perform the operation system-wide' + ), + 'sys-prefix': ( + {'BaseContribNbextensionsInstallApp': {'sys_prefix': True}}, + 'Use sys.prefix as the prefix for installing' + ), + # below flags apply only to nbextensions, not server extensions + 'overwrite': ( + {'BaseContribNbextensionsInstallApp': {'overwrite': True}}, + 'Force overwrite of existing nbextension files, ' + 'regardless of modification time' + ), + 'symlink': ( + {'BaseContribNbextensionsInstallApp': {'symlink': True}}, + 'Create symlinks for nbextensions instead of copying files' + ), + } + + user = Bool(True, config=True, help='Whether to do a user install') + sys_prefix = Bool(False, config=True, + help='Use the sys.prefix as the prefix') + + # settings pertaining to nbextensions installation only + overwrite = Bool(False, config=True, + help='Force overwrite of existing nbextension files') + symlink = Bool(False, config=True, + help='Create symlinks instead of copying nbextension files') + prefix = Unicode( + '', config=True, + help='Installation prefix, currently only used for nbextensions') + nbextensions_dir = Unicode( + '', config=True, + help='Full path to nbextensions dir ' + '(consider instead using sys_prefix, prefix or user)') + + def parse_command_line(self, argv=None): + """ + Overriden to check for conflicting flags + + Since notebook version doesn't do it very well + """ + conflicting_flags = set(['--user', '--system', '--sys-prefix']) + + if len(conflicting_flags.intersection(set(argv))) > 1: + raise ArgumentConflict( + 'cannot specify more than one of user, sys_prefix, or system') + return super(BaseContribNbextensionsInstallApp, + self).parse_command_line(argv) + +BaseContribNbextensionsInstallApp.flags['s'] = ( + BaseContribNbextensionsInstallApp.flags['symlink']) + + +class InstallContribNbextensionsApp(BaseContribNbextensionsInstallApp): + """Install all jupyter_contrib_nbextensions.""" + + name = 'jupyter contrib nbextension install' + description = ( + 'Install all jupyter_contrib_nbextensions.' + ) + + def start(self): + """Perform the App's actions as configured.""" + if self.extra_args: + sys.exit('{} takes no extra arguments'.format(self.name)) + self.log.info('{} {}'.format(self.name, ' '.join(self.argv))) + return install( + user=self.user, sys_prefix=self.sys_prefix, prefix=self.prefix, + nbextensions_dir=self.nbextensions_dir, logger=self.log, + overwrite=self.overwrite, symlink=self.symlink) + + +class UninstallContribNbextensionsApp(BaseContribNbextensionsInstallApp): + """Uninstall all jupyter_contrib_nbextensions.""" + + name = 'jupyter contrib nbextension uninstall' + description = ( + 'Uninstall all jupyter_contrib_nbextensions.' + ) + + def start(self): + """Perform the App's actions as configured.""" + if self.extra_args: + sys.exit('{} takes no extra arguments'.format(self.name)) + self.log.info('{} {}'.format(self.name, ' '.join(self.argv))) + return uninstall( + user=self.user, sys_prefix=self.sys_prefix, prefix=self.prefix, + nbextensions_dir=self.nbextensions_dir, logger=self.log) + + +class ContribNbextensionsApp(BaseContribNbextensionsApp): + """Main jupyter_contrib_nbextensions application.""" + + name = 'jupyter contrib nbextension' + description = ( + 'Install or uninstall all of jupyter_contrib_nbextensions.' + ) + examples = '\n'.join([ + 'jupyter contrib nbextension install # {}'.format(install.__doc__), + 'jupyter contrib nbextension uninstall # {}'.format(uninstall.__doc__) + ]) + subcommands = dict( + install=(InstallContribNbextensionsApp, install.__doc__), + uninstall=(UninstallContribNbextensionsApp, uninstall.__doc__), + ) + + def start(self): + """Perform the App's functions as configured.""" + super(ContribNbextensionsApp, self).start() + + # The above should have called a subcommand and raised NoStart; if we + # get here, it didn't, so we should self.log.info a message. + subcmds = ', '.join(sorted(self.subcommands)) + sys.exit('Please supply at least one subcommand: {}'.format(subcmds)) + + +def jupyter_contrib_core_app_subcommands(): + """Return dict of subcommands for use by jupyter_contrib_core.""" + return dict( + nbextension=(ContribNbextensionsApp, + ContribNbextensionsApp.description) + ) + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- + +main = ContribNbextensionsApp.launch_instance + +if __name__ == '__main__': + main() diff --git a/src/jupyter_contrib_nbextensions/install.py b/src/jupyter_contrib_nbextensions/install.py index d386764..e21c7b2 100644 --- a/src/jupyter_contrib_nbextensions/install.py +++ b/src/jupyter_contrib_nbextensions/install.py @@ -1,169 +1,187 @@ # -*- coding: utf-8 -*- -# Install notebook extensions +"""API to install/remove all jupyter_contrib_nbextensions.""" -from __future__ import print_function -from jupyter_core.paths import jupyter_config_dir, jupyter_data_dir -from notebook import version_info -from jupyter_nbextensions_configurator.application import main as jnc_app_main -from traitlets.config.loader import Config, JSONFileConfigLoader +from __future__ import ( + absolute_import, division, print_function, unicode_literals, +) + +import errno import os -import sys -import json + import psutil +from jupyter_contrib_core.notebook_compat import nbextensions, serverextensions +from traitlets.config import Config +from traitlets.config.manager import BaseJSONConfigManager -marker = '#--- nbextensions configuration ---' -debug = False - -def remove_old_config(configdata): - """ Remove old configuration entries - - :param configdata: python configuration data - """ - marker_found = [] - lines = configdata.splitlines() - for i, l in enumerate(lines): - if l.find(marker) >= 0: - marker_found.append(i) - start = marker_found[0] - end = marker_found[-1] - return '\n'.join(lines[0:start] + lines[end+1:]) +import jupyter_contrib_nbextensions.nbconvert_support -def make_backup(filename): - import shutil - backup = filename + ".bak" - if os.path.exists(filename): - shutil.copy(filename,backup) +class NotebookRunningError(Exception): + pass -def update_config(config_file): - """ Update .py configuration file with new path to extensions +def notebook_is_running(): + """Return true if a notebook process appears to be running.""" + for p in psutil.process_iter(): + # p.name() can throw exceptions due to zombie processes on Mac OS X, so + # ignore psutil.ZombieProcess + # (See https://code.google.com/p/psutil/issues/detail?id=428) - :param config_file: name of the config file to be updated - """ - if debug is True: - print("Configuring %s" % config_file) - make_backup(config_file) + # It isn't enough to search just the process name, we have to + # search the process command to see if jupyter-notebook is running. - new_config = "import sys\nsys.path.append({0!r})".format(os.path.join(data_dir, 'extensions')) - # add config - with open(config_file, 'a+') as f: - f.seek(0) - pyconfig = f.read() - - if pyconfig.find(marker) >= 0: - pyconfig = remove_old_config(pyconfig) - - pyconfig = marker + '\n' + new_config + '\n' + marker + '\n' + pyconfig - # write config file - with open(config_file, 'w') as f: - f.write(pyconfig) + # Checking the process command can cause an AccessDenied exception to + # be thrown for system owned processes, ignore those as well + try: + # use lower, since python may be Python, e.g. on OSX + if ('python' or 'jupyter') in p.name().lower(): + for arg in p.cmdline(): + # the missing k is deliberate! + # The usual string 'jupyter-notebook' can get truncated. + if 'jupyter-noteboo' in arg: + return True + except (psutil.ZombieProcess, psutil.AccessDenied): + pass + return False +def toggle_install(install, user=False, sys_prefix=False, overwrite=False, + symlink=False, prefix=None, nbextensions_dir=None, + logger=None): + """Install or remove all jupyter_contrib_nbextensions.""" + if notebook_is_running(): + raise NotebookRunningError( + 'Cannot configure while the Jupyter notebook server is running') -for p in psutil.process_iter(): - # p.name() can crash due to zombie processes on Mac OS X, so - # ignore exceptions due to zombie processes. - # (See https://code.google.com/p/psutil/issues/detail?id=428) - # Also, searching just the process name for string, "jupyter-notebook" may - # not be enough - may have to search the process command to see if - # jupyter-notebook is running. Checking the process command can cause an - # AccessDenied exception to be thrown for system owned processes, so skip - # those as well - try: - if ("python" or "jupyter") in p.name(): - c = p.cmdline() - if len(c) == 2 and "jupyter-notebook" in c[1]: - print("Cannot configure while the Jupyter notebook server is running") - exit(1) - # Ignore errors caused by zombie processes. Also ignore access - # denied exceptions that are thrown when checking the process - # comand of processes that do not belong to the user - except (psutil.ZombieProcess, psutil.AccessDenied): - pass + user = False if sys_prefix else user + config_dir = nbextensions._get_config_dir(user=user, sys_prefix=sys_prefix) -if len(sys.argv) == 2 and sys.argv[1] == "debug": - debug = True + verb = 'Installing' if install else 'Uninstalling' + if logger: + logger.info( + '{} jupyter_contrib_nbextensions, using config in {}'.format( + verb, config_dir)) -print("Configuring the Jupyter notebook extensions.") + # Configure the jupyter_nbextensions_configurator serverextension to load + if install: + serverextensions.toggle_serverextension_python( + 'jupyter_nbextensions_configurator', + enabled=True, user=user, sys_prefix=sys_prefix, logger=logger) -# Get the local configuration file path -# Use $PREFIX for Anaconda -data_dir = os.getenv('PREFIX', None) -if data_dir is None: - data_dir = jupyter_data_dir() -else: - data_dir = os.path.join(data_dir, 'share/jupyter') - -if debug is True: - print("Extensions and templates path: %s" % data_dir) - -config_dir = jupyter_config_dir() -print("Configuration files directory: %s" % config_dir) -if os.path.exists(config_dir) is False: - os.mkdir(config_dir) - if debug is True: - print("Creating directory %s" % config_dir) - - - -def load_json_config(json_filename): - """ Load config as JSON file - :param json_filename: Filename of JSON file - :return: Traitlets based configuration - """ - json_config = os.path.join(jupyter_config_dir(), json_filename) - if debug is True: print("Configuring %s" % json_config) - if os.path.isfile(json_config) is True: - cl = JSONFileConfigLoader(json_config) - config = cl.load_config() + # nbextensions: + kwargs = dict(user=user, sys_prefix=sys_prefix, prefix=prefix, + nbextensions_dir=nbextensions_dir, logger=logger) + if install: + nbextensions.install_nbextension_python( + jupyter_contrib_nbextensions.__name__, + overwrite=overwrite, symlink=symlink, **kwargs) else: - config = Config() - return config + nbextensions.uninstall_nbextension_python( + jupyter_contrib_nbextensions.__name__, **kwargs) + + # Set extra template path, pre- and post-processors for nbconvert + cm = BaseJSONConfigManager(config_dir=config_dir) + config_basename = 'jupyter_nbconvert_config' + config = cm.get(config_basename) + # avoid warnings about unset version + config.setdefault('version', 1) + if logger: + logger.info( + u'- Editing config: {}'.format(cm.file_name(config_basename))) + + # Set extra template path, pre- and post-processors for nbconvert + if logger: + logger.info('-- Configuring nbconvert template path') + # our templates directory + _update_config_list(config, 'Exporter.template_path', [ + '.', + jupyter_contrib_nbextensions.nbconvert_support.templates_directory(), + ], install) + # our preprocessors + if logger: + logger.info('-- Configuring nbconvert preprocessors') + proc_mod = 'jupyter_contrib_nbextensions.nbconvert_support' + _update_config_list(config, 'Exporter.preprocessors', [ + proc_mod + '.CodeFoldingPreprocessor', + proc_mod + '.PyMarkdownPreprocessor', + ], install) + # our postprocessor class + if logger: + logger.info('-- Configuring nbconvert postprocessor_class') + if install: + config.setdefault( + 'NbConvertApp', Config())['postprocessor_class'] = ( + proc_mod + '.EmbedPostProcessor') + else: + nbconvert_conf = config.get('NbConvertApp', Config()) + if (nbconvert_conf.get('postprocessor_class') == + proc_mod + '.EmbedPostProcessor'): + nbconvert_conf.pop('postprocessor_class') + if len(nbconvert_conf) < 1: + config.pop('NbConvertApp') + if logger: + logger.info( + u'- Writing config: {}'.format(cm.file_name(config_basename))) + _set_managed_config(cm, config_basename, config, logger=logger) -def save_json_config(json_file, newconfig): - """ Save config as JSON file - :param json_file: Filename of JSON file - :param newconfig: New traitlets based configuration +def install(user=False, sys_prefix=False, prefix=None, nbextensions_dir=None, + logger=None, overwrite=False, symlink=False): + """Edit jupyter config files to use jupyter_contrib_nbextensions things.""" + return toggle_install( + True, user=user, sys_prefix=sys_prefix, prefix=prefix, + nbextensions_dir=nbextensions_dir, logger=logger, + overwrite=overwrite, symlink=symlink) + + +def uninstall(user=False, sys_prefix=False, prefix=None, nbextensions_dir=None, + logger=None): + """Edit jupyter config files to not use jupyter_contrib_nbextensions.""" + return toggle_install( + False, user=user, sys_prefix=sys_prefix, prefix=prefix, + nbextensions_dir=nbextensions_dir, logger=logger) + +# ----------------------------------------------------------------------------- +# Private API +# ----------------------------------------------------------------------------- + + +def _set_managed_config(cm, config_basename, config, logger=None): + """Write config owned by the given config manager, removing if empty.""" + config_path = cm.file_name(config_basename) + msg = 'config file {}'.format(config_path) + if len(config) > ('version' in config): + if logger: + logger.info('-- Writing updated {}'.format(msg)) + # use set to ensure removed keys get removed + cm.set(config_basename, config) + else: + if logger: + logger.info('-- Removing now-empty {}'.format(msg)) + try: + os.remove(config_path) + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + + +def _update_config_list(config, list_key, values, insert): """ - s = json.dumps(newconfig, indent=2, separators=(',', ': '), sort_keys=True) - json_config = os.path.join(jupyter_config_dir(), json_file) - make_backup(json_config) - with open(json_config, 'w') as f: - f.write(s) + Add or remove items as required to/from a config value which is a list. -# Update nbconvert JSON configuration -json_file = 'jupyter_nbconvert_config.json' -config = load_json_config(json_file) - -# Set template path, pre- and postprocessors of notebook extensions -newconfig = Config() -newconfig.Exporter.template_path = ['.', os.path.join(data_dir, 'templates')] -newconfig.Exporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor", "pre_pymarkdown.PyMarkdownPreprocessor"] -newconfig.NbConvertApp.postprocessor_class = 'post_embedhtml.EmbedPostProcessor' -config.merge(newconfig) -config.version = 1 -save_json_config(json_file, config) - -# Update nbconvert PY configuration -py_config = os.path.join(jupyter_config_dir(), 'jupyter_nbconvert_config.py') -update_config(py_config) - -# Update notebook JSON configuration -json_file = 'jupyter_notebook_config.json' -config = load_json_config(json_file) - -# Add template path -newconfig = Config() -newconfig.NotebookApp.extra_template_paths = [os.path.join(jupyter_data_dir(),'templates') ] -config.merge(newconfig) -config.version = 1 -save_json_config(json_file, config) - -# Update notebook PY configuration -py_config = os.path.join(jupyter_config_dir(), 'jupyter_notebook_config.py') -update_config(py_config) - -# Configure the jupyter_nbextensions_configurator serverextension -jnc_app_main(['enable', '--user', '--debug']) + This exists in order to avoid clobbering values other than those which we + wish to add/remove, and to neatly remove a list when it ends up empty. + """ + section, list_key = list_key.split('.') + conf_list = config.setdefault(section, Config()).setdefault(list_key, []) + list_alteration_method = 'append' if insert else 'remove' + for val in values: + if (val in conf_list) != insert: + getattr(conf_list, list_alteration_method)(val) + if not insert: + # remove empty list + if len(conf_list) == 0: + config[section].pop(list_key) + # remove empty section + if len(config[section]) == 0: + config.pop(section)