From 5b3aa56c0f2c27b898ac6f06a516f16b625afa86 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 16 Mar 2014 22:48:01 +0530 Subject: [PATCH 01/36] Adds functionality to generate ipython notebooks for the gallery examples --- doc/ext/plot2rst.py | 104 +++++++++++++++++- .../applications/notebook/sample.ipynb | 24 ++++ .../auto_examples/notebook/sample.ipynb | 24 ++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 doc/source/auto_examples/applications/notebook/sample.ipynb create mode 100644 doc/source/auto_examples/notebook/sample.ipynb diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 3b26aeb9..d1abc20a 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -70,6 +70,8 @@ import shutil import token import tokenize import traceback +import json +import copy import numpy as np import matplotlib @@ -94,6 +96,13 @@ CODE_LINK = """ """ +NOTEBOOK_LINK = """ + +**ipython Notebook source code:** :download:`download <{0}>` +(generated using ``skimage`` |version|) + +""" + TOCTREE_TEMPLATE = """ .. toctree:: :hidden: @@ -305,16 +314,19 @@ def write_example(src_name, src_dir, rst_dir, cfg): image_dir = rst_dir.pjoin('images') thumb_dir = image_dir.pjoin('thumb') + notebook_dir = rst_dir.pjoin('notebook') image_dir.makedirs() thumb_dir.makedirs() + notebook_dir.makedirs() base_image_name = os.path.splitext(src_name)[0] image_path = image_dir.pjoin(base_image_name + '_{0}.png') basename, py_ext = os.path.splitext(src_name) rst_path = rst_dir.pjoin(basename + cfg.source_suffix) + notebook_path = notebook_dir.pjoin(basename + '.ipynb') - if _plots_are_current(src_path, image_path) and rst_path.exists: + if _plots_are_current(src_path, image_path) and rst_path.exists and notebook_path.exists: return blocks = split_code_and_text_blocks(example_file) @@ -341,6 +353,9 @@ def write_example(src_name, src_dir, rst_dir, cfg): example_rst += LITERALINCLUDE.format(**code_info) example_rst += CODE_LINK.format(src_name) + ipnotebook_name = src_name.replace('.py', '.ipynb') + ipnotebook_name = './notebook/' + ipnotebook_name + example_rst += NOTEBOOK_LINK.format(ipnotebook_name) f = open(rst_path,'w') f.write(example_rst) @@ -359,6 +374,93 @@ def write_example(src_name, src_dir, rst_dir, cfg): else: shutil.copy(cfg.plot2rst_default_thumb, thumb_path) + save_ipython_notebook(example_file, notebook_dir, notebook_path, basename) + + +def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): + sample_notebook_path = notebook_dir.pjoin('sample.ipynb') + sample = open(sample_notebook_path, 'r') + output = open(notebook_path, 'w') + pythonfile = open(example_file, 'r') + + test = json.load(sample) + code = pythonfile.readlines() + + cell1 = { + "cell_type": "code", + "collapsed": False, + "input": [ + "# Code Goes Here" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + + cell2 = { + "cell_type": "markdown", + "metadata": {}, + "source": [ + 'Markdown Goes Here' + ] + } + + + # Done to cluster together multiple '\n's into one + modified_code = [] + for line in code: + if not modified_code or modified_code[-1] != line: + modified_code.append(line) + + segment_number = 0 + segment_has_begun = True + docstring = False + source = [] + + for line in modified_code: + # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, + # So, ignore it, as already added in cell type 2 + if line == "\n": + if segment_has_begun is True and source: + segment_number += 1 + # we've found text segments within the docstring + if docstring is True: + test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"][segment_number]["source"] = source + else: + test["worksheets"][0]["cells"].append(copy.deepcopy(cell1)) + test["worksheets"][0]["cells"][segment_number]["input"] = source + source = [] + # if its a comment + elif line.strip(' ')[0] == '#': + segment_number += 1 + line = line.strip(' #') + test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"][segment_number]["source"] = line + elif line == "\"\"\"\n": + if docstring is False: + docstring = True + # Indicates, completion of docstring, add whatever in source to markdown (cell type 2) + elif docstring is True: + docstring = False + # Write leftover docstring if any left + if source: + segment_number += 1 + test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"][segment_number]["source"] = source + source = [] + else: + # some text segment is continuing, so add to source + source.append(line) + # if basename == 'plot_brief': + # pdb.set_trace() + + json.dump(test, output, indent=2) + + sample.close() + output.close() + pythonfile.close() + def save_thumbnail(image, thumb_path, shape): """Save image as a thumbnail with the specified shape. diff --git a/doc/source/auto_examples/applications/notebook/sample.ipynb b/doc/source/auto_examples/applications/notebook/sample.ipynb new file mode 100644 index 00000000..80219e00 --- /dev/null +++ b/doc/source/auto_examples/applications/notebook/sample.ipynb @@ -0,0 +1,24 @@ +{ + "metadata": { + "name":"" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%matplotlib inline" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/doc/source/auto_examples/notebook/sample.ipynb b/doc/source/auto_examples/notebook/sample.ipynb new file mode 100644 index 00000000..80219e00 --- /dev/null +++ b/doc/source/auto_examples/notebook/sample.ipynb @@ -0,0 +1,24 @@ +{ + "metadata": { + "name":"" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%matplotlib inline" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file From a047fdcb74f2f636cf798605deaf08a7e7c0d758 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Wed, 19 Mar 2014 11:45:56 +0530 Subject: [PATCH 02/36] Fixes for making code more pythonic, follow conventions --- doc/ext/plot2rst.py | 50 +++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index d1abc20a..b81eb090 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -98,7 +98,7 @@ CODE_LINK = """ NOTEBOOK_LINK = """ -**ipython Notebook source code:** :download:`download <{0}>` +**IPython Notebook:** :download:`download <{0}>` (generated using ``skimage`` |version|) """ @@ -379,14 +379,11 @@ def write_example(src_name, src_dir, rst_dir, cfg): def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): sample_notebook_path = notebook_dir.pjoin('sample.ipynb') - sample = open(sample_notebook_path, 'r') - output = open(notebook_path, 'w') - pythonfile = open(example_file, 'r') + with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: + test = json.load(sample) + code = pythonfile.readlines() - test = json.load(sample) - code = pythonfile.readlines() - - cell1 = { + cell_code = { "cell_type": "code", "collapsed": False, "input": [ @@ -397,7 +394,8 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): "outputs": [] } - cell2 = { + # cell type markdown + cell_md = { "cell_type": "markdown", "metadata": {}, "source": [ @@ -406,11 +404,10 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): } - # Done to cluster together multiple '\n's into one + # Done to cluster together multiple '\n's into one. + # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" modified_code = [] - for line in code: - if not modified_code or modified_code[-1] != line: - modified_code.append(line) + modified_code = [code[i] for i in range(len(code)) if i==0 or code[i]!=code[i-1]] segment_number = 0 segment_has_begun = True @@ -419,47 +416,42 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): for line in modified_code: # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, - # So, ignore it, as already added in cell type 2 + # So, ignore it, as already added in cell type markdown if line == "\n": if segment_has_begun is True and source: segment_number += 1 # we've found text segments within the docstring if docstring is True: - test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) test["worksheets"][0]["cells"][segment_number]["source"] = source else: - test["worksheets"][0]["cells"].append(copy.deepcopy(cell1)) + test["worksheets"][0]["cells"].append(copy.deepcopy(cell_code)) test["worksheets"][0]["cells"][segment_number]["input"] = source source = [] - # if its a comment - elif line.strip(' ')[0] == '#': + # if it's a comment + elif line.strip().startswith('#'): segment_number += 1 line = line.strip(' #') - test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) test["worksheets"][0]["cells"][segment_number]["source"] = line - elif line == "\"\"\"\n": + elif line == '"""\n': if docstring is False: docstring = True - # Indicates, completion of docstring, add whatever in source to markdown (cell type 2) + # Indicates, completion of docstring, add whatever in source to markdown (cell type markdown) elif docstring is True: docstring = False # Write leftover docstring if any left if source: segment_number += 1 - test["worksheets"][0]["cells"].append(copy.deepcopy(cell2)) + test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) test["worksheets"][0]["cells"][segment_number]["source"] = source source = [] else: # some text segment is continuing, so add to source source.append(line) - # if basename == 'plot_brief': - # pdb.set_trace() - json.dump(test, output, indent=2) - - sample.close() - output.close() - pythonfile.close() + with open(notebook_path, 'w') as output: + json.dump(test, output, indent=2) def save_thumbnail(image, thumb_path, shape): From da36df0bc9ceabe58f2dea8f810ba36689319813 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Wed, 9 Apr 2014 23:41:28 +0530 Subject: [PATCH 03/36] added Notebook class for creating IPython notebooks, better name for main object variable --- doc/ext/plot2rst.py | 102 +++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index b81eb090..db6cc346 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -166,6 +166,58 @@ class Path(str): def __iadd__(self, other): return self.__add__(other) +class Notebook(): + """Notebook object for generating an IPython notebook from an example file""" + + + def __init__(self, sample_notebook_path, example_file): + # Object variables, gives the ability to personalise per object + # cell type code + self.cell_code = { + "cell_type": "code", + "collapsed": False, + "input": [ + "# Code Goes Here" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + + # cell type markdown + self.cell_md = { + "cell_type": "markdown", + "metadata": {}, + "source": [ + 'Markdown Goes Here' + ] + } + + self.cell_type = {'input':self.cell_code, 'source': self.cell_md} + with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: + self.template = json.load(sample) + self.code = pythonfile.readlines() + # Adds an extra newline at the end, which aids in extraction of text segments + self.code.append('\n') + + def getModifiedCode(self): + # Done to cluster together multiple '\n's into one. + # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" + modified_code = [] + modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] + return modified_code + + def addcell(self, segment_number, typeOfValue, value): + if typeOfValue in ['source', 'input']: + self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[typeOfValue])) + self.template["worksheets"][0]["cells"][segment_number][typeOfValue] = value + + def jsondump(self, notebook_path): + # writes the template to file + with open(notebook_path, 'w') as output: + json.dump(self.template, output, indent=2) + + def setup(app): app.connect('builder-inited', generate_example_galleries) @@ -379,41 +431,16 @@ def write_example(src_name, src_dir, rst_dir, cfg): def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): sample_notebook_path = notebook_dir.pjoin('sample.ipynb') - with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: - test = json.load(sample) - code = pythonfile.readlines() - - cell_code = { - "cell_type": "code", - "collapsed": False, - "input": [ - "# Code Goes Here" - ], - "language": "python", - "metadata": {}, - "outputs": [] - } - - # cell type markdown - cell_md = { - "cell_type": "markdown", - "metadata": {}, - "source": [ - 'Markdown Goes Here' - ] - } - - - # Done to cluster together multiple '\n's into one. - # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" - modified_code = [] - modified_code = [code[i] for i in range(len(code)) if i==0 or code[i]!=code[i-1]] + + nb = Notebook(sample_notebook_path, example_file) segment_number = 0 segment_has_begun = True docstring = False source = [] + modified_code = nb.getModifiedCode() + for line in modified_code: # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, # So, ignore it, as already added in cell type markdown @@ -422,18 +449,15 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): segment_number += 1 # we've found text segments within the docstring if docstring is True: - test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) - test["worksheets"][0]["cells"][segment_number]["source"] = source + nb.addcell(segment_number, 'source', source) else: - test["worksheets"][0]["cells"].append(copy.deepcopy(cell_code)) - test["worksheets"][0]["cells"][segment_number]["input"] = source + nb.addcell(segment_number, 'input', source) source = [] # if it's a comment elif line.strip().startswith('#'): segment_number += 1 line = line.strip(' #') - test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) - test["worksheets"][0]["cells"][segment_number]["source"] = line + nb.addcell(segment_number, 'source', line) elif line == '"""\n': if docstring is False: docstring = True @@ -443,15 +467,13 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): # Write leftover docstring if any left if source: segment_number += 1 - test["worksheets"][0]["cells"].append(copy.deepcopy(cell_md)) - test["worksheets"][0]["cells"][segment_number]["source"] = source + nb.addcell(segment_number, 'source', source) source = [] else: # some text segment is continuing, so add to source source.append(line) - - with open(notebook_path, 'w') as output: - json.dump(test, output, indent=2) + + nb.jsondump(notebook_path) def save_thumbnail(image, thumb_path, shape): From db795bc79625182942fcefb4cafb1fd975741005 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Thu, 10 Apr 2014 16:46:14 +0530 Subject: [PATCH 04/36] fixes use of camel case --- doc/ext/plot2rst.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index db6cc346..6d471cdb 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -207,10 +207,10 @@ class Notebook(): modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] return modified_code - def addcell(self, segment_number, typeOfValue, value): - if typeOfValue in ['source', 'input']: - self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[typeOfValue])) - self.template["worksheets"][0]["cells"][segment_number][typeOfValue] = value + def addcell(self, segment_number, type_of_value, value): + if type_of_value in ['source', 'input']: + self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) + self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value def jsondump(self, notebook_path): # writes the template to file From a6f396b059b6e5bedeaa9ad3b53073b8cd63a5f2 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Thu, 10 Apr 2014 16:47:40 +0530 Subject: [PATCH 05/36] added newline at end of file --- doc/source/auto_examples/applications/notebook/sample.ipynb | 3 ++- doc/source/auto_examples/notebook/sample.ipynb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/auto_examples/applications/notebook/sample.ipynb b/doc/source/auto_examples/applications/notebook/sample.ipynb index 80219e00..fac5f53e 100644 --- a/doc/source/auto_examples/applications/notebook/sample.ipynb +++ b/doc/source/auto_examples/applications/notebook/sample.ipynb @@ -21,4 +21,5 @@ "metadata": {} } ] -} \ No newline at end of file +} + diff --git a/doc/source/auto_examples/notebook/sample.ipynb b/doc/source/auto_examples/notebook/sample.ipynb index 80219e00..fac5f53e 100644 --- a/doc/source/auto_examples/notebook/sample.ipynb +++ b/doc/source/auto_examples/notebook/sample.ipynb @@ -21,4 +21,5 @@ "metadata": {} } ] -} \ No newline at end of file +} + From 235a9b3ec1334184ed029a95cbbd39dc5e3d1194 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Thu, 10 Apr 2014 18:04:31 +0530 Subject: [PATCH 06/36] moves everything related to generating the IPython notebook, to a different file --- doc/ext/notebook.py | 100 ++++++++++++++++++++++++++++++++++++++++++ doc/ext/plot2rst.py | 103 +------------------------------------------- 2 files changed, 102 insertions(+), 101 deletions(-) create mode 100644 doc/ext/notebook.py diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py new file mode 100644 index 00000000..9f0a2b48 --- /dev/null +++ b/doc/ext/notebook.py @@ -0,0 +1,100 @@ +import json +import copy + +class Notebook(): + """Notebook object for generating an IPython notebook from an example file""" + + + def __init__(self, sample_notebook_path, example_file): + # Object variables, gives the ability to personalise per object + # cell type code + self.cell_code = { + "cell_type": "code", + "collapsed": False, + "input": [ + "# Code Goes Here" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + + # cell type markdown + self.cell_md = { + "cell_type": "markdown", + "metadata": {}, + "source": [ + 'Markdown Goes Here' + ] + } + + self.cell_type = {'input':self.cell_code, 'source': self.cell_md} + with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: + self.template = json.load(sample) + self.code = pythonfile.readlines() + # Adds an extra newline at the end, which aids in extraction of text segments + self.code.append('\n') + + def getModifiedCode(self): + # Done to cluster together multiple '\n's into one. + # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" + modified_code = [] + modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] + return modified_code + + def addcell(self, segment_number, type_of_value, value): + if type_of_value in ['source', 'input']: + self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) + self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value + + def jsondump(self, notebook_path): + # writes the template to file + with open(notebook_path, 'w') as output: + json.dump(self.template, output, indent=2) + + +def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): + sample_notebook_path = notebook_dir.pjoin('sample.ipynb') + + nb = Notebook(sample_notebook_path, example_file) + + segment_number = 0 + segment_has_begun = True + docstring = False + source = [] + + modified_code = nb.getModifiedCode() + + for line in modified_code: + # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, + # So, ignore it, as already added in cell type markdown + if line == "\n": + if segment_has_begun is True and source: + segment_number += 1 + # we've found text segments within the docstring + if docstring is True: + nb.addcell(segment_number, 'source', source) + else: + nb.addcell(segment_number, 'input', source) + source = [] + # if it's a comment + elif line.strip().startswith('#'): + segment_number += 1 + line = line.strip(' #') + nb.addcell(segment_number, 'source', line) + elif line == '"""\n': + if docstring is False: + docstring = True + # Indicates, completion of docstring, add whatever in source to markdown (cell type markdown) + elif docstring is True: + docstring = False + # Write leftover docstring if any left + if source: + segment_number += 1 + nb.addcell(segment_number, 'source', source) + source = [] + else: + # some text segment is continuing, so add to source + source.append(line) + + nb.jsondump(notebook_path) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 6d471cdb..14c7713b 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -70,8 +70,6 @@ import shutil import token import tokenize import traceback -import json -import copy import numpy as np import matplotlib @@ -82,6 +80,8 @@ from skimage import io from skimage import transform from skimage.util.dtype import dtype_range +from notebook import * + LITERALINCLUDE = """ .. literalinclude:: {src_name} @@ -166,58 +166,6 @@ class Path(str): def __iadd__(self, other): return self.__add__(other) -class Notebook(): - """Notebook object for generating an IPython notebook from an example file""" - - - def __init__(self, sample_notebook_path, example_file): - # Object variables, gives the ability to personalise per object - # cell type code - self.cell_code = { - "cell_type": "code", - "collapsed": False, - "input": [ - "# Code Goes Here" - ], - "language": "python", - "metadata": {}, - "outputs": [] - } - - # cell type markdown - self.cell_md = { - "cell_type": "markdown", - "metadata": {}, - "source": [ - 'Markdown Goes Here' - ] - } - - self.cell_type = {'input':self.cell_code, 'source': self.cell_md} - with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: - self.template = json.load(sample) - self.code = pythonfile.readlines() - # Adds an extra newline at the end, which aids in extraction of text segments - self.code.append('\n') - - def getModifiedCode(self): - # Done to cluster together multiple '\n's into one. - # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" - modified_code = [] - modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] - return modified_code - - def addcell(self, segment_number, type_of_value, value): - if type_of_value in ['source', 'input']: - self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) - self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value - - def jsondump(self, notebook_path): - # writes the template to file - with open(notebook_path, 'w') as output: - json.dump(self.template, output, indent=2) - - def setup(app): app.connect('builder-inited', generate_example_galleries) @@ -429,53 +377,6 @@ def write_example(src_name, src_dir, rst_dir, cfg): save_ipython_notebook(example_file, notebook_dir, notebook_path, basename) -def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): - sample_notebook_path = notebook_dir.pjoin('sample.ipynb') - - nb = Notebook(sample_notebook_path, example_file) - - segment_number = 0 - segment_has_begun = True - docstring = False - source = [] - - modified_code = nb.getModifiedCode() - - for line in modified_code: - # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, - # So, ignore it, as already added in cell type markdown - if line == "\n": - if segment_has_begun is True and source: - segment_number += 1 - # we've found text segments within the docstring - if docstring is True: - nb.addcell(segment_number, 'source', source) - else: - nb.addcell(segment_number, 'input', source) - source = [] - # if it's a comment - elif line.strip().startswith('#'): - segment_number += 1 - line = line.strip(' #') - nb.addcell(segment_number, 'source', line) - elif line == '"""\n': - if docstring is False: - docstring = True - # Indicates, completion of docstring, add whatever in source to markdown (cell type markdown) - elif docstring is True: - docstring = False - # Write leftover docstring if any left - if source: - segment_number += 1 - nb.addcell(segment_number, 'source', source) - source = [] - else: - # some text segment is continuing, so add to source - source.append(line) - - nb.jsondump(notebook_path) - - def save_thumbnail(image, thumb_path, shape): """Save image as a thumbnail with the specified shape. From 694739ee0206b84a5d0f263f8ec3ff1dae3d171f Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Thu, 10 Apr 2014 18:40:59 +0530 Subject: [PATCH 07/36] updated import statement for creating ipython notebooks, better name in place for jsonify --- doc/ext/notebook.py | 4 ++-- doc/ext/plot2rst.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 9f0a2b48..39b77211 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -47,7 +47,7 @@ class Notebook(): self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value - def jsondump(self, notebook_path): + def json(self, notebook_path): # writes the template to file with open(notebook_path, 'w') as output: json.dump(self.template, output, indent=2) @@ -97,4 +97,4 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): # some text segment is continuing, so add to source source.append(line) - nb.jsondump(notebook_path) + nb.json(notebook_path) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 14c7713b..9216365c 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -80,7 +80,7 @@ from skimage import io from skimage import transform from skimage.util.dtype import dtype_range -from notebook import * +from notebook import save_ipython_notebook LITERALINCLUDE = """ From 65adb7935b39385d8e4f2c64cf8a120371ae49f9 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Fri, 11 Apr 2014 00:30:21 +0530 Subject: [PATCH 08/36] adds documentation --- doc/ext/notebook.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 39b77211..39ab9a09 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -36,24 +36,36 @@ class Notebook(): self.code.append('\n') def getModifiedCode(self): - # Done to cluster together multiple '\n's into one. - # For ex - "import xyz\n\n\n print 2" becomes "import xyz\n print 2" + """ Clusters multiple '\n's into one. + For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ modified_code = [] modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] return modified_code def addcell(self, segment_number, type_of_value, value): + """ Adds a notebook cell, by updating the json template. Cell differs with type of value """ if type_of_value in ['source', 'input']: self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value def json(self, notebook_path): - # writes the template to file + """ Writes the template to file (json) """ with open(notebook_path, 'w') as output: json.dump(self.template, output, indent=2) def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): + """ Saves a Python file as an IPython notebook + + Parameters + ---------- + example_file : 'str' + path for source Python file + notebook_dir : 'str' + directory for saving the notebook files + notebook_path : 'str' + path for saving the notebook file (includes the filename) + """ sample_notebook_path = notebook_dir.pjoin('sample.ipynb') nb = Notebook(sample_notebook_path, example_file) From 5180da54ada08b22ed8d0d973f76416589a82d51 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Fri, 11 Apr 2014 00:45:22 +0530 Subject: [PATCH 09/36] removes redundant passing of an extra parameter --- doc/ext/notebook.py | 2 +- doc/ext/plot2rst.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 39ab9a09..7a4de781 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -54,7 +54,7 @@ class Notebook(): json.dump(self.template, output, indent=2) -def save_ipython_notebook(example_file, notebook_dir, notebook_path, basename): +def save_ipython_notebook(example_file, notebook_dir, notebook_path): """ Saves a Python file as an IPython notebook Parameters diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 9216365c..7de4f589 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -374,7 +374,7 @@ def write_example(src_name, src_dir, rst_dir, cfg): else: shutil.copy(cfg.plot2rst_default_thumb, thumb_path) - save_ipython_notebook(example_file, notebook_dir, notebook_path, basename) + save_ipython_notebook(example_file, notebook_dir, notebook_path) def save_thumbnail(image, thumb_path, shape): From 1e34359f04bba68e88ddd7bd8e59f1ad204ba17f Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 12:19:36 +0530 Subject: [PATCH 10/36] Fix PEP8 --- doc/ext/notebook.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 7a4de781..e65ca8c5 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -1,34 +1,37 @@ import json import copy -class Notebook(): - """Notebook object for generating an IPython notebook from an example file""" +class Notebook(): + """ + Notebook object for generating an IPython notebook + from an example Python file. + """ def __init__(self, sample_notebook_path, example_file): # Object variables, gives the ability to personalise per object # cell type code self.cell_code = { - "cell_type": "code", - "collapsed": False, - "input": [ - "# Code Goes Here" - ], - "language": "python", - "metadata": {}, - "outputs": [] + "cell_type": "code", + "collapsed": False, + "input": [ + "# Code Goes Here" + ], + "language": "python", + "metadata": {}, + "outputs": [] } # cell type markdown self.cell_md = { - "cell_type": "markdown", - "metadata": {}, - "source": [ - 'Markdown Goes Here' - ] + "cell_type": "markdown", + "metadata": {}, + "source": [ + 'Markdown Goes Here' + ] } - self.cell_type = {'input':self.cell_code, 'source': self.cell_md} + self.cell_type = {'input': self.cell_code, 'source': self.cell_md} with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: self.template = json.load(sample) self.code = pythonfile.readlines() @@ -39,7 +42,7 @@ class Notebook(): """ Clusters multiple '\n's into one. For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ modified_code = [] - modified_code = [self.code[i] for i in range(len(self.code)) if i==0 or self.code[i]!=self.code[i-1]] + modified_code = [self.code[i] for i in range(len(self.code)) if i == 0 or self.code[i] != self.code[i-1]] return modified_code def addcell(self, segment_number, type_of_value, value): @@ -55,7 +58,7 @@ class Notebook(): def save_ipython_notebook(example_file, notebook_dir, notebook_path): - """ Saves a Python file as an IPython notebook + """ Saves a Python file as an IPython notebook Parameters ---------- @@ -67,7 +70,7 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path): path for saving the notebook file (includes the filename) """ sample_notebook_path = notebook_dir.pjoin('sample.ipynb') - + nb = Notebook(sample_notebook_path, example_file) segment_number = 0 @@ -108,5 +111,5 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path): else: # some text segment is continuing, so add to source source.append(line) - + nb.json(notebook_path) From 0c9ee09f041557ced6b1bf56fcd6f2c1f58a8610 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 12:42:30 +0530 Subject: [PATCH 11/36] Fix camel case, more descriptive function name --- doc/ext/notebook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index e65ca8c5..74cf881d 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -38,7 +38,7 @@ class Notebook(): # Adds an extra newline at the end, which aids in extraction of text segments self.code.append('\n') - def getModifiedCode(self): + def filter_continous_duplication(self): """ Clusters multiple '\n's into one. For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ modified_code = [] @@ -78,7 +78,7 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path): docstring = False source = [] - modified_code = nb.getModifiedCode() + modified_code = nb.filter_continous_duplication() for line in modified_code: # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, From a53f880256b1189ed28df38a094a2e2c54d30225 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 12:51:45 +0530 Subject: [PATCH 12/36] Fix typo, better function name and docstring summary --- doc/ext/notebook.py | 8 ++++---- doc/ext/plot2rst.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 74cf881d..ab67f72b 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -38,7 +38,7 @@ class Notebook(): # Adds an extra newline at the end, which aids in extraction of text segments self.code.append('\n') - def filter_continous_duplication(self): + def filter_continuous_duplication(self): """ Clusters multiple '\n's into one. For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ modified_code = [] @@ -57,8 +57,8 @@ class Notebook(): json.dump(self.template, output, indent=2) -def save_ipython_notebook(example_file, notebook_dir, notebook_path): - """ Saves a Python file as an IPython notebook +def python_to_notebook(example_file, notebook_dir, notebook_path): + """ Convert a Python file to an IPython notebook. Parameters ---------- @@ -78,7 +78,7 @@ def save_ipython_notebook(example_file, notebook_dir, notebook_path): docstring = False source = [] - modified_code = nb.filter_continous_duplication() + modified_code = nb.filter_continuous_duplication() for line in modified_code: # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 7de4f589..6131c50e 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -374,7 +374,7 @@ def write_example(src_name, src_dir, rst_dir, cfg): else: shutil.copy(cfg.plot2rst_default_thumb, thumb_path) - save_ipython_notebook(example_file, notebook_dir, notebook_path) + python_to_notebook(example_file, notebook_dir, notebook_path) def save_thumbnail(image, thumb_path, shape): From 07970bd0b0b02dd7fd8bbe0f924c6d7309eebd2b Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 13:11:45 +0530 Subject: [PATCH 13/36] Fix PEP8, line lengths for comments --- doc/ext/notebook.py | 13 +++++++++---- doc/ext/plot2rst.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index ab67f72b..b62401fa 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -35,7 +35,8 @@ class Notebook(): with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: self.template = json.load(sample) self.code = pythonfile.readlines() - # Adds an extra newline at the end, which aids in extraction of text segments + # Adds an extra newline at the end, + # this aids in extraction of text segments self.code.append('\n') def filter_continuous_duplication(self): @@ -46,7 +47,9 @@ class Notebook(): return modified_code def addcell(self, segment_number, type_of_value, value): - """ Adds a notebook cell, by updating the json template. Cell differs with type of value """ + """ Adds a notebook cell, by updating the json template. + Cell differs with type of value. + """ if type_of_value in ['source', 'input']: self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value @@ -81,7 +84,8 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): modified_code = nb.filter_continuous_duplication() for line in modified_code: - # A linebreak indicates a segment has ended. If the text segment had only comments, then source is blank, + # A linebreak indicates a segment has ended. + # If the text segment had only comments, then source is blank, # So, ignore it, as already added in cell type markdown if line == "\n": if segment_has_begun is True and source: @@ -100,7 +104,8 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): elif line == '"""\n': if docstring is False: docstring = True - # Indicates, completion of docstring, add whatever in source to markdown (cell type markdown) + # Indicates, completion of docstring, + # add whatever in source to markdown (cell type markdown) elif docstring is True: docstring = False # Write leftover docstring if any left diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 6131c50e..e951729f 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -357,7 +357,7 @@ def write_example(src_name, src_dir, rst_dir, cfg): ipnotebook_name = './notebook/' + ipnotebook_name example_rst += NOTEBOOK_LINK.format(ipnotebook_name) - f = open(rst_path,'w') + f = open(rst_path, 'w') f.write(example_rst) f.flush() From 62a91370f8f7be96ff2ddd6557c14ceac03fa734 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 15:08:28 +0530 Subject: [PATCH 14/36] Add __all__ to specify modules which should be imported --- doc/ext/notebook.py | 2 ++ doc/ext/plot2rst.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index b62401fa..43ede80c 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -1,3 +1,5 @@ +__all__ = ['python_to_notebook'] + import json import copy diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index e951729f..07b54470 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -80,7 +80,7 @@ from skimage import io from skimage import transform from skimage.util.dtype import dtype_range -from notebook import save_ipython_notebook +from notebook import python_to_notebook LITERALINCLUDE = """ From fcc607f4d10a185e6427e93f0a10e10c417669e4 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 16:02:13 +0530 Subject: [PATCH 15/36] Add default value for adding cell, better naming for type of value --- doc/ext/notebook.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 43ede80c..210c6878 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -33,7 +33,7 @@ class Notebook(): ] } - self.cell_type = {'input': self.cell_code, 'source': self.cell_md} + self.cell_type = {'input_code': self.cell_code, 'input_markdown': self.cell_md} with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: self.template = json.load(sample) self.code = pythonfile.readlines() @@ -48,11 +48,11 @@ class Notebook(): modified_code = [self.code[i] for i in range(len(self.code)) if i == 0 or self.code[i] != self.code[i-1]] return modified_code - def addcell(self, segment_number, type_of_value, value): + def addcell(self, segment_number, value, type_of_value='input_code'): """ Adds a notebook cell, by updating the json template. Cell differs with type of value. """ - if type_of_value in ['source', 'input']: + if type_of_value in ['input_markdown', 'input_code']: self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value @@ -94,15 +94,15 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): segment_number += 1 # we've found text segments within the docstring if docstring is True: - nb.addcell(segment_number, 'source', source) + nb.addcell(segment_number, source, 'input_markdown') else: - nb.addcell(segment_number, 'input', source) + nb.addcell(segment_number, source, 'input_code') source = [] # if it's a comment elif line.strip().startswith('#'): segment_number += 1 line = line.strip(' #') - nb.addcell(segment_number, 'source', line) + nb.addcell(segment_number, line, 'input_markdown') elif line == '"""\n': if docstring is False: docstring = True @@ -113,7 +113,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # Write leftover docstring if any left if source: segment_number += 1 - nb.addcell(segment_number, 'source', source) + nb.addcell(segment_number, source, 'input_markdown') source = [] else: # some text segment is continuing, so add to source From 6c0a5706c3766999841e8747bb78b78062cf5b6a Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Sun, 13 Apr 2014 20:49:46 +0530 Subject: [PATCH 16/36] Remove sample.ipynb, template now inline --- doc/ext/notebook.py | 43 ++++++++++++++++--- .../applications/notebook/sample.ipynb | 25 ----------- .../auto_examples/notebook/sample.ipynb | 25 ----------- 3 files changed, 38 insertions(+), 55 deletions(-) delete mode 100644 doc/source/auto_examples/applications/notebook/sample.ipynb delete mode 100644 doc/source/auto_examples/notebook/sample.ipynb diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 210c6878..6507020c 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -3,6 +3,31 @@ __all__ = ['python_to_notebook'] import json import copy +sample = """{ + "metadata": { + "name":"" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%matplotlib inline" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +}""" + class Notebook(): """ @@ -33,14 +58,21 @@ class Notebook(): ] } - self.cell_type = {'input_code': self.cell_code, 'input_markdown': self.cell_md} - with open(sample_notebook_path, 'r') as sample, open(example_file, 'r') as pythonfile: - self.template = json.load(sample) + self.cell_type = {'input': self.cell_code, 'source': self.cell_md} + self.keys = {'input_code': 'input', 'input_markdown': 'source'} + with open(example_file, 'r') as pythonfile: + self.template = json.loads(sample) self.code = pythonfile.readlines() # Adds an extra newline at the end, # this aids in extraction of text segments self.code.append('\n') + def fetchkey(self, type_of_value): + """ Returns the key required for insertion into notebook, + based on the type of value. + """ + return self.keys[type_of_value] + def filter_continuous_duplication(self): """ Clusters multiple '\n's into one. For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ @@ -53,8 +85,9 @@ class Notebook(): Cell differs with type of value. """ if type_of_value in ['input_markdown', 'input_code']: - self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[type_of_value])) - self.template["worksheets"][0]["cells"][segment_number][type_of_value] = value + key = self.fetchkey(type_of_value) + self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[key])) + self.template["worksheets"][0]["cells"][segment_number][key] = value def json(self, notebook_path): """ Writes the template to file (json) """ diff --git a/doc/source/auto_examples/applications/notebook/sample.ipynb b/doc/source/auto_examples/applications/notebook/sample.ipynb deleted file mode 100644 index fac5f53e..00000000 --- a/doc/source/auto_examples/applications/notebook/sample.ipynb +++ /dev/null @@ -1,25 +0,0 @@ -{ - "metadata": { - "name":"" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ - { - "cells": [ - { - "cell_type": "code", - "collapsed": false, - "input": [ - "%matplotlib inline" - ], - "language": "python", - "metadata": {}, - "outputs": [] - } - ], - "metadata": {} - } - ] -} - diff --git a/doc/source/auto_examples/notebook/sample.ipynb b/doc/source/auto_examples/notebook/sample.ipynb deleted file mode 100644 index fac5f53e..00000000 --- a/doc/source/auto_examples/notebook/sample.ipynb +++ /dev/null @@ -1,25 +0,0 @@ -{ - "metadata": { - "name":"" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ - { - "cells": [ - { - "cell_type": "code", - "collapsed": false, - "input": [ - "%matplotlib inline" - ], - "language": "python", - "metadata": {}, - "outputs": [] - } - ], - "metadata": {} - } - ] -} - From c9a981445d7c7a3ebec1e9636e146f4abbad8fd9 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 02:25:18 +0530 Subject: [PATCH 17/36] Add and improve docstrings, JSON dumping and duplicate removal generalised --- doc/ext/notebook.py | 138 ++++++++++++++++++++++++++++++++------------ 1 file changed, 100 insertions(+), 38 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 6507020c..c062585b 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -29,13 +29,43 @@ sample = """{ }""" +def remove_continuous_duplicates(code): + """ Remove duplicates of elements appearing consecutively. + + Parameters + ---------- + code : list of str + + Returns + ------- + modified_code : list of str + + Notes + ----- + We create a new list and add elements to it which do not have + duplicates appearing consecutively. + One use case here, + 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' + + """ + modified_code = [] + modified_code = [self.code[i] for i in range(len(self.code)) if i == 0 or self.code[i] != self.code[i-1]] + return modified_code + + class Notebook(): """ - Notebook object for generating an IPython notebook + Notebook object for generating an IPython notebook, from an example Python file. + + Parameters + ---------- + example_file : str + Path for example file. + """ - def __init__(self, sample_notebook_path, example_file): + def __init__(self, example_file): # Object variables, gives the ability to personalise per object # cell type code self.cell_code = { @@ -59,7 +89,7 @@ class Notebook(): } self.cell_type = {'input': self.cell_code, 'source': self.cell_md} - self.keys = {'input_code': 'input', 'input_markdown': 'source'} + self.valuetype_to_celltype = {'code': 'input', 'markdown': 'source'} with open(example_file, 'r') as pythonfile: self.template = json.loads(sample) self.code = pythonfile.readlines() @@ -67,32 +97,64 @@ class Notebook(): # this aids in extraction of text segments self.code.append('\n') - def fetchkey(self, type_of_value): - """ Returns the key required for insertion into notebook, - based on the type of value. - """ - return self.keys[type_of_value] + def fetch_key(self, type_of_value): + """ Find the key required for insertion into notebook. - def filter_continuous_duplication(self): - """ Clusters multiple '\n's into one. - For ex - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' """ - modified_code = [] - modified_code = [self.code[i] for i in range(len(self.code)) if i == 0 or self.code[i] != self.code[i-1]] - return modified_code + Parameters + ---------- + type_of_value : str + Type of data, to be inserted in a cell. + + Returns + ------- + str + Key which reflects what is the cell type. + + Notes + ----- + type_of_value is either, 'code', which maps to cell of type + code('input') or 'markdown', which maps to cell of type markdown('source'). - def addcell(self, segment_number, value, type_of_value='input_code'): - """ Adds a notebook cell, by updating the json template. - Cell differs with type of value. """ - if type_of_value in ['input_markdown', 'input_code']: - key = self.fetchkey(type_of_value) + return self.valuetype_to_celltype[type_of_value] + + def add_cell(self, segment_number, value, type_of_value='code'): + """ Adds a notebook cell. + + Parameters + ---------- + segment_number : int + Newline separated sections in example file, are segments. + Code and markdown written together in such a section are, + treated as different segments. Each cell has content from + one section. + value : str + The actual content to be saved in the cell. + type_of_value : str, optional + The type of content in the segment. + The default value will add a cell of type code. + + Notes + ----- + The cell is only added in the notebook if the segment is of type, + markdown or code. + + """ + if type_of_value in ['markdown', 'code']: + key = self.fetch_key(type_of_value) self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[key])) self.template["worksheets"][0]["cells"][segment_number][key] = value - def json(self, notebook_path): - """ Writes the template to file (json) """ - with open(notebook_path, 'w') as output: - json.dump(self.template, output, indent=2) + def json(self): + """ Dumps the template JSON to string. + + Returns + ------- + str + The template JSON converted to a string with a two char indent. + + """ + return json.dump(self.template, indent=2) def python_to_notebook(example_file, notebook_dir, notebook_path): @@ -100,23 +162,22 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): Parameters ---------- - example_file : 'str' - path for source Python file - notebook_dir : 'str' - directory for saving the notebook files - notebook_path : 'str' - path for saving the notebook file (includes the filename) - """ - sample_notebook_path = notebook_dir.pjoin('sample.ipynb') + example_file : str + Path for source Python file. + notebook_dir : str + Directory for saving the notebook files. + notebook_path : str + Path for saving the notebook file (includes the filename). - nb = Notebook(sample_notebook_path, example_file) + """ + nb = Notebook(example_file) segment_number = 0 segment_has_begun = True docstring = False source = [] - modified_code = nb.filter_continuous_duplication() + modified_code = remove_continuous_duplicates(nb.code) for line in modified_code: # A linebreak indicates a segment has ended. @@ -127,15 +188,15 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): segment_number += 1 # we've found text segments within the docstring if docstring is True: - nb.addcell(segment_number, source, 'input_markdown') + nb.add_cell(segment_number, source, 'markdown') else: - nb.addcell(segment_number, source, 'input_code') + nb.add_cell(segment_number, source, 'code') source = [] # if it's a comment elif line.strip().startswith('#'): segment_number += 1 line = line.strip(' #') - nb.addcell(segment_number, line, 'input_markdown') + nb.add_cell(segment_number, line, 'markdown') elif line == '"""\n': if docstring is False: docstring = True @@ -146,10 +207,11 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # Write leftover docstring if any left if source: segment_number += 1 - nb.addcell(segment_number, source, 'input_markdown') + nb.add_cell(segment_number, source, 'markdown') source = [] else: # some text segment is continuing, so add to source source.append(line) - nb.json(notebook_path) + with open(notebook_path, 'w') as output: + output.write(nb.json(notebook_path)) From a57cf0db4bd42b545796483e03ee97ac474319af Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 02:37:29 +0530 Subject: [PATCH 18/36] Improve docstring for add_cell --- doc/ext/notebook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index c062585b..bb2a6c1c 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -124,10 +124,10 @@ class Notebook(): Parameters ---------- segment_number : int - Newline separated sections in example file, are segments. - Code and markdown written together in such a section are, - treated as different segments. Each cell has content from - one section. + Newline separated portions in example file, are sections. + Code and markdown written together in such a section are further + treated as different segments. Each cell has content from one + segment. value : str The actual content to be saved in the cell. type_of_value : str, optional From 9cbea5fcba6785ecb617103a67d76707e1dbc3cd Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 02:59:00 +0530 Subject: [PATCH 19/36] Fix to incorporate a more generalised duplicate removal and JSON dump to string --- doc/ext/notebook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index bb2a6c1c..9efe122e 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -49,7 +49,7 @@ def remove_continuous_duplicates(code): """ modified_code = [] - modified_code = [self.code[i] for i in range(len(self.code)) if i == 0 or self.code[i] != self.code[i-1]] + modified_code = [code[i] for i in range(len(code)) if i == 0 or code[i] != code[i-1]] return modified_code @@ -154,7 +154,7 @@ class Notebook(): The template JSON converted to a string with a two char indent. """ - return json.dump(self.template, indent=2) + return json.dumps(self.template, indent=2) def python_to_notebook(example_file, notebook_dir, notebook_path): @@ -214,4 +214,4 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): source.append(line) with open(notebook_path, 'w') as output: - output.write(nb.json(notebook_path)) + output.write(nb.json()) From 29b1ce62a8a73a3a90e37d0ad8a9d04d3e1c86cf Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:08:07 +0530 Subject: [PATCH 20/36] Improve docstring for consecutive duplicate removal, more readable --- doc/ext/notebook.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 9efe122e..a91c3033 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -29,34 +29,35 @@ sample = """{ }""" -def remove_continuous_duplicates(code): +def remove_consecutive_duplicates(inp): """ Remove duplicates of elements appearing consecutively. Parameters ---------- - code : list of str + inp : list + Input list. Returns ------- - modified_code : list of str + modified_inp : list + Output list, with no consecutive duplicates. - Notes - ----- - We create a new list and add elements to it which do not have - duplicates appearing consecutively. - One use case here, - 'import xyz\n\n\n print 2' becomes 'import xyz\n print 2' + Examples + -------- + >>> input = [1, 2, 3, 3, 4, 5, 6, 6] + >>> output = remove_consecutive_duplicates(input) + >>> output + [1, 2, 3, 4, 5, 6] """ - modified_code = [] - modified_code = [code[i] for i in range(len(code)) if i == 0 or code[i] != code[i-1]] - return modified_code + modified_inp = [inp[0]] + [inp[i] for i in range(1, len(inp)) if inp[i] != inp[i-1]] + return modified_inp class Notebook(): """ - Notebook object for generating an IPython notebook, - from an example Python file. + Notebook object for generating an IPython notebook from an example Python + file. Parameters ---------- @@ -177,7 +178,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): docstring = False source = [] - modified_code = remove_continuous_duplicates(nb.code) + modified_code = remove_consecutive_duplicates(nb.code) for line in modified_code: # A linebreak indicates a segment has ended. From 202244688cb9d8de818d37a528dbfbfcca3a400c Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:09:41 +0530 Subject: [PATCH 21/36] Remove extra spaces --- doc/ext/notebook.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index a91c3033..01a2e94c 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -30,7 +30,7 @@ sample = """{ def remove_consecutive_duplicates(inp): - """ Remove duplicates of elements appearing consecutively. + """Remove duplicates of elements appearing consecutively. Parameters ---------- @@ -99,7 +99,7 @@ class Notebook(): self.code.append('\n') def fetch_key(self, type_of_value): - """ Find the key required for insertion into notebook. + """Find the key required for insertion into notebook. Parameters ---------- @@ -120,7 +120,7 @@ class Notebook(): return self.valuetype_to_celltype[type_of_value] def add_cell(self, segment_number, value, type_of_value='code'): - """ Adds a notebook cell. + """Adds a notebook cell. Parameters ---------- @@ -147,7 +147,7 @@ class Notebook(): self.template["worksheets"][0]["cells"][segment_number][key] = value def json(self): - """ Dumps the template JSON to string. + """Dumps the template JSON to string. Returns ------- @@ -159,7 +159,7 @@ class Notebook(): def python_to_notebook(example_file, notebook_dir, notebook_path): - """ Convert a Python file to an IPython notebook. + """Convert a Python file to an IPython notebook. Parameters ---------- From 3ddd5ce109065a50fbbf21a7d840e8f3462d5daa Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:16:39 +0530 Subject: [PATCH 22/36] Remove fetch_key, an overkill --- doc/ext/notebook.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 01a2e94c..7ca41475 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -98,27 +98,6 @@ class Notebook(): # this aids in extraction of text segments self.code.append('\n') - def fetch_key(self, type_of_value): - """Find the key required for insertion into notebook. - - Parameters - ---------- - type_of_value : str - Type of data, to be inserted in a cell. - - Returns - ------- - str - Key which reflects what is the cell type. - - Notes - ----- - type_of_value is either, 'code', which maps to cell of type - code('input') or 'markdown', which maps to cell of type markdown('source'). - - """ - return self.valuetype_to_celltype[type_of_value] - def add_cell(self, segment_number, value, type_of_value='code'): """Adds a notebook cell. @@ -135,14 +114,9 @@ class Notebook(): The type of content in the segment. The default value will add a cell of type code. - Notes - ----- - The cell is only added in the notebook if the segment is of type, - markdown or code. - """ if type_of_value in ['markdown', 'code']: - key = self.fetch_key(type_of_value) + key = self.valuetype_to_celltype[type_of_value] self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[key])) self.template["worksheets"][0]["cells"][segment_number][key] = value From ddf8969655b2783d8874362386ffb3f971003430 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:18:04 +0530 Subject: [PATCH 23/36] Add string options to the docstring --- doc/ext/notebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 7ca41475..8d44b187 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -110,7 +110,7 @@ class Notebook(): segment. value : str The actual content to be saved in the cell. - type_of_value : str, optional + type_of_value : {'code', 'markdown'} The type of content in the segment. The default value will add a cell of type code. From e104c687aa0dc70358bef75fca3db68d9e5b6bf2 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:28:25 +0530 Subject: [PATCH 24/36] More pythonic --- doc/ext/notebook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 8d44b187..b3123a51 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -162,7 +162,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): if segment_has_begun is True and source: segment_number += 1 # we've found text segments within the docstring - if docstring is True: + if docstring: nb.add_cell(segment_number, source, 'markdown') else: nb.add_cell(segment_number, source, 'code') @@ -175,9 +175,9 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): elif line == '"""\n': if docstring is False: docstring = True - # Indicates, completion of docstring, + # Indicates, completion of docstring # add whatever in source to markdown (cell type markdown) - elif docstring is True: + elif docstring: docstring = False # Write leftover docstring if any left if source: From 468860acdd23471226d37f79f9fb23b8fcd7d26b Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 04:49:19 +0530 Subject: [PATCH 25/36] Add class to __all__, remove duplicates now private --- doc/ext/notebook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index b3123a51..5c57fa16 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -1,4 +1,4 @@ -__all__ = ['python_to_notebook'] +__all__ = ['python_to_notebook', 'Notebook'] import json import copy @@ -29,7 +29,7 @@ sample = """{ }""" -def remove_consecutive_duplicates(inp): +def _remove_consecutive_duplicates(inp): """Remove duplicates of elements appearing consecutively. Parameters @@ -152,7 +152,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): docstring = False source = [] - modified_code = remove_consecutive_duplicates(nb.code) + modified_code = _remove_consecutive_duplicates(nb.code) for line in modified_code: # A linebreak indicates a segment has ended. From f675539aad3926c537667d736b4dcaaa9cc7f839 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Mon, 14 Apr 2014 21:53:39 +0530 Subject: [PATCH 26/36] Remove segment_number, directly change the last cell --- doc/ext/notebook.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 5c57fa16..1e04ad70 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -98,27 +98,23 @@ class Notebook(): # this aids in extraction of text segments self.code.append('\n') - def add_cell(self, segment_number, value, type_of_value='code'): + def add_cell(self, value, type_of_value='code'): """Adds a notebook cell. Parameters ---------- - segment_number : int - Newline separated portions in example file, are sections. - Code and markdown written together in such a section are further - treated as different segments. Each cell has content from one - segment. value : str The actual content to be saved in the cell. type_of_value : {'code', 'markdown'} - The type of content in the segment. + The type of content. The default value will add a cell of type code. """ if type_of_value in ['markdown', 'code']: key = self.valuetype_to_celltype[type_of_value] self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[key])) - self.template["worksheets"][0]["cells"][segment_number][key] = value + # assign value to the last cell + self.template["worksheets"][0]["cells"][-1][key] = value def json(self): """Dumps the template JSON to string. @@ -147,7 +143,10 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): """ nb = Notebook(example_file) - segment_number = 0 + # Newline separated portions in example file, are sections. + # Code and markdown written together in such a section are further + # treated as different segments. Each cell has content from one + # segment. segment_has_begun = True docstring = False source = [] @@ -160,18 +159,16 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # So, ignore it, as already added in cell type markdown if line == "\n": if segment_has_begun is True and source: - segment_number += 1 # we've found text segments within the docstring if docstring: - nb.add_cell(segment_number, source, 'markdown') + nb.add_cell(source, 'markdown') else: - nb.add_cell(segment_number, source, 'code') + nb.add_cell(source, 'code') source = [] # if it's a comment elif line.strip().startswith('#'): - segment_number += 1 line = line.strip(' #') - nb.add_cell(segment_number, line, 'markdown') + nb.add_cell(line, 'markdown') elif line == '"""\n': if docstring is False: docstring = True @@ -181,8 +178,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): docstring = False # Write leftover docstring if any left if source: - segment_number += 1 - nb.add_cell(segment_number, source, 'markdown') + nb.add_cell(source, 'markdown') source = [] else: # some text segment is continuing, so add to source From 8edc6059eac9e9920a05d0ef35c90cfa1e10d531 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Tue, 15 Apr 2014 14:58:22 +0530 Subject: [PATCH 27/36] Fix PEP8, more readable --- doc/ext/notebook.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 1e04ad70..97217c28 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -29,17 +29,17 @@ sample = """{ }""" -def _remove_consecutive_duplicates(inp): +def _remove_consecutive_duplicates(x): """Remove duplicates of elements appearing consecutively. Parameters ---------- - inp : list + x : list Input list. Returns ------- - modified_inp : list + modified_x : list Output list, with no consecutive duplicates. Examples @@ -50,8 +50,8 @@ def _remove_consecutive_duplicates(inp): [1, 2, 3, 4, 5, 6] """ - modified_inp = [inp[0]] + [inp[i] for i in range(1, len(inp)) if inp[i] != inp[i-1]] - return modified_inp + modified_x = [x[0]] + [x[i] for i in range(1, len(x)) if x[i] != x[i-1]] + return modified_x class Notebook(): @@ -67,7 +67,7 @@ class Notebook(): """ def __init__(self, example_file): - # Object variables, gives the ability to personalise per object + # Object variables, give the ability to personalise, each object # cell type code self.cell_code = { "cell_type": "code", @@ -94,12 +94,12 @@ class Notebook(): with open(example_file, 'r') as pythonfile: self.template = json.loads(sample) self.code = pythonfile.readlines() - # Adds an extra newline at the end, + # Add an extra newline at the end, # this aids in extraction of text segments self.code.append('\n') def add_cell(self, value, type_of_value='code'): - """Adds a notebook cell. + """Add a notebook cell. Parameters ---------- @@ -112,12 +112,13 @@ class Notebook(): """ if type_of_value in ['markdown', 'code']: key = self.valuetype_to_celltype[type_of_value] - self.template["worksheets"][0]["cells"].append(copy.deepcopy(self.cell_type[key])) + cells = self.template["worksheets"][0]["cells"] + cells.append(copy.deepcopy(self.cell_type[key])) # assign value to the last cell - self.template["worksheets"][0]["cells"][-1][key] = value + cells[-1][key] = value def json(self): - """Dumps the template JSON to string. + """Dump the template JSON to string. Returns ------- From c089c624b186410cd3a5cfee5ecce3ad20c097d6 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Tue, 15 Apr 2014 16:54:43 +0530 Subject: [PATCH 28/36] More generic Notebook class --- doc/ext/notebook.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 97217c28..fc17199b 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -66,7 +66,7 @@ class Notebook(): """ - def __init__(self, example_file): + def __init__(self): # Object variables, give the ability to personalise, each object # cell type code self.cell_code = { @@ -91,12 +91,6 @@ class Notebook(): self.cell_type = {'input': self.cell_code, 'source': self.cell_md} self.valuetype_to_celltype = {'code': 'input', 'markdown': 'source'} - with open(example_file, 'r') as pythonfile: - self.template = json.loads(sample) - self.code = pythonfile.readlines() - # Add an extra newline at the end, - # this aids in extraction of text segments - self.code.append('\n') def add_cell(self, value, type_of_value='code'): """Add a notebook cell. @@ -142,7 +136,13 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): Path for saving the notebook file (includes the filename). """ - nb = Notebook(example_file) + nb = Notebook() + with open(example_file, 'r') as pythonfile: + nb.template = json.loads(sample) + nb.code = pythonfile.readlines() + # Add an extra newline at the end, + # this aids in extraction of text segments + nb.code.append('\n') # Newline separated portions in example file, are sections. # Code and markdown written together in such a section are further From bc7f3d69e2f3fc80b9a30dc211586b65c3e1a82c Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Tue, 15 Apr 2014 23:17:57 +0530 Subject: [PATCH 29/36] More pythonic if statements --- doc/ext/notebook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index fc17199b..35ccc125 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -159,7 +159,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # If the text segment had only comments, then source is blank, # So, ignore it, as already added in cell type markdown if line == "\n": - if segment_has_begun is True and source: + if segment_has_begun and source: # we've found text segments within the docstring if docstring: nb.add_cell(source, 'markdown') @@ -171,7 +171,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): line = line.strip(' #') nb.add_cell(line, 'markdown') elif line == '"""\n': - if docstring is False: + if not docstring: docstring = True # Indicates, completion of docstring # add whatever in source to markdown (cell type markdown) From e79c1ba66256c121878a02c314087982d218fc2b Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Wed, 16 Apr 2014 13:57:38 +0530 Subject: [PATCH 30/36] Add warning, consistent use of quotes, correction in docs --- doc/ext/notebook.py | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 35ccc125..5cefdef0 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -2,6 +2,7 @@ __all__ = ['python_to_notebook', 'Notebook'] import json import copy +import warnings sample = """{ "metadata": { @@ -59,32 +60,26 @@ class Notebook(): Notebook object for generating an IPython notebook from an example Python file. - Parameters - ---------- - example_file : str - Path for example file. - """ def __init__(self): - # Object variables, give the ability to personalise, each object # cell type code self.cell_code = { - "cell_type": "code", - "collapsed": False, - "input": [ - "# Code Goes Here" + 'cell_type': 'code', + 'collapsed': False, + 'input': [ + '# Code Goes Here' ], - "language": "python", - "metadata": {}, - "outputs": [] + 'language': 'python', + 'metadata': {}, + 'outputs': [] } # cell type markdown self.cell_md = { - "cell_type": "markdown", - "metadata": {}, - "source": [ + 'cell_type': 'markdown', + 'metadata': {}, + 'source': [ 'Markdown Goes Here' ] } @@ -92,24 +87,26 @@ class Notebook(): self.cell_type = {'input': self.cell_code, 'source': self.cell_md} self.valuetype_to_celltype = {'code': 'input', 'markdown': 'source'} - def add_cell(self, value, type_of_value='code'): + def add_cell(self, value, cell_type='code'): """Add a notebook cell. Parameters ---------- value : str The actual content to be saved in the cell. - type_of_value : {'code', 'markdown'} + cell_type : {'code', 'markdown'} The type of content. - The default value will add a cell of type code. + The default value will add a cell of type 'code'. """ - if type_of_value in ['markdown', 'code']: - key = self.valuetype_to_celltype[type_of_value] - cells = self.template["worksheets"][0]["cells"] + if cell_type in ['markdown', 'code']: + key = self.valuetype_to_celltype[cell_type] + cells = self.template['worksheets'][0]['cells'] cells.append(copy.deepcopy(self.cell_type[key])) # assign value to the last cell cells[-1][key] = value + else: + warnings.warn('Unsupported cell type %s, data ignored' % cell_type) def json(self): """Dump the template JSON to string. @@ -158,7 +155,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # A linebreak indicates a segment has ended. # If the text segment had only comments, then source is blank, # So, ignore it, as already added in cell type markdown - if line == "\n": + if line == '\n': if segment_has_begun and source: # we've found text segments within the docstring if docstring: From dbf9f0b230f4acf80cd8f9f78183b480f24bc218 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Wed, 16 Apr 2014 17:38:55 +0530 Subject: [PATCH 31/36] Remove use of redundant variables, cleanup --- doc/ext/notebook.py | 17 +++++++---------- doc/ext/plot2rst.py | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 5cefdef0..36c6fa1f 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -120,15 +120,13 @@ class Notebook(): return json.dumps(self.template, indent=2) -def python_to_notebook(example_file, notebook_dir, notebook_path): +def python_to_notebook(example_file, notebook_path): """Convert a Python file to an IPython notebook. Parameters ---------- example_file : str Path for source Python file. - notebook_dir : str - Directory for saving the notebook files. notebook_path : str Path for saving the notebook file (includes the filename). @@ -145,18 +143,17 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): # Code and markdown written together in such a section are further # treated as different segments. Each cell has content from one # segment. - segment_has_begun = True docstring = False source = [] - modified_code = _remove_consecutive_duplicates(nb.code) + code = _remove_consecutive_duplicates(nb.code) - for line in modified_code: + for line in code: # A linebreak indicates a segment has ended. - # If the text segment had only comments, then source is blank, - # So, ignore it, as already added in cell type markdown + # If the text segment had only comments, ignore the blank source as + # already added in cell type markdown if line == '\n': - if segment_has_begun and source: + if source: # we've found text segments within the docstring if docstring: nb.add_cell(source, 'markdown') @@ -165,7 +162,7 @@ def python_to_notebook(example_file, notebook_dir, notebook_path): source = [] # if it's a comment elif line.strip().startswith('#'): - line = line.strip(' #') + line = line.lstrip(' #') nb.add_cell(line, 'markdown') elif line == '"""\n': if not docstring: diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 07b54470..128cd4f9 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -374,7 +374,7 @@ def write_example(src_name, src_dir, rst_dir, cfg): else: shutil.copy(cfg.plot2rst_default_thumb, thumb_path) - python_to_notebook(example_file, notebook_dir, notebook_path) + python_to_notebook(example_file, notebook_path) def save_thumbnail(image, thumb_path, shape): From 7b1ec0ae9603adab926b2da01e00e16646b03aff Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 16 Apr 2014 14:58:40 +0200 Subject: [PATCH 32/36] Rename _remove_consecutive_duplicates to _squash_repeats --- doc/ext/notebook.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 36c6fa1f..a306f11d 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -30,8 +30,8 @@ sample = """{ }""" -def _remove_consecutive_duplicates(x): - """Remove duplicates of elements appearing consecutively. +def _squash_repeats(x): + """Reduce repeating elements to a single occurrance. Parameters ---------- @@ -40,19 +40,17 @@ def _remove_consecutive_duplicates(x): Returns ------- - modified_x : list - Output list, with no consecutive duplicates. + list + A copy of `x` with repeating elements squashed. Examples -------- >>> input = [1, 2, 3, 3, 4, 5, 6, 6] - >>> output = remove_consecutive_duplicates(input) - >>> output + >>> print _squash_repeats(input) [1, 2, 3, 4, 5, 6] """ - modified_x = [x[0]] + [x[i] for i in range(1, len(x)) if x[i] != x[i-1]] - return modified_x + return [x[0]] + [x[i] for i in range(1, len(x)) if x[i] != x[i-1]] class Notebook(): @@ -146,7 +144,7 @@ def python_to_notebook(example_file, notebook_path): docstring = False source = [] - code = _remove_consecutive_duplicates(nb.code) + code = _squash_repeats(nb.code) for line in code: # A linebreak indicates a segment has ended. From a9107bbd5c70e23b18e58bf00417da5495f3be77 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 16 Apr 2014 15:01:41 +0200 Subject: [PATCH 33/36] Update Notebook docstring --- doc/ext/notebook.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index a306f11d..3827eecf 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -53,11 +53,9 @@ def _squash_repeats(x): return [x[0]] + [x[i] for i in range(1, len(x)) if x[i] != x[i-1]] -class Notebook(): +class Notebook(object): """ - Notebook object for generating an IPython notebook from an example Python - file. - + Notebook object for building an IPython notebook cell-by-cell. """ def __init__(self): @@ -91,10 +89,9 @@ class Notebook(): Parameters ---------- value : str - The actual content to be saved in the cell. + Cell content. cell_type : {'code', 'markdown'} - The type of content. - The default value will add a cell of type 'code'. + Type of content (default is 'code'). """ if cell_type in ['markdown', 'code']: @@ -104,15 +101,15 @@ class Notebook(): # assign value to the last cell cells[-1][key] = value else: - warnings.warn('Unsupported cell type %s, data ignored' % cell_type) + warnings.warn('Ignoring unsupported cell type (%s)' % cell_type) def json(self): - """Dump the template JSON to string. + """Return a JSON representation of the notebook. Returns ------- str - The template JSON converted to a string with a two char indent. + JSON notebook. """ return json.dumps(self.template, indent=2) From bbd7817eadc51b5151f26415026a9ac34b001ed0 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 16 Apr 2014 15:03:19 +0200 Subject: [PATCH 34/36] Add unit tests --- doc/ext/notebook.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index 3827eecf..e2f7d62b 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -4,6 +4,7 @@ import json import copy import warnings + sample = """{ "metadata": { "name":"" @@ -176,3 +177,12 @@ def python_to_notebook(example_file, notebook_path): with open(notebook_path, 'w') as output: output.write(nb.json()) + + +def test_foo(): + assert 1==1 + + +if __name__ == "__main__": + import numpy.testing as npt + npt.run_module_suite() From c597fa47d46508fa135a24374f70721b21f17229 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 16 Apr 2014 16:45:06 +0200 Subject: [PATCH 35/36] Add tests for notebook. Remove python_to_notebook utility function. --- doc/ext/notebook.py | 110 ++++++++++---------------------------------- 1 file changed, 24 insertions(+), 86 deletions(-) diff --git a/doc/ext/notebook.py b/doc/ext/notebook.py index e2f7d62b..054a9c44 100644 --- a/doc/ext/notebook.py +++ b/doc/ext/notebook.py @@ -5,7 +5,8 @@ import copy import warnings -sample = """{ +# Skeleton notebook in JSON format +skeleton_nb = """{ "metadata": { "name":"" }, @@ -31,29 +32,6 @@ sample = """{ }""" -def _squash_repeats(x): - """Reduce repeating elements to a single occurrance. - - Parameters - ---------- - x : list - Input list. - - Returns - ------- - list - A copy of `x` with repeating elements squashed. - - Examples - -------- - >>> input = [1, 2, 3, 3, 4, 5, 6, 6] - >>> print _squash_repeats(input) - [1, 2, 3, 4, 5, 6] - - """ - return [x[0]] + [x[i] for i in range(1, len(x)) if x[i] != x[i-1]] - - class Notebook(object): """ Notebook object for building an IPython notebook cell-by-cell. @@ -81,6 +59,7 @@ class Notebook(object): ] } + self.template = json.loads(skeleton_nb) self.cell_type = {'input': self.cell_code, 'source': self.cell_md} self.valuetype_to_celltype = {'code': 'input', 'markdown': 'source'} @@ -116,71 +95,30 @@ class Notebook(object): return json.dumps(self.template, indent=2) -def python_to_notebook(example_file, notebook_path): - """Convert a Python file to an IPython notebook. - - Parameters - ---------- - example_file : str - Path for source Python file. - notebook_path : str - Path for saving the notebook file (includes the filename). - - """ +def test_notebook_basic(): nb = Notebook() - with open(example_file, 'r') as pythonfile: - nb.template = json.loads(sample) - nb.code = pythonfile.readlines() - # Add an extra newline at the end, - # this aids in extraction of text segments - nb.code.append('\n') - - # Newline separated portions in example file, are sections. - # Code and markdown written together in such a section are further - # treated as different segments. Each cell has content from one - # segment. - docstring = False - source = [] - - code = _squash_repeats(nb.code) - - for line in code: - # A linebreak indicates a segment has ended. - # If the text segment had only comments, ignore the blank source as - # already added in cell type markdown - if line == '\n': - if source: - # we've found text segments within the docstring - if docstring: - nb.add_cell(source, 'markdown') - else: - nb.add_cell(source, 'code') - source = [] - # if it's a comment - elif line.strip().startswith('#'): - line = line.lstrip(' #') - nb.add_cell(line, 'markdown') - elif line == '"""\n': - if not docstring: - docstring = True - # Indicates, completion of docstring - # add whatever in source to markdown (cell type markdown) - elif docstring: - docstring = False - # Write leftover docstring if any left - if source: - nb.add_cell(source, 'markdown') - source = [] - else: - # some text segment is continuing, so add to source - source.append(line) - - with open(notebook_path, 'w') as output: - output.write(nb.json()) + assert(json.loads(nb.json()) == json.loads(skeleton_nb)) -def test_foo(): - assert 1==1 +def test_notebook_add(): + nb = Notebook() + + str1 = 'hello world' + str2 = 'f = lambda x: x * x' + + nb.add_cell(str1, cell_type='markdown') + nb.add_cell(str2, cell_type='code') + + d = json.loads(nb.json()) + cells = d['worksheets'][0]['cells'] + values = [c['input'] if c['cell_type'] == 'code' else c['source'] + for c in cells] + + assert values[1] == str1 + assert values[2] == str2 + + assert cells[1]['cell_type'] == 'markdown' + assert cells[2]['cell_type'] == 'code' if __name__ == "__main__": From 5dc647c860fc3312738e4f53db654dcba90611cb Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 16 Apr 2014 16:45:25 +0200 Subject: [PATCH 36/36] Update plot2rst to generate notebooks --- doc/ext/plot2rst.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/doc/ext/plot2rst.py b/doc/ext/plot2rst.py index 128cd4f9..258a6929 100644 --- a/doc/ext/plot2rst.py +++ b/doc/ext/plot2rst.py @@ -80,7 +80,9 @@ from skimage import io from skimage import transform from skimage.util.dtype import dtype_range -from notebook import python_to_notebook +from notebook import Notebook + +from docutils.core import publish_parts LITERALINCLUDE = """ @@ -326,7 +328,8 @@ def write_example(src_name, src_dir, rst_dir, cfg): rst_path = rst_dir.pjoin(basename + cfg.source_suffix) notebook_path = notebook_dir.pjoin(basename + '.ipynb') - if _plots_are_current(src_path, image_path) and rst_path.exists and notebook_path.exists: + if _plots_are_current(src_path, image_path) and rst_path.exists and \ + notebook_path.exists: return blocks = split_code_and_text_blocks(example_file) @@ -374,7 +377,23 @@ def write_example(src_name, src_dir, rst_dir, cfg): else: shutil.copy(cfg.plot2rst_default_thumb, thumb_path) - python_to_notebook(example_file, notebook_path) + # Export example to IPython notebook + nb = Notebook() + + for (cell_type, _, content) in blocks: + content = content.rstrip('\n') + + if cell_type == 'code': + nb.add_cell(content, cell_type='code') + else: + content = content.replace('"""', '') + content = '\n'.join([line for line in content.split('\n') if + not line.startswith('.. image')]) + html = publish_parts(content, writer_name='html')['html_body'] + nb.add_cell(html, cell_type='markdown') + + with open(notebook_path, 'w') as f: + f.write(nb.json()) def save_thumbnail(image, thumb_path, shape):