From d58917a7982a9adfaf216f0f505270eebd9b3331 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sat, 19 Aug 2017 17:39:25 +0200 Subject: [PATCH 01/21] Changed regex parsing in embedhtml.py to XML parsing with lxml, regex can catch all sorts of stupid things --- setup.py | 1 + .../nbconvert_support/embedhtml.py | 39 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 49c7210..61fa165 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ if you encounter any problems, and create a new issue if needed! 'pyyaml', 'tornado', 'traitlets >=4.1', + 'lxml>=3.8.0' ], extras_require={ 'test': [ diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 4e16255..e34bdb3 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -2,6 +2,7 @@ import base64 import re +import lxml.etree as ET from nbconvert.exporters.html import HTMLExporter from ipython_genutils.ipstruct import Struct import os @@ -24,15 +25,18 @@ class EmbedHTMLExporter(HTMLExporter): jupyter nbconvert --to html_embed mynotebook.ipynb """ - def replfunc(self, match): + def replfunc(self, node): """Replace source url or file link with base64 encoded blob.""" - url = match.group(1) + url = node.attrib["src"] imgformat = url.split('.')[-1] + + if url.startswith('data'): + return #Already in base64 Format + + self.log.info("try embedding url: %s, format: %s" % (url, imgformat)) + if url.startswith('http'): data = urlopen(url).read() - elif url.startswith('data'): - img = ' Date: Sat, 19 Aug 2017 17:44:24 +0200 Subject: [PATCH 02/21] some manicure --- .../nbconvert_support/embedhtml.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index e34bdb3..062f973 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -24,7 +24,7 @@ class EmbedHTMLExporter(HTMLExporter): jupyter nbconvert --to html_embed mynotebook.ipynb """ - + def replfunc(self, node): """Replace source url or file link with base64 encoded blob.""" url = node.attrib["src"] @@ -63,24 +63,27 @@ class EmbedHTMLExporter(HTMLExporter): prefix = "data:image/" + imgformat + ';base64,' node.attrib["src"] = prefix + b64_data - + def from_notebook_node(self, nb, resources=None, **kw): output, resources = super( EmbedHTMLExporter, self).from_notebook_node(nb, resources) self.path = resources['metadata']['path'] - self.log.info("path: %s" % self.path) - + + #Get attachements self.attachments = Struct() for cell in nb.cells: if 'attachments' in cell.keys(): self.attachments += cell['attachments'] + + # Parse HTML and replace tags with the embedded data parser = ET.HTMLParser() root = ET.fromstring(output, parser = parser) nodes = root.findall(".//img") for n in nodes: self.replfunc(n) + # Convert back to HTML embedded_output = ET.tostring(root, method="html") return embedded_output, resources From b5477be9a283b80c76d31e7dfc3e4a3fa0375d6c Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sun, 20 Aug 2017 20:22:08 +0200 Subject: [PATCH 03/21] typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 61fa165..e4c0f01 100755 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ if you encounter any problems, and create a new issue if needed! 'pyyaml', 'tornado', 'traitlets >=4.1', - 'lxml>=3.8.0' + 'lxml >=3.8.0' ], extras_require={ 'test': [ From b5322b75c3c12c0583d8cfc325b2bc7e3fa75962 Mon Sep 17 00:00:00 2001 From: Colton R Bukowsky Date: Mon, 21 Aug 2017 17:51:30 -0700 Subject: [PATCH 04/21] Fixed some typos and PEP8 lint flags --- .../nbconvert_support/embedhtml.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 062f973..5c9b7a2 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -1,8 +1,7 @@ """Embed graphics into HTML Exporter class""" import base64 -import re -import lxml.etree as ET +import lxml.etree as et from nbconvert.exporters.html import HTMLExporter from ipython_genutils.ipstruct import Struct import os @@ -31,7 +30,7 @@ class EmbedHTMLExporter(HTMLExporter): imgformat = url.split('.')[-1] if url.startswith('data'): - return #Already in base64 Format + return # Already in base64 Format self.log.info("try embedding url: %s, format: %s" % (url, imgformat)) @@ -53,12 +52,11 @@ class EmbedHTMLExporter(HTMLExporter): with open(filename, 'rb') as f: data = f.read() - b64_data = base64.b64encode(data).decode("utf-8") if imgformat == "svg": prefix = "data:image/svg+xml;base64," elif imgformat == "pdf": - prefix= "data:application/pdf;base64," + prefix = "data:application/pdf;base64," else: prefix = "data:image/" + imgformat + ';base64,' @@ -70,20 +68,20 @@ class EmbedHTMLExporter(HTMLExporter): self.path = resources['metadata']['path'] - #Get attachements + # Get attachments self.attachments = Struct() for cell in nb.cells: if 'attachments' in cell.keys(): self.attachments += cell['attachments'] # Parse HTML and replace tags with the embedded data - parser = ET.HTMLParser() - root = ET.fromstring(output, parser = parser) + parser = et.HTMLParser() + root = et.fromstring(output, parser=parser) nodes = root.findall(".//img") for n in nodes: self.replfunc(n) # Convert back to HTML - embedded_output = ET.tostring(root, method="html") + embedded_output = et.tostring(root, method="html") return embedded_output, resources From 449058c7abdcef85c0d01774874149c022e53991 Mon Sep 17 00:00:00 2001 From: Colton R Bukowsky Date: Mon, 21 Aug 2017 17:52:13 -0700 Subject: [PATCH 05/21] Added a preprocessor for embedding html into cells. --- .../nbconvert_support/pre_embedhtml.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py new file mode 100644 index 0000000..5687277 --- /dev/null +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +"""Nbconvert preprocessor for the embedding img sources into the cells.""" + +import re + +from nbconvert.preprocessors import Preprocessor +from .embedhtml import EmbedHTMLExporter, et + + +class PyMarkdownPreprocessor(Preprocessor, EmbedHTMLExporter): + """ + + :mod:`nbconvert` Preprocessor which embeds graphics as base64 into markdown cells. + + This :class:`~nbconvert.preprocessors.Preprocessor` replaces kernel code in + markdown cells with the results stored in the cell metadata. + """ + + def preprocess_cell(self, cell, resources, index): + """ + Preprocess cell + + Parameters + ---------- + cell : NotebookNode cell + Notebook cell being processed + resources : dictionary + Additional resources used in the conversion process. Allows + preprocessors to pass variables into the Jinja engine. + index : int + Index of the cell being processed (see base.py) + """ + + self.path = resources['metadata']['path'] + if cell.cell_type == "markdown": + if 'attachments' in cell.keys(): + self.attachments += cell['attachments'] + # Parse HTML and replace tags with the embedded data + parser = et.HTMLParser() + root = et.fromstring(cell, parser=parser) + nodes = root.findall(".//img") + for n in nodes: + # replfunc comes from the EmbedHTMLExporter class, and is all that is really needed from there + self.replfunc(n) + + # Convert back to HTML + embedded_output = et.tostring(root, method="html") + + return embedded_output, resources + else: + return cell, resources From b5b383797ab3bac445987f2bea4ef86e50591668 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Wed, 23 Aug 2017 20:56:21 +0200 Subject: [PATCH 06/21] linting errors and manicure --- .../nbconvert_support/embedhtml.py | 15 ++++++++------- .../nbconvert_support/pre_embedhtml.py | 10 +++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 5c9b7a2..d1c9726 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -23,15 +23,15 @@ class EmbedHTMLExporter(HTMLExporter): jupyter nbconvert --to html_embed mynotebook.ipynb """ - + def replfunc(self, node): """Replace source url or file link with base64 encoded blob.""" url = node.attrib["src"] imgformat = url.split('.')[-1] - + if url.startswith('data'): return # Already in base64 Format - + self.log.info("try embedding url: %s, format: %s" % (url, imgformat)) if url.startswith('http'): @@ -46,7 +46,8 @@ class EmbedHTMLExporter(HTMLExporter): img = ' Date: Thu, 24 Aug 2017 22:46:06 +0200 Subject: [PATCH 07/21] preserve tags in all cases --- .../nbconvert_support/embedhtml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index d1c9726..45f4a79 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -43,9 +43,9 @@ class EmbedHTMLExporter(HTMLExporter): for imgformat in self.config.NbConvertBase.display_data_priority: if imgformat in available_formats.keys(): b64_data = self.attachments[imgname][imgformat] - img = ' Date: Thu, 31 Aug 2017 08:21:51 +0200 Subject: [PATCH 08/21] import reordering --- src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 45f4a79..3a2bdef 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -1,10 +1,10 @@ """Embed graphics into HTML Exporter class""" +import os import base64 import lxml.etree as et from nbconvert.exporters.html import HTMLExporter from ipython_genutils.ipstruct import Struct -import os try: from urllib.request import urlopen # py3 From 0adb6a200f9387a78e6af0ed91e1b3ab3f459cb4 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Mon, 18 Sep 2017 22:04:09 +0200 Subject: [PATCH 09/21] Review Changes: lint, isort, conda and pre_embedhtml.py removed --- conda.recipe/meta.yaml | 1 + .../nbconvert_support/collapsible_headings.py | 1 + .../nbconvert_support/embedhtml.py | 6 +-- .../nbconvert_support/pre_embedhtml.py | 51 ------------------- 4 files changed, 5 insertions(+), 54 deletions(-) delete mode 100644 src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 69ef93d..fdbc1a8 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -33,6 +33,7 @@ requirements: - setuptools - tornado - traitlets >=4.1 + - lxml >=3.8.0 test: imports: diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/collapsible_headings.py b/src/jupyter_contrib_nbextensions/nbconvert_support/collapsible_headings.py index c8c0973..e9e6880 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/collapsible_headings.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/collapsible_headings.py @@ -6,6 +6,7 @@ import os from notebook.services.config import ConfigManager from jupyter_contrib_nbextensions import __file__ as contrib_init + from .exporter_inliner import ExporterInliner diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 3a2bdef..eed7dff 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -1,17 +1,17 @@ """Embed graphics into HTML Exporter class""" -import os import base64 +import os + import lxml.etree as et -from nbconvert.exporters.html import HTMLExporter from ipython_genutils.ipstruct import Struct +from nbconvert.exporters.html import HTMLExporter try: from urllib.request import urlopen # py3 except ImportError: from urllib2 import urlopen - class EmbedHTMLExporter(HTMLExporter): """ :mod:`nbconvert` Exporter which embeds graphics as base64 into html. diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py deleted file mode 100644 index 99578f8..0000000 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/pre_embedhtml.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -"""Nbconvert preprocessor for the embedding img sources into the cells.""" - -from nbconvert.preprocessors import Preprocessor -from .embedhtml import EmbedHTMLExporter, et - - -class PyMarkdownPreprocessor(Preprocessor, EmbedHTMLExporter): - """ - :mod:`nbconvert` Preprocessor which embeds graphics as base64 into markdown - cells. - - This :class:`~nbconvert.preprocessors.Preprocessor` replaces kernel code in - markdown cells with the results stored in the cell metadata. - """ - - def preprocess_cell(self, cell, resources, index): - """ - Preprocess cell - - Parameters - ---------- - cell : NotebookNode cell - Notebook cell being processed - resources : dictionary - Additional resources used in the conversion process. Allows - preprocessors to pass variables into the Jinja engine. - index : int - Index of the cell being processed (see base.py) - """ - - self.path = resources['metadata']['path'] - if cell.cell_type == "markdown": - if 'attachments' in cell.keys(): - self.attachments += cell['attachments'] - # Parse HTML and replace tags with the embedded data - parser = et.HTMLParser() - root = et.fromstring(cell, parser=parser) - nodes = root.findall(".//img") - - for n in nodes: - # replfunc comes from the EmbedHTMLExporter class, and is all - # that is really needed from there - self.replfunc(n) - - # Convert back to HTML - embedded_output = et.tostring(root, method="html") - - return embedded_output, resources - else: - return cell, resources From 7ded79f0b13dda81347b85e4d9e974064487d6b4 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Mon, 18 Sep 2017 22:50:19 +0200 Subject: [PATCH 10/21] make lint happy --- src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index eed7dff..79739e9 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -12,6 +12,7 @@ try: except ImportError: from urllib2 import urlopen + class EmbedHTMLExporter(HTMLExporter): """ :mod:`nbconvert` Exporter which embeds graphics as base64 into html. From d248502ae52514554bb81bad039c0e57774a3a49 Mon Sep 17 00:00:00 2001 From: Josh Barnes Date: Tue, 19 Sep 2017 01:14:58 +0100 Subject: [PATCH 11/21] force pip to install lxml from binary instead of source on appveyor [skip travis] --- appveyor.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b6f3cff..d371907 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,6 +8,10 @@ cache: environment: COVERALLS_REPO_TOKEN: secure: lFyaxdbvCvXKM+PjmN9FToU8DhsdS474RgaW/bNAu4IBnn7QbfZzDYrjKw33V6Oo + global: + # lxml will not build appropriately from source on Windows without the + # appropriate libxml headers. As a result, make pip use binary packages. + PIP_ONLY_BINARY: lxml matrix: - TOXENV: 'py27-notebook' @@ -39,13 +43,13 @@ environment: PYTHON_HOME: C:\Python34 PYTHON_VERSION: '3.4' PYTHON_ARCH: '32' - + - TOXENV: 'py34-notebook43' TOXPYTHON: C:\Python34\python.exe PYTHON_HOME: C:\Python34 PYTHON_VERSION: '3.4' PYTHON_ARCH: '32' - + - TOXENV: 'py34-notebook44' TOXPYTHON: C:\Python34\python.exe PYTHON_HOME: C:\Python34 From b46725929179702d242043e688f799d8de25f38f Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Tue, 19 Sep 2017 19:49:56 +0200 Subject: [PATCH 12/21] Better code logic --- .../nbconvert_support/embedhtml.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index 79739e9..e64c816 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -29,6 +29,8 @@ class EmbedHTMLExporter(HTMLExporter): """Replace source url or file link with base64 encoded blob.""" url = node.attrib["src"] imgformat = url.split('.')[-1] + b64_data = None + prefix = None if url.startswith('data'): return # Already in base64 Format @@ -36,7 +38,7 @@ class EmbedHTMLExporter(HTMLExporter): self.log.info("try embedding url: %s, format: %s" % (url, imgformat)) if url.startswith('http'): - data = urlopen(url).read() + b64_data = base64.b64encode(urlopen(url).read()).decode("utf-8") elif url.startswith('attachment'): imgname = url.split(':')[1] available_formats = self.attachments[imgname] @@ -44,23 +46,21 @@ class EmbedHTMLExporter(HTMLExporter): for imgformat in self.config.NbConvertBase.display_data_priority: if imgformat in available_formats.keys(): b64_data = self.attachments[imgname][imgformat] - node.attrib["src"] = "data:%s;base64," % imgformat \ - + b64_data - return + prefix = "data:%s;base64," % imgformat raise ValueError("""Could not find attachment for image '%s' in notebook""" % imgname) else: filename = os.path.join(self.path, url) with open(filename, 'rb') as f: - data = f.read() + b64_data = base64.b64encode(f.read()).decode("utf-8") - b64_data = base64.b64encode(data).decode("utf-8") - if imgformat == "svg": - prefix = "data:image/svg+xml;base64," - elif imgformat == "pdf": - prefix = "data:application/pdf;base64," - else: - prefix = "data:image/" + imgformat + ';base64,' + if prefix is None: + if imgformat == "svg": + prefix = "data:image/svg+xml;base64," + elif imgformat == "pdf": + prefix = "data:application/pdf;base64," + else: + prefix = "data:image/" + imgformat + ';base64,' node.attrib["src"] = prefix + b64_data From 545286864fc46df73887a809a1c45ba933b51d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Hasch?= Date: Sat, 23 Sep 2017 18:41:48 +0200 Subject: [PATCH 13/21] cherry-pick: Calculate filesize without CR in test --- tests/test_exporters.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8c3816a..9aae1d8 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -22,6 +22,14 @@ def _with_tmp_cwd(func): return func_wrapper +def _filesize_without_cr(name): + """Calculate file size without additional CR (Windows) """ + with io.open(name, 'r', encoding='utf-8') as f: + data = f.read().replace('\r', '') + size = len(data) + return size + + class TestNbConvertExporters(TestsBase): def check_stuff_gets_embedded(self, nb, exporter_name, to_be_included=[]): @@ -31,19 +39,19 @@ class TestNbConvertExporters(TestsBase): write(nb, f, 4) # convert with default exporter - self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) + (stdout, stderr) = self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) + print(stdout) nb_dst_filename = nb_basename + '.html' assert os.path.isfile(nb_dst_filename) - statinfo = os.stat(nb_dst_filename) - + filesize = _filesize_without_cr(nb_dst_filename) os.remove(nb_dst_filename) # convert with embedding exporter - self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) - statinfo_e = os.stat(nb_dst_filename) + (stdout, stderr) = self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) + print(stdout) + filesize_e = _filesize_without_cr(nb_dst_filename) assert os.path.isfile(nb_dst_filename) - - assert statinfo_e.st_size > statinfo.st_size + assert filesize_e > filesize with io.open(nb_dst_filename, 'r', encoding='utf-8') as f: embedded_nb = f.read() From 156967a6ec249fd31074eba5146cd25c20e72b04 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sat, 23 Sep 2017 19:12:11 +0200 Subject: [PATCH 14/21] dont loose HTML DOCTYPE, and unicode lxml export --- .../nbconvert_support/embedhtml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index e64c816..fcd29c2 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -84,6 +84,8 @@ class EmbedHTMLExporter(HTMLExporter): self.replfunc(n) # Convert back to HTML - embedded_output = et.tostring(root, method="html") + embedded_output = et.tostring(root.getroottree(), + method="html", + encoding='unicode') return embedded_output, resources From 3a04bc6c9f1c3bf0685f49c78784784622288283 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sat, 23 Sep 2017 19:21:40 +0200 Subject: [PATCH 15/21] remove print in tests --- tests/test_exporters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 9aae1d8..dc76fe5 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -40,7 +40,6 @@ class TestNbConvertExporters(TestsBase): # convert with default exporter (stdout, stderr) = self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) - print(stdout) nb_dst_filename = nb_basename + '.html' assert os.path.isfile(nb_dst_filename) filesize = _filesize_without_cr(nb_dst_filename) @@ -48,7 +47,6 @@ class TestNbConvertExporters(TestsBase): # convert with embedding exporter (stdout, stderr) = self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) - print(stdout) filesize_e = _filesize_without_cr(nb_dst_filename) assert os.path.isfile(nb_dst_filename) assert filesize_e > filesize From ac62cab0b77a17121b4dff55526219bfb7e300c5 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sat, 23 Sep 2017 19:23:05 +0200 Subject: [PATCH 16/21] remove returns stdout/stderr --- tests/test_exporters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index dc76fe5..86949a1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -39,14 +39,14 @@ class TestNbConvertExporters(TestsBase): write(nb, f, 4) # convert with default exporter - (stdout, stderr) = self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) + self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) nb_dst_filename = nb_basename + '.html' assert os.path.isfile(nb_dst_filename) filesize = _filesize_without_cr(nb_dst_filename) os.remove(nb_dst_filename) # convert with embedding exporter - (stdout, stderr) = self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) + self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) filesize_e = _filesize_without_cr(nb_dst_filename) assert os.path.isfile(nb_dst_filename) assert filesize_e > filesize From 677e90229cedba507b207137fad24f2167dde0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Hasch?= Date: Sat, 23 Sep 2017 18:41:48 +0200 Subject: [PATCH 17/21] test appveyor ..., python3 works, whats wrong with 2.7? --- tests/test_exporters.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 86949a1..2aec992 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -25,6 +25,7 @@ def _with_tmp_cwd(func): def _filesize_without_cr(name): """Calculate file size without additional CR (Windows) """ with io.open(name, 'r', encoding='utf-8') as f: + print(data.encode("utf-8")) data = f.read().replace('\r', '') size = len(data) return size @@ -39,14 +40,16 @@ class TestNbConvertExporters(TestsBase): write(nb, f, 4) # convert with default exporter - self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) + (stdout, stderr) = self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) + print(stdout) nb_dst_filename = nb_basename + '.html' assert os.path.isfile(nb_dst_filename) filesize = _filesize_without_cr(nb_dst_filename) os.remove(nb_dst_filename) # convert with embedding exporter - self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) + (stdout, stderr) = self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) + print(stdout) filesize_e = _filesize_without_cr(nb_dst_filename) assert os.path.isfile(nb_dst_filename) assert filesize_e > filesize From 3279e77a80d96e9355616cadbbcccce9439cd036 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sun, 24 Sep 2017 00:05:20 +0200 Subject: [PATCH 18/21] rewrote tests, -> check_html with functor --- tests/test_exporters.py | 61 ++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 2aec992..7589977 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,7 +7,7 @@ from functools import wraps from nbconvert.tests.base import TestsBase from nbformat import v4, write - +from lxml import etree as et def path_in_data(rel_path): """Return an absolute path from a relative path in tests/data.""" @@ -21,44 +21,23 @@ def _with_tmp_cwd(func): return func(self, *args, **kwargs) return func_wrapper - -def _filesize_without_cr(name): - """Calculate file size without additional CR (Windows) """ - with io.open(name, 'r', encoding='utf-8') as f: - print(data.encode("utf-8")) - data = f.read().replace('\r', '') - size = len(data) - return size - - class TestNbConvertExporters(TestsBase): - def check_stuff_gets_embedded(self, nb, exporter_name, to_be_included=[]): + def check_html(self, nb, exporter_name, check_func): nb_basename = 'notebook' nb_src_filename = nb_basename + '.ipynb' with io.open(nb_src_filename, 'w', encoding='utf-8') as f: write(nb, f, 4) - # convert with default exporter - (stdout, stderr) = self.nbconvert('--to {} "{}"'.format('html', nb_src_filename)) - print(stdout) - nb_dst_filename = nb_basename + '.html' - assert os.path.isfile(nb_dst_filename) - filesize = _filesize_without_cr(nb_dst_filename) - os.remove(nb_dst_filename) - # convert with embedding exporter - (stdout, stderr) = self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) - print(stdout) - filesize_e = _filesize_without_cr(nb_dst_filename) - assert os.path.isfile(nb_dst_filename) - assert filesize_e > filesize + nb_dst_filename = nb_basename + '.html' + self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename)) - with io.open(nb_dst_filename, 'r', encoding='utf-8') as f: + with open(nb_dst_filename, 'rb') as f: embedded_nb = f.read() - - for txt in to_be_included: - assert txt in embedded_nb + parser = et.HTMLParser() + root = et.fromstring(embedded_nb, parser=parser) + check_func(byte_string=embedded_nb, root_node=root) @_with_tmp_cwd def test_embedhtml(self): @@ -69,8 +48,14 @@ class TestNbConvertExporters(TestsBase): source="![testimage]({})".format(path_in_data('icon.png')) ), ]) - self.check_stuff_gets_embedded( - nb, 'html_embed', to_be_included=['base64']) + + def check(byte_string, root_node): + nodes = root_node.findall(".//img") + for n in nodes: + url = n.attrib["src"] + assert url.startswith('data') + + self.check_html(nb, 'html_embed', check_func=check) @_with_tmp_cwd def test_htmltoc2(self): @@ -79,8 +64,11 @@ class TestNbConvertExporters(TestsBase): v4.new_code_cell(source="a = 'world'"), v4.new_markdown_cell(source="# Heading"), ]) - self.check_stuff_gets_embedded( - nb, 'html_toc', to_be_included=['toc2']) + + def check(byte_string, root_node): + assert b'toc2' in byte_string + + self.check_html(nb, 'html_toc', check_func=check) @_with_tmp_cwd def test_html_collapsible_headings(self): @@ -93,5 +81,8 @@ class TestNbConvertExporters(TestsBase): v4.new_markdown_cell(source=('### level 3 heading')), v4.new_code_cell(source='a = range(1,10)'), ]) - self.check_stuff_gets_embedded( - nb, 'html_ch', to_be_included=['collapsible_headings']) + + def check(byte_string, root_node): + assert b'collapsible_headings' in byte_string + + self.check_html(nb, 'html_ch', check_func=check) From 6fb863f6fa439b649c10294ae63661a80e100f94 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sun, 24 Sep 2017 00:07:26 +0200 Subject: [PATCH 19/21] make lint happy --- tests/test_exporters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 7589977..e35bc43 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -9,6 +9,7 @@ from nbconvert.tests.base import TestsBase from nbformat import v4, write from lxml import etree as et + def path_in_data(rel_path): """Return an absolute path from a relative path in tests/data.""" return os.path.join(os.path.dirname(__file__), 'data', rel_path) @@ -21,6 +22,7 @@ def _with_tmp_cwd(func): return func(self, *args, **kwargs) return func_wrapper + class TestNbConvertExporters(TestsBase): def check_html(self, nb, exporter_name, check_func): From bb4040e68bfdb828b601c59424e37af9ba1a36b4 Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Sun, 24 Sep 2017 15:26:52 +0200 Subject: [PATCH 20/21] make isort happy --- tests/test_exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index e35bc43..7afcea9 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -5,9 +5,9 @@ import io import os from functools import wraps +from lxml import etree as et from nbconvert.tests.base import TestsBase from nbformat import v4, write -from lxml import etree as et def path_in_data(rel_path): From 27e58f8fb28685a087dd88b9bb17bebd7b4c89af Mon Sep 17 00:00:00 2001 From: Gabriel Nuetzi Date: Tue, 26 Sep 2017 21:17:50 +0200 Subject: [PATCH 21/21] bugfix --- .../nbconvert_support/embedhtml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py index fcd29c2..ca5fb0e 100644 --- a/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py +++ b/src/jupyter_contrib_nbextensions/nbconvert_support/embedhtml.py @@ -47,8 +47,9 @@ class EmbedHTMLExporter(HTMLExporter): if imgformat in available_formats.keys(): b64_data = self.attachments[imgname][imgformat] prefix = "data:%s;base64," % imgformat - raise ValueError("""Could not find attachment for image '%s' - in notebook""" % imgname) + if b64_data is None: + raise ValueError("""Could not find attachment for image '%s' + in notebook""" % imgname) else: filename = os.path.join(self.path, url) with open(filename, 'rb') as f: