Included sphinx-gallery files

I had to make a slightly modification in one of sphinx-gallery's files to
make it work with scikit-image. I opened a pull request on sphinx-gallery
github, hopefully we can delete the local copy after their next release.
This commit is contained in:
emmanuelle
2016-05-10 21:49:15 +02:00
parent 844858f01b
commit 52b0e16a78
10 changed files with 1795 additions and 0 deletions
+13
View File
@@ -0,0 +1,13 @@
"""
==============
Sphinx Gallery
==============
"""
import os
__version__ = '0.1.2'
def glr_path_static():
"""Returns path to packaged static files"""
return os.path.abspath(os.path.join(os.path.dirname(__file__), '_static'))
Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 52 KiB

+127
View File
@@ -0,0 +1,127 @@
/*
Sphinx-Gallery is has compatible CSS to fix default sphinx themes
Tested for Sphinx 1.3.1 for all themes: default, alabaster, sphinxdoc,
scrolls, agogo, traditional, nature, haiku, pyramid
Tested for Read the docs theme 0.1.7 */
.sphx-glr-thumbcontainer {
background: #fff;
border: solid #fff 1px;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
border-radius: 5px;
box-shadow: none;
float: left;
margin: 5px;
min-height: 230px;
padding-top: 5px;
position: relative;
}
.sphx-glr-thumbcontainer:hover {
border: solid #b4ddfc 1px;
box-shadow: 0 0 15px rgba(142, 176, 202, 0.5);
}
.sphx-glr-thumbcontainer a.internal {
bottom: 0;
display: block;
left: 0;
padding: 150px 10px 0;
position: absolute;
right: 0;
top: 0;
}
/* Next one is to avoid Sphinx traditional theme to cover all the
thumbnail with its default link Background color */
.sphx-glr-thumbcontainer a.internal:hover {
background-color: transparent;
}
.sphx-glr-thumbcontainer p {
margin: 0 0 .1em 0;
}
.sphx-glr-thumbcontainer .figure {
margin: 10px;
width: 160px;
}
.sphx-glr-thumbcontainer img {
display: inline;
max-height: 160px;
width: 160px;
}
.sphx-glr-thumbcontainer[tooltip]:hover:after {
background: rgba(0, 0, 0, 0.8);
-webkit-border-radius: 5px;
-moz-border-radius: 5px;
border-radius: 5px;
color: #fff;
content: attr(tooltip);
left: 95%;
padding: 5px 15px;
position: absolute;
z-index: 98;
width: 220px;
bottom: 52%;
}
.sphx-glr-thumbcontainer[tooltip]:hover:before {
border: solid;
border-color: #333 transparent;
border-width: 18px 0 0 20px;
bottom: 58%;
content: '';
left: 85%;
position: absolute;
z-index: 99;
}
.highlight-pytb pre {
background-color: #ffe4e4;
border: 1px solid #f66;
margin-top: 10px;
padding: 7px;
}
.sphx-glr-script-out {
color: #888;
margin: 0;
}
.sphx-glr-script-out .highlight {
background-color: transparent;
margin-left: 2.5em;
margin-top: -1.4em;
}
.sphx-glr-script-out .highlight pre {
background-color: #fafae2;
border: 0;
max-height: 30em;
overflow: auto;
padding-left: 1ex;
margin: 0px;
word-break: break-word;
}
.sphx-glr-script-out + p {
margin-top: 1.8em;
}
blockquote.sphx-glr-script-out {
margin-left: 0pt;
}
.sphx-glr-download {
background-color: #ffc;
border: 1px solid #c2c22d;
border-radius: 4px;
margin: 1em auto 1ex auto;
max-width: 45ex;
padding: 1ex;
}
.sphx-glr-download a {
color: #4b4600;
}
ul.sphx-glr-horizontal {
list-style: none;
padding: 0;
}
ul.sphx-glr-horizontal li {
display: inline;
}
ul.sphx-glr-horizontal img {
height: auto !important;
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

+180
View File
@@ -0,0 +1,180 @@
# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
========================
Backreferences Generator
========================
Reviews generated example files in order to keep track of used modules
"""
from __future__ import print_function
import ast
import os
# Try Python 2 first, otherwise load from Python 3
try:
import cPickle as pickle
except ImportError:
import pickle
class NameFinder(ast.NodeVisitor):
"""Finds the longest form of variable names and their imports in code
Only retains names from imported modules.
"""
def __init__(self):
super(NameFinder, self).__init__()
self.imported_names = {}
self.accessed_names = set()
def visit_Import(self, node, prefix=''):
for alias in node.names:
local_name = alias.asname or alias.name
self.imported_names[local_name] = prefix + alias.name
def visit_ImportFrom(self, node):
self.visit_Import(node, node.module + '.')
def visit_Name(self, node):
self.accessed_names.add(node.id)
def visit_Attribute(self, node):
attrs = []
while isinstance(node, ast.Attribute):
attrs.append(node.attr)
node = node.value
if isinstance(node, ast.Name):
# This is a.b, not e.g. a().b
attrs.append(node.id)
self.accessed_names.add('.'.join(reversed(attrs)))
else:
# need to get a in a().b
self.visit(node)
def get_mapping(self):
for name in self.accessed_names:
local_name = name.split('.', 1)[0]
remainder = name[len(local_name):]
if local_name in self.imported_names:
# Join import path to relative path
full_name = self.imported_names[local_name] + remainder
yield name, full_name
def get_short_module_name(module_name, obj_name):
""" Get the shortest possible module name """
parts = module_name.split('.')
short_name = module_name
for i in range(len(parts) - 1, 0, -1):
short_name = '.'.join(parts[:i])
try:
exec('from %s import %s' % (short_name, obj_name))
except Exception: # libraries can throw all sorts of exceptions...
# get the last working module name
short_name = '.'.join(parts[:(i + 1)])
break
return short_name
def identify_names(code):
"""Builds a codeobj summary by identifying and resolving used names
>>> code = '''
... from a.b import c
... import d as e
... print(c)
... e.HelloWorld().f.g
... '''
>>> for name, o in sorted(identify_names(code).items()):
... print(name, o['name'], o['module'], o['module_short'])
c c a.b a.b
e.HelloWorld HelloWorld d d
"""
finder = NameFinder()
finder.visit(ast.parse(code))
example_code_obj = {}
for name, full_name in finder.get_mapping():
# name is as written in file (e.g. np.asarray)
# full_name includes resolved import path (e.g. numpy.asarray)
module, attribute = full_name.rsplit('.', 1)
# get shortened module name
module_short = get_short_module_name(module, attribute)
cobj = {'name': attribute, 'module': module,
'module_short': module_short}
example_code_obj[name] = cobj
return example_code_obj
def scan_used_functions(example_file, gallery_conf):
"""save variables so we can later add links to the documentation"""
example_code_obj = identify_names(open(example_file).read())
if example_code_obj:
codeobj_fname = example_file[:-3] + '_codeobj.pickle'
with open(codeobj_fname, 'wb') as fid:
pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)
backrefs = set('{module_short}.{name}'.format(**entry)
for entry in example_code_obj.values()
if entry['module'].startswith(gallery_conf['doc_module']))
return backrefs
THUMBNAIL_TEMPLATE = """
.. raw:: html
<div class="sphx-glr-thumbcontainer" tooltip="{snippet}">
.. only:: html
.. figure:: /{thumbnail}
:ref:`sphx_glr_{ref_name}`
.. raw:: html
</div>
"""
BACKREF_THUMBNAIL_TEMPLATE = THUMBNAIL_TEMPLATE + """
.. only:: not html
* :ref:`sphx_glr_{ref_name}`
"""
def _thumbnail_div(full_dir, fname, snippet, is_backref=False):
"""Generates RST to place a thumbnail in a gallery"""
thumb = os.path.join(full_dir, 'images', 'thumb',
'sphx_glr_%s_thumb.png' % fname[:-3])
ref_name = os.path.join(full_dir, fname).replace(os.path.sep, '_')
template = BACKREF_THUMBNAIL_TEMPLATE if is_backref else THUMBNAIL_TEMPLATE
return template.format(snippet=snippet, thumbnail=thumb, ref_name=ref_name)
def write_backreferences(seen_backrefs, gallery_conf,
target_dir, fname, snippet):
"""Writes down back reference files, which include a thumbnail list
of examples using a certain module"""
example_file = os.path.join(target_dir, fname)
backrefs = scan_used_functions(example_file, gallery_conf)
for backref in backrefs:
include_path = os.path.join(gallery_conf['mod_example_dir'],
'%s.examples' % backref)
seen = backref in seen_backrefs
with open(include_path, 'a' if seen else 'w') as ex_file:
if not seen:
heading = '\n\nExamples using ``%s``' % backref
ex_file.write(heading + '\n')
ex_file.write('^' * len(heading) + '\n')
ex_file.write(_thumbnail_div(target_dir, fname, snippet,
is_backref=True))
seen_backrefs.add(backref)
+436
View File
@@ -0,0 +1,436 @@
# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
###############################################################################
# Documentation link resolver objects
from __future__ import print_function
import gzip
import os
import posixpath
import re
import shelve
import sys
# Try Python 2 first, otherwise load from Python 3
try:
import cPickle as pickle
import urllib2 as urllib
from urllib2 import HTTPError, URLError
except ImportError:
import pickle
import urllib.request
import urllib.error
import urllib.parse
from urllib.error import HTTPError, URLError
from io import StringIO
def _get_data(url):
"""Helper function to get data over http or from a local file"""
if url.startswith('http://'):
# Try Python 2, use Python 3 on exception
try:
resp = urllib.urlopen(url)
encoding = resp.headers.dict.get('content-encoding', 'plain')
except AttributeError:
resp = urllib.request.urlopen(url)
encoding = resp.headers.get('content-encoding', 'plain')
data = resp.read()
if encoding == 'plain':
pass
elif encoding == 'gzip':
data = StringIO(data)
data = gzip.GzipFile(fileobj=data).read()
else:
raise RuntimeError('unknown encoding')
else:
with open(url, 'r') as fid:
data = fid.read()
return data
def get_data(url, gallery_dir):
"""Persistent dictionary usage to retrieve the search indexes"""
# shelve keys need to be str in python 2
if sys.version_info[0] == 2 and isinstance(url, unicode):
url = url.encode('utf-8')
cached_file = os.path.join(gallery_dir, 'searchindex')
search_index = shelve.open(cached_file)
if url in search_index:
data = search_index[url]
else:
data = _get_data(url)
search_index[url] = data
search_index.close()
return data
def _select_block(str_in, start_tag, end_tag):
"""Select first block delimited by start_tag and end_tag"""
start_pos = str_in.find(start_tag)
if start_pos < 0:
raise ValueError('start_tag not found')
depth = 0
for pos in range(start_pos, len(str_in)):
if str_in[pos] == start_tag:
depth += 1
elif str_in[pos] == end_tag:
depth -= 1
if depth == 0:
break
sel = str_in[start_pos + 1:pos]
return sel
def _parse_dict_recursive(dict_str):
"""Parse a dictionary from the search index"""
dict_out = dict()
pos_last = 0
pos = dict_str.find(':')
while pos >= 0:
key = dict_str[pos_last:pos]
if dict_str[pos + 1] == '[':
# value is a list
pos_tmp = dict_str.find(']', pos + 1)
if pos_tmp < 0:
raise RuntimeError('error when parsing dict')
value = dict_str[pos + 2: pos_tmp].split(',')
# try to convert elements to int
for i in range(len(value)):
try:
value[i] = int(value[i])
except ValueError:
pass
elif dict_str[pos + 1] == '{':
# value is another dictionary
subdict_str = _select_block(dict_str[pos:], '{', '}')
value = _parse_dict_recursive(subdict_str)
pos_tmp = pos + len(subdict_str)
else:
raise ValueError('error when parsing dict: unknown elem')
key = key.strip('"')
if len(key) > 0:
dict_out[key] = value
pos_last = dict_str.find(',', pos_tmp)
if pos_last < 0:
break
pos_last += 1
pos = dict_str.find(':', pos_last)
return dict_out
def parse_sphinx_searchindex(searchindex):
"""Parse a Sphinx search index
Parameters
----------
searchindex : str
The Sphinx search index (contents of searchindex.js)
Returns
-------
filenames : list of str
The file names parsed from the search index.
objects : dict
The objects parsed from the search index.
"""
# Make sure searchindex uses UTF-8 encoding
if hasattr(searchindex, 'decode'):
searchindex = searchindex.decode('UTF-8')
# parse objects
query = 'objects:'
pos = searchindex.find(query)
if pos < 0:
raise ValueError('"objects:" not found in search index')
sel = _select_block(searchindex[pos:], '{', '}')
objects = _parse_dict_recursive(sel)
# parse filenames
query = 'filenames:'
pos = searchindex.find(query)
if pos < 0:
raise ValueError('"filenames:" not found in search index')
filenames = searchindex[pos + len(query) + 1:]
filenames = filenames[:filenames.find(']')]
filenames = [f.strip('"') for f in filenames.split(',')]
return filenames, objects
class SphinxDocLinkResolver(object):
""" Resolve documentation links using searchindex.js generated by Sphinx
Parameters
----------
doc_url : str
The base URL of the project website.
searchindex : str
Filename of searchindex, relative to doc_url.
extra_modules_test : list of str
List of extra module names to test.
relative : bool
Return relative links (only useful for links to documentation of this
package).
"""
def __init__(self, doc_url, gallery_dir, searchindex='searchindex.js',
extra_modules_test=None, relative=False):
self.doc_url = doc_url
self.gallery_dir = gallery_dir
self.relative = relative
self._link_cache = {}
self.extra_modules_test = extra_modules_test
self._page_cache = {}
if doc_url.startswith('http://'):
if relative:
raise ValueError('Relative links are only supported for local '
'URLs (doc_url cannot start with "http://)"')
searchindex_url = doc_url + '/' + searchindex
else:
searchindex_url = os.path.join(doc_url, searchindex)
# detect if we are using relative links on a Windows system
if os.name.lower() == 'nt' and not doc_url.startswith('http://'):
if not relative:
raise ValueError('You have to use relative=True for the local'
' package on a Windows system.')
self._is_windows = True
else:
self._is_windows = False
# download and initialize the search index
sindex = get_data(searchindex_url, gallery_dir)
filenames, objects = parse_sphinx_searchindex(sindex)
self._searchindex = dict(filenames=filenames, objects=objects)
def _get_link(self, cobj):
"""Get a valid link, False if not found"""
fname_idx = None
full_name = cobj['module_short'] + '.' + cobj['name']
if full_name in self._searchindex['objects']:
value = self._searchindex['objects'][full_name]
if isinstance(value, dict):
value = value[next(iter(value.keys()))]
fname_idx = value[0]
elif cobj['module_short'] in self._searchindex['objects']:
value = self._searchindex['objects'][cobj['module_short']]
if cobj['name'] in value.keys():
fname_idx = value[cobj['name']][0]
if fname_idx is not None:
fname = self._searchindex['filenames'][fname_idx] + '.html'
if self._is_windows:
fname = fname.replace('/', '\\')
link = os.path.join(self.doc_url, fname)
else:
link = posixpath.join(self.doc_url, fname)
if hasattr(link, 'decode'):
link = link.decode('utf-8', 'replace')
if link in self._page_cache:
html = self._page_cache[link]
else:
html = get_data(link, self.gallery_dir)
self._page_cache[link] = html
# test if cobj appears in page
comb_names = [cobj['module_short'] + '.' + cobj['name']]
if self.extra_modules_test is not None:
for mod in self.extra_modules_test:
comb_names.append(mod + '.' + cobj['name'])
url = False
if hasattr(html, 'decode'):
# Decode bytes under Python 3
html = html.decode('utf-8', 'replace')
for comb_name in comb_names:
if hasattr(comb_name, 'decode'):
# Decode bytes under Python 3
comb_name = comb_name.decode('utf-8', 'replace')
if comb_name in html:
url = link + u'#' + comb_name
link = url
else:
link = False
return link
def resolve(self, cobj, this_url):
"""Resolve the link to the documentation, returns None if not found
Parameters
----------
cobj : dict
Dict with information about the "code object" for which we are
resolving a link.
cobi['name'] : function or class name (str)
cobj['module_short'] : shortened module name (str)
cobj['module'] : module name (str)
this_url: str
URL of the current page. Needed to construct relative URLs
(only used if relative=True in constructor).
Returns
-------
link : str | None
The link (URL) to the documentation.
"""
full_name = cobj['module_short'] + '.' + cobj['name']
link = self._link_cache.get(full_name, None)
if link is None:
# we don't have it cached
link = self._get_link(cobj)
# cache it for the future
self._link_cache[full_name] = link
if link is False or link is None:
# failed to resolve
return None
if self.relative:
link = os.path.relpath(link, start=this_url)
if self._is_windows:
# replace '\' with '/' so it on the web
link = link.replace('\\', '/')
# for some reason, the relative link goes one directory too high up
link = link[3:]
return link
def _embed_code_links(app, gallery_conf, gallery_dir):
# Add resolvers for the packages for which we want to show links
doc_resolvers = {}
for this_module, url in gallery_conf['reference_url'].items():
try:
if url is None:
doc_resolvers[this_module] = SphinxDocLinkResolver(
app.builder.outdir,
gallery_dir,
relative=True)
else:
doc_resolvers[this_module] = SphinxDocLinkResolver(url,
gallery_dir)
except HTTPError as e:
print("The following HTTP Error has occurred:\n")
print(e.code)
except URLError as e:
print("\n...\n"
"Warning: Embedding the documentation hyperlinks requires "
"Internet access.\nPlease check your network connection.\n"
"Unable to continue embedding `{0}` links due to a URL "
"Error:\n".format(this_module))
print(e.args)
html_gallery_dir = os.path.abspath(os.path.join(app.builder.outdir,
gallery_dir))
# patterns for replacement
link_pattern = '<a href="%s">%s</a>'
orig_pattern = '<span class="n">%s</span>'
period = '<span class="o">.</span>'
for dirpath, _, filenames in os.walk(html_gallery_dir):
for fname in filenames:
print('\tprocessing: %s' % fname)
full_fname = os.path.join(html_gallery_dir, dirpath, fname)
subpath = dirpath[len(html_gallery_dir) + 1:]
pickle_fname = os.path.join(gallery_dir, subpath,
fname[:-5] + '_codeobj.pickle')
if os.path.exists(pickle_fname):
# we have a pickle file with the objects to embed links for
with open(pickle_fname, 'rb') as fid:
example_code_obj = pickle.load(fid)
fid.close()
str_repl = {}
# generate replacement strings with the links
for name, cobj in example_code_obj.items():
this_module = cobj['module'].split('.')[0]
if this_module not in doc_resolvers:
continue
try:
link = doc_resolvers[this_module].resolve(cobj,
full_fname)
except (HTTPError, URLError) as e:
print("The following error has occurred:\n")
print(repr(e))
continue
if link is not None:
parts = name.split('.')
name_html = period.join(orig_pattern % part
for part in parts)
str_repl[name_html] = link_pattern % (link, name_html)
# do the replacement in the html file
# ensure greediness
names = sorted(str_repl, key=len, reverse=True)
expr = re.compile(r'(?<!\.)\b' + # don't follow . or word
'|'.join(re.escape(name)
for name in names))
def substitute_link(match):
return str_repl[match.group()]
if len(str_repl) > 0:
with open(full_fname, 'rb') as fid:
lines_in = fid.readlines()
with open(full_fname, 'wb') as fid:
for line in lines_in:
line = line.decode('utf-8')
line = expr.sub(substitute_link, line)
fid.write(line.encode('utf-8'))
print('[done]')
def embed_code_links(app, exception):
"""Embed hyperlinks to documentation into example code"""
if exception is not None:
return
# No need to waste time embedding hyperlinks when not running the examples
# XXX: also at the time of writing this fixes make html-noplot
# for some reason I don't fully understand
if not app.builder.config.plot_gallery:
return
# XXX: Whitelist of builders for which it makes sense to embed
# hyperlinks inside the example html. Note that the link embedding
# require searchindex.js to exist for the links to the local doc
# and there does not seem to be a good way of knowing which
# builders creates a searchindex.js.
if app.builder.name not in ['html', 'readthedocs']:
return
print('Embedding documentation hyperlinks in examples..')
gallery_conf = app.config.sphinx_gallery_conf
gallery_dirs = gallery_conf['gallery_dirs']
if not isinstance(gallery_dirs, list):
gallery_dirs = [gallery_dirs]
for gallery_dir in gallery_dirs:
_embed_code_links(app, gallery_conf, gallery_dir)
+170
View File
@@ -0,0 +1,170 @@
# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
========================
Sphinx-Gallery Generator
========================
Attaches Sphinx-Gallery to Sphinx in order to generate the galleries
when building the documentation.
"""
from __future__ import division, print_function, absolute_import
import re
import os
from . import glr_path_static
from .gen_rst import generate_dir_rst
from .docs_resolv import embed_code_links
def clean_gallery_out(build_dir):
"""Deletes images under the sphx_glr namespace in the build directory"""
# Sphinx hack: sphinx copies generated images to the build directory
# each time the docs are made. If the desired image name already
# exists, it appends a digit to prevent overwrites. The problem is,
# the directory is never cleared. This means that each time you build
# the docs, the number of images in the directory grows.
#
# This question has been asked on the sphinx development list, but there
# was no response: http://osdir.com/ml/sphinx-dev/2011-02/msg00123.html
#
# The following is a hack that prevents this behavior by clearing the
# image build directory from gallery images each time the docs are built.
# If sphinx changes their layout between versions, this will not
# work (though it should probably not cause a crash).
# Tested successfully on Sphinx 1.0.7
build_image_dir = os.path.join(build_dir, '_images')
if os.path.exists(build_image_dir):
filelist = os.listdir(build_image_dir)
for filename in filelist:
if filename.startswith('sphx_glr') and filename.endswith('png'):
os.remove(os.path.join(build_image_dir, filename))
def generate_gallery_rst(app):
"""Generate the Main examples gallery reStructuredText
Start the sphinx-gallery configuration and recursively scan the examples
directories in order to populate the examples gallery
"""
try:
plot_gallery = eval(app.builder.config.plot_gallery)
except TypeError:
plot_gallery = bool(app.builder.config.plot_gallery)
gallery_conf.update(app.config.sphinx_gallery_conf)
gallery_conf.update(plot_gallery=plot_gallery)
gallery_conf.update(abort_on_example_error=app.builder.config.abort_on_example_error)
# this assures I can call the config in other places
app.config.sphinx_gallery_conf = gallery_conf
app.config.html_static_path.append(glr_path_static())
clean_gallery_out(app.builder.outdir)
examples_dirs = gallery_conf['examples_dirs']
gallery_dirs = gallery_conf['gallery_dirs']
if not isinstance(examples_dirs, list):
examples_dirs = [examples_dirs]
if not isinstance(gallery_dirs, list):
gallery_dirs = [gallery_dirs]
mod_examples_dir = os.path.relpath(gallery_conf['mod_example_dir'],
app.builder.srcdir)
seen_backrefs = set()
computation_times = []
# cd to the appropriate directory regardless of sphinx configuration
working_dir = os.getcwd()
os.chdir(app.builder.srcdir)
for examples_dir, gallery_dir in zip(examples_dirs, gallery_dirs):
examples_dir = os.path.relpath(examples_dir,
app.builder.srcdir)
gallery_dir = os.path.relpath(gallery_dir,
app.builder.srcdir)
for workdir in [examples_dir, gallery_dir, mod_examples_dir]:
if not os.path.exists(workdir):
os.makedirs(workdir)
# we create an index.rst with all examples
fhindex = open(os.path.join(gallery_dir, 'index.rst'), 'w')
# Here we don't use an os.walk, but we recurse only twice: flat is
# better than nested.
this_fhindex, this_computation_times = \
generate_dir_rst(examples_dir, gallery_dir, gallery_conf,
seen_backrefs)
computation_times += this_computation_times
fhindex.write(this_fhindex)
for directory in sorted(os.listdir(examples_dir)):
if os.path.isdir(os.path.join(examples_dir, directory)):
src_dir = os.path.join(examples_dir, directory)
target_dir = os.path.join(gallery_dir, directory)
this_fhindex, this_computation_times = \
generate_dir_rst(src_dir, target_dir, gallery_conf,
seen_backrefs)
fhindex.write(this_fhindex)
computation_times += this_computation_times
fhindex.flush()
# Back to initial directory
os.chdir(working_dir)
print("Computation time summary:")
for time_elapsed, fname in sorted(computation_times)[::-1]:
if time_elapsed is not None:
print("\t- %s : %.2g sec" % (fname, time_elapsed))
else:
print("\t- %s : not run" % fname)
def touch_empty_backreferences(app, what, name, obj, options, lines):
"""Generate empty back-reference example files
This avoids inclusion errors/warnings if there are no gallery
examples for a class / module that is being parsed by autodoc"""
examples_path = os.path.join(app.srcdir,
app.config.sphinx_gallery_conf["mod_example_dir"],
"%s.examples" % name)
if not os.path.exists(examples_path):
# touch file
open(examples_path, 'w').close()
gallery_conf = {
'filename_pattern': re.escape(os.sep) + 'plot',
'examples_dirs': '../examples',
'gallery_dirs': 'auto_examples',
'mod_example_dir': os.path.join('modules', 'generated'),
'doc_module': (),
'reference_url': {},
}
def setup(app):
"""Setup sphinx-gallery sphinx extension"""
app.add_config_value('plot_gallery', True, 'html')
app.add_config_value('abort_on_example_error', False, 'html')
app.add_config_value('sphinx_gallery_conf', gallery_conf, 'html')
app.add_stylesheet('gallery.css')
if 'sphinx.ext.autodoc' in app._extensions:
app.connect('autodoc-process-docstring', touch_empty_backreferences)
app.connect('builder-inited', generate_gallery_rst)
app.connect('build-finished', embed_code_links)
def setup_module():
# HACK: Stop nosetests running setup() above
pass
+656
View File
@@ -0,0 +1,656 @@
# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
==================
RST file generator
==================
Generate the rst files for the examples by iterating over the python
example files.
Files that generate images should start with 'plot'
"""
# Don't use unicode_literals here (be explicit with u"..." instead) otherwise
# tricky errors come up with exec(code_blocks, ...) calls
from __future__ import division, print_function, absolute_import
from time import time
import ast
import codecs
import hashlib
import os
import re
import shutil
import subprocess
import sys
import traceback
import warnings
# Try Python 2 first, otherwise load from Python 3
from textwrap import dedent
try:
# textwrap indent only exists in python 3
from textwrap import indent
except ImportError:
def indent(text, prefix, predicate=None):
"""Adds 'prefix' to the beginning of selected lines in 'text'.
If 'predicate' is provided, 'prefix' will only be added to the lines
where 'predicate(line)' is True. If 'predicate' is not provided,
it will default to adding 'prefix' to all non-empty lines that do not
consist solely of whitespace characters.
"""
if predicate is None:
def predicate(line):
return line.strip()
def prefixed_lines():
for line in text.splitlines(True):
yield (prefix + line if predicate(line) else line)
return ''.join(prefixed_lines())
from io import StringIO
try:
# make sure that the Agg backend is set before importing any
# matplotlib
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
except ImportError:
# this script can be imported by nosetest to find tests to run: we should
# not impose the matplotlib requirement in that case.
pass
from . import glr_path_static
from .backreferences import write_backreferences, _thumbnail_div
from .notebook import Notebook
try:
basestring
except NameError:
basestring = str
unicode = str
###############################################################################
class Tee(object):
"""A tee object to redirect streams to multiple outputs"""
def __init__(self, file1, file2):
self.file1 = file1
self.file2 = file2
def write(self, data):
self.file1.write(data)
self.file2.write(data)
def flush(self):
self.file1.flush()
self.file2.flush()
class MixedEncodingStringIO(StringIO):
"""Helper when both ASCII and unicode strings will be written"""
def write(self, data):
if not isinstance(data, unicode):
data = data.decode('utf-8')
StringIO.write(self, data)
###############################################################################
CODE_DOWNLOAD = """**Total running time of the script:**
({0:.0f} minutes {1:.3f} seconds)\n\n
\n.. container:: sphx-glr-download
**Download Python source code:** :download:`{2} <{2}>`\n
\n.. container:: sphx-glr-download
**Download IPython notebook:** :download:`{3} <{3}>`\n"""
# The following strings are used when we have several pictures: we use
# an html div tag that our CSS uses to turn the lists into horizontal
# lists.
HLIST_HEADER = """
.. rst-class:: sphx-glr-horizontal
"""
HLIST_IMAGE_TEMPLATE = """
*
.. image:: /%s
:scale: 47
"""
SINGLE_IMAGE = """
.. image:: /%s
:align: center
"""
# This one could contain unicode
CODE_OUTPUT = u""".. rst-class:: sphx-glr-script-out
Out::
{0}\n"""
def get_docstring_and_rest(filename):
"""Separate `filename` content between docstring and the rest
Strongly inspired from ast.get_docstring.
Returns
-------
docstring: str
docstring of `filename`
rest: str
`filename` content without the docstring
"""
# can't use codecs.open(filename, 'r', 'utf-8') here b/c ast doesn't
# seem to work with unicode strings in Python2.7
# "SyntaxError: encoding declaration in Unicode string"
with open(filename, 'rb') as f:
content = f.read()
node = ast.parse(content)
if not isinstance(node, ast.Module):
raise TypeError("This function only supports modules. "
"You provided {0}".format(node.__class__.__name__))
if node.body and isinstance(node.body[0], ast.Expr) and \
isinstance(node.body[0].value, ast.Str):
docstring_node = node.body[0]
docstring = docstring_node.value.s
if hasattr(docstring, 'decode'): # python2.7
docstring = docstring.decode('utf-8')
# This get the content of the file after the docstring last line
# Note: 'maxsplit' argument is not a keyword argument in python2
rest = content.decode('utf-8').split('\n', docstring_node.lineno)[-1]
return docstring, rest
else:
raise ValueError(('Could not find docstring in file "{0}". '
'A docstring is required by sphinx-gallery')
.format(filename))
def split_code_and_text_blocks(source_file):
"""Return list with source file separated into code and text blocks.
Returns
-------
blocks : list of (label, content)
List where each element is a tuple with the label ('text' or 'code'),
and content string of block.
"""
docstring, rest_of_content = get_docstring_and_rest(source_file)
blocks = [('text', docstring)]
pattern = re.compile(
r'(?P<header_line>^#{20,}.*)\s(?P<text_content>(?:^#.*\s)*)',
flags=re.M)
pos_so_far = 0
for match in re.finditer(pattern, rest_of_content):
match_start_pos, match_end_pos = match.span()
code_block_content = rest_of_content[pos_so_far:match_start_pos]
text_content = match.group('text_content')
sub_pat = re.compile('^#', flags=re.M)
text_block_content = dedent(re.sub(sub_pat, '', text_content)).lstrip()
if code_block_content.strip():
blocks.append(('code', code_block_content))
if text_block_content.strip():
blocks.append(('text', text_block_content))
pos_so_far = match_end_pos
remaining_content = rest_of_content[pos_so_far:]
if remaining_content.strip():
blocks.append(('code', remaining_content))
return blocks
def codestr2rst(codestr, lang='python'):
"""Return reStructuredText code block from code string"""
code_directive = "\n.. code-block:: {0}\n\n".format(lang)
indented_block = indent(codestr, ' ' * 4)
return code_directive + indented_block
def text2string(content):
"""Returns a string without the extra triple quotes"""
try:
return ast.literal_eval(content) + '\n'
except Exception:
return content + '\n'
def extract_thumbnail_number(text):
""" Pull out the thumbnail image number specified in the docstring. """
# check whether the user has specified a specific thumbnail image
pattr = re.compile("^\s*#\s*sphinx_gallery_thumbnail_number\s*=\s*([0-9]+)\s*$", flags=re.MULTILINE)
match = pattr.search(text)
if match is None:
# by default, use the first figure created
thumbnail_number = 1
else:
thumbnail_number = int(match.groups()[0])
return thumbnail_number
def extract_intro(filename):
""" Extract the first paragraph of module-level docstring. max:95 char"""
docstring, _ = get_docstring_and_rest(filename)
# lstrip is just in case docstring has a '\n\n' at the beginning
paragraphs = docstring.lstrip().split('\n\n')
if len(paragraphs) > 1:
first_paragraph = re.sub('\n', ' ', paragraphs[1])
first_paragraph = (first_paragraph[:95] + '...'
if len(first_paragraph) > 95 else first_paragraph)
else:
raise ValueError(
"Example docstring should have a header for the example title "
"and at least a paragraph explaining what the example is about. "
"Please check the example file:\n {}\n".format(filename))
return first_paragraph
def get_md5sum(src_file):
"""Returns md5sum of file"""
with open(src_file, 'r') as src_data:
src_content = src_data.read()
# data needs to be encoded in python3 before hashing
if sys.version_info[0] == 3:
src_content = src_content.encode('utf-8')
src_md5 = hashlib.md5(src_content).hexdigest()
return src_md5
def check_md5sum_change(src_file):
"""Returns True if src_file has a different md5sum"""
src_md5 = get_md5sum(src_file)
src_md5_file = src_file + '.md5'
src_file_changed = True
if os.path.exists(src_md5_file):
with open(src_md5_file, 'r') as file_checksum:
ref_md5 = file_checksum.read()
if src_md5 == ref_md5:
src_file_changed = False
if src_file_changed:
with open(src_md5_file, 'w') as file_checksum:
file_checksum.write(src_md5)
return src_file_changed
def _plots_are_current(src_file, image_path):
"""Test existence of image file and no change in md5sum of
example"""
has_image = os.path.exists(image_path)
src_file_changed = check_md5sum_change(src_file)
return has_image and not src_file_changed
def save_figures(image_path, fig_count, gallery_conf):
"""Save all open matplotlib figures of the example code-block
Parameters
----------
image_path : str
Path where plots are saved (format string which accepts figure number)
fig_count : int
Previous figure number count. Figure number add from this number
Returns
-------
list of strings containing the full path to each figure
"""
figure_list = []
fig_managers = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
for fig_mngr in fig_managers:
# Set the fig_num figure as the current figure as we can't
# save a figure that's not the current figure.
fig = plt.figure(fig_mngr.num)
kwargs = {}
to_rgba = matplotlib.colors.colorConverter.to_rgba
for attr in ['facecolor', 'edgecolor']:
fig_attr = getattr(fig, 'get_' + attr)()
default_attr = matplotlib.rcParams['figure.' + attr]
if to_rgba(fig_attr) != to_rgba(default_attr):
kwargs[attr] = fig_attr
current_fig = image_path.format(fig_count + fig_mngr.num)
fig.savefig(current_fig, **kwargs)
figure_list.append(current_fig)
if gallery_conf.get('find_mayavi_figures', False):
from mayavi import mlab
e = mlab.get_engine()
last_matplotlib_fig_num = fig_count + len(figure_list)
total_fig_num = last_matplotlib_fig_num + len(e.scenes)
mayavi_fig_nums = range(last_matplotlib_fig_num + 1, total_fig_num + 1)
for scene, mayavi_fig_num in zip(e.scenes, mayavi_fig_nums):
current_fig = image_path.format(mayavi_fig_num)
mlab.savefig(current_fig, figure=scene)
# make sure the image is not too large
scale_image(current_fig, current_fig, 850, 999)
figure_list.append(current_fig)
mlab.close(all=True)
return figure_list
def scale_image(in_fname, out_fname, max_width, max_height):
"""Scales an image with the same aspect ratio centered in an
image with a given max_width and max_height
if in_fname == out_fname the image can only be scaled down
"""
# local import to avoid testing dependency on PIL:
try:
from PIL import Image
except ImportError:
import Image
img = Image.open(in_fname)
width_in, height_in = img.size
scale_w = max_width / float(width_in)
scale_h = max_height / float(height_in)
if height_in * scale_w <= max_height:
scale = scale_w
else:
scale = scale_h
if scale >= 1.0 and in_fname == out_fname:
return
width_sc = int(round(scale * width_in))
height_sc = int(round(scale * height_in))
# resize the image
img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)
# insert centered
thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
pos_insert = ((max_width - width_sc) // 2, (max_height - height_sc) // 2)
thumb.paste(img, pos_insert)
thumb.save(out_fname)
# Use optipng to perform lossless compression on the resized image if
# software is installed
if os.environ.get('SKLEARN_DOC_OPTIPNG', False):
try:
subprocess.call(["optipng", "-quiet", "-o", "9", out_fname])
except Exception:
warnings.warn('Install optipng to reduce the size of the \
generated images')
def save_thumbnail(thumbnail_image_path, base_image_name, gallery_conf):
"""Save the thumbnail image"""
thumb_dir = os.path.join(os.path.dirname(thumbnail_image_path), 'thumb')
if not os.path.exists(thumb_dir):
os.makedirs(thumb_dir)
thumb_file = os.path.join(thumb_dir,
'sphx_glr_%s_thumb.png' % base_image_name)
if os.path.exists(thumbnail_image_path):
scale_image(thumbnail_image_path, thumb_file, 400, 280)
elif not os.path.exists(thumb_file):
# create something to replace the thumbnail
default_thumb_file = os.path.join(glr_path_static(), 'no_image.png')
default_thumb_file = gallery_conf.get("default_thumb_file",
default_thumb_file)
scale_image(default_thumb_file, thumb_file, 200, 140)
def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
"""Generate the gallery reStructuredText for an example directory"""
if not os.path.exists(os.path.join(src_dir, 'README.txt')):
print(80 * '_')
print('Example directory %s does not have a README.txt file' %
src_dir)
print('Skipping this directory')
print(80 * '_')
return "", [] # because string is an expected return type
fhindex = open(os.path.join(src_dir, 'README.txt')).read()
if not os.path.exists(target_dir):
os.makedirs(target_dir)
sorted_listdir = [fname for fname in sorted(os.listdir(src_dir))
if fname.endswith('.py')]
entries_text = []
computation_times = []
for fname in sorted_listdir:
amount_of_code, time_elapsed = \
generate_file_rst(fname, target_dir, src_dir, gallery_conf)
computation_times.append((time_elapsed, fname))
new_fname = os.path.join(src_dir, fname)
intro = extract_intro(new_fname)
write_backreferences(seen_backrefs, gallery_conf,
target_dir, fname, intro)
this_entry = _thumbnail_div(target_dir, fname, intro) + """
.. toctree::
:hidden:
/%s/%s\n""" % (target_dir, fname[:-3])
entries_text.append((amount_of_code, this_entry))
# sort to have the smallest entries in the beginning
entries_text.sort()
for _, entry_text in entries_text:
fhindex += entry_text
# clear at the end of the section
fhindex += """.. raw:: html\n
<div style='clear:both'></div>\n\n"""
return fhindex, computation_times
def execute_script(code_block, example_globals, image_path, fig_count,
src_file, gallery_conf):
"""Executes the code block of the example file"""
time_elapsed = 0
stdout = ''
# We need to execute the code
print('plotting code blocks in %s' % src_file)
plt.close('all')
cwd = os.getcwd()
# Redirect output to stdout and
orig_stdout = sys.stdout
try:
# First cd in the original example dir, so that any file
# created by the example get created in this directory
os.chdir(os.path.dirname(src_file))
my_buffer = MixedEncodingStringIO()
my_stdout = Tee(sys.stdout, my_buffer)
sys.stdout = my_stdout
t_start = time()
# don't use unicode_literals at the top of this file or you get
# nasty errors here on Py2.7
exec(code_block, example_globals)
time_elapsed = time() - t_start
sys.stdout = orig_stdout
my_stdout = my_buffer.getvalue().strip().expandtabs()
# raise RuntimeError
if my_stdout:
stdout = CODE_OUTPUT.format(indent(my_stdout, u' ' * 4))
os.chdir(cwd)
figure_list = save_figures(image_path, fig_count, gallery_conf)
# Depending on whether we have one or more figures, we're using a
# horizontal list or a single rst call to 'image'.
image_list = ""
if len(figure_list) == 1:
figure_name = figure_list[0]
image_list = SINGLE_IMAGE % figure_name.lstrip('/')
elif len(figure_list) > 1:
image_list = HLIST_HEADER
for figure_name in figure_list:
image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')
except Exception:
formatted_exception = traceback.format_exc()
sys.stdout = orig_stdout # need this here so these lines don't bomb
print(80 * '_')
print('%s is not compiling:' % src_file)
print(formatted_exception)
print(80 * '_')
figure_list = []
image_list = codestr2rst(formatted_exception, lang='pytb')
# Overrides the output thumbnail in the gallery for easy identification
broken_img = os.path.join(glr_path_static(), 'broken_example.png')
shutil.copyfile(broken_img, os.path.join(cwd, image_path.format(1)))
fig_count += 1 # raise count to avoid overwriting image
# Breaks build on first example error
# XXX This check can break during testing e.g. if you uncomment the
# `raise RuntimeError` by the `my_stdout` call, maybe use `.get()`?
if gallery_conf['abort_on_example_error']:
raise
finally:
os.chdir(cwd)
sys.stdout = orig_stdout
print(" - time elapsed : %.2g sec" % time_elapsed)
code_output = u"\n{0}\n\n{1}\n\n".format(image_list, stdout)
return code_output, time_elapsed, fig_count + len(figure_list)
def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
""" Generate the rst file for a given example.
Returns the amout of code (in characters) of the corresponding
files.
"""
src_file = os.path.join(src_dir, fname)
example_file = os.path.join(target_dir, fname)
shutil.copyfile(src_file, example_file)
image_dir = os.path.join(target_dir, 'images')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
base_image_name = os.path.splitext(fname)[0]
image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png'
image_path_template = os.path.join(image_dir, image_fname)
script_blocks = split_code_and_text_blocks(example_file)
# read specification of the figure to display as thumbnail from main text
_, content = get_docstring_and_rest(example_file)
thumbnail_number = extract_thumbnail_number(content)
amount_of_code = sum([len(bcontent)
for blabel, bcontent in script_blocks
if blabel == 'code'])
first_image_path = image_path_template.format(1)
if _plots_are_current(example_file, first_image_path):
return amount_of_code, 0
time_elapsed = 0
ref_fname = example_file.replace(os.path.sep, '_')
example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname)
example_nb = Notebook(fname, target_dir)
filename_pattern = gallery_conf.get('filename_pattern')
if re.search(filename_pattern, src_file) and gallery_conf['plot_gallery']:
example_globals = {
# A lot of examples contains 'print(__doc__)' for example in
# scikit-learn so that running the example prints some useful
# information. Because the docstring has been separated from
# the code blocks in sphinx-gallery, __doc__ is actually
# __builtin__.__doc__ in the execution context and we do not
# want to print it
'__doc__': '',
# Examples may contain if __name__ == '__main__' guards
# for in example scikit-learn if the example uses multiprocessing
'__name__': '__main__'}
fig_count = 0
# A simple example has two blocks: one for the
# example introduction/explanation and one for the code
is_example_notebook_like = len(script_blocks) > 2
for blabel, bcontent in script_blocks:
if blabel == 'code':
code_output, rtime, fig_count = execute_script(bcontent,
example_globals,
image_path_template,
fig_count,
src_file,
gallery_conf)
time_elapsed += rtime
example_nb.add_code_cell(bcontent)
if is_example_notebook_like:
example_rst += codestr2rst(bcontent) + '\n'
example_rst += code_output
else:
example_rst += code_output
if 'sphx-glr-script-out' in code_output:
# Add some vertical space after output
example_rst += "\n\n|\n\n"
example_rst += codestr2rst(bcontent) + '\n'
else:
example_rst += text2string(bcontent) + '\n'
example_nb.add_markdown_cell(text2string(bcontent))
else:
for blabel, bcontent in script_blocks:
if blabel == 'code':
example_rst += codestr2rst(bcontent) + '\n'
example_nb.add_code_cell(bcontent)
else:
example_rst += bcontent + '\n'
example_nb.add_markdown_cell(text2string(bcontent))
thumbnail_image_path = image_path_template.format(thumbnail_number)
save_thumbnail(thumbnail_image_path, base_image_name, gallery_conf)
time_m, time_s = divmod(time_elapsed, 60)
example_nb.save_file()
with codecs.open(os.path.join(target_dir, base_image_name + '.rst'),
mode='w', encoding='utf-8') as f:
example_rst += CODE_DOWNLOAD.format(time_m, time_s, fname,
example_nb.file_name)
f.write(example_rst)
return amount_of_code, time_elapsed
+123
View File
@@ -0,0 +1,123 @@
# -*- coding: utf-8 -*-
r"""
============================
Parser for Jupyter notebooks
============================
Class that holds the Ipython notebook information
"""
# Author: Óscar Nájera
# License: 3-clause BSD
from __future__ import division, absolute_import, print_function
import json
import os
import re
import sys
def ipy_notebook_skeleton():
"""Returns a dictionary with the elements of a Jupyter notebook"""
py_version = sys.version_info
notebook_skeleton = {
"cells": [],
"metadata": {
"kernelspec": {
"display_name": "Python " + str(py_version[0]),
"language": "python",
"name": "python" + str(py_version[0])
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": py_version[0]
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython" + str(py_version[0]),
"version": '{0}.{1}.{2}'.format(*sys.version_info[:3])
}
},
"nbformat": 4,
"nbformat_minor": 0
}
return notebook_skeleton
def rst2md(text):
"""Converts the RST text from the examples docstrigs and comments
into markdown text for the IPython notebooks"""
top_heading = re.compile(r'^=+$\s^([\w\s-]+)^=+$', flags=re.M)
text = re.sub(top_heading, r'# \1', text)
math_eq = re.compile(r'^\.\. math::((?:.+)?(?:\n+^ .+)*)', flags=re.M)
text = re.sub(math_eq,
lambda match: r'$${0}$$'.format(match.group(1).strip()),
text)
inline_math = re.compile(r':math:`(.+)`')
text = re.sub(inline_math, r'$\1$', text)
return text
class Notebook(object):
"""Ipython notebook object
Constructs the file cell-by-cell and writes it at the end"""
def __init__(self, file_name, target_dir):
"""Declare the skeleton of the notebook
Parameters
----------
file_name : str
original script file name, .py extension will be renamed
target_dir: str
directory where notebook file is to be saved
"""
self.file_name = file_name.replace('.py', '.ipynb')
self.write_file = os.path.join(target_dir, self.file_name)
self.work_notebook = ipy_notebook_skeleton()
self.add_code_cell("%matplotlib inline")
def add_code_cell(self, code):
"""Add a code cell to the notebook
Parameters
----------
code : str
Cell content
"""
code_cell = {
"cell_type": "code",
"execution_count": None,
"metadata": {"collapsed": False},
"outputs": [],
"source": [code.strip()]
}
self.work_notebook["cells"].append(code_cell)
def add_markdown_cell(self, text):
"""Add a markdown cell to the notebook
Parameters
----------
code : str
Cell content
"""
markdown_cell = {
"cell_type": "markdown",
"metadata": {},
"source": [rst2md(text)]
}
self.work_notebook["cells"].append(markdown_cell)
def save_file(self):
"""Saves the notebook to a file"""
with open(self.write_file, 'w') as out_nb:
json.dump(self.work_notebook, out_nb, indent=2)