wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-06-28 20:38:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into orders

Browse Source 
Conflicts:
	zipline/sources.py
This commit is contained in:
fawce
2012-02-26 07:04:30 -05:00
parent 757d1f75dc 98ed6fc410
commit fc69a1772b
96 changed files with 882 additions and 9334 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pyenv/.Python
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/Python
.pyenv/bin/activate
-76
View File
@@ -1,76 +0,0 @@
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
# reset old environment variables
if [ -n "$_OLD_VIRTUAL_PATH" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
hash -r
fi
if [ -n "$_OLD_VIRTUAL_PS1" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "$1" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelavent variables
deactivate nondestructive
VIRTUAL_ENV="/Users/fawce/projects/quantopian/qsim/.pyenv"
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "$PYTHONHOME" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then
_OLD_VIRTUAL_PS1="$PS1"
if [ "x" != x ] ; then
PS1="$PS1"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
hash -r
fi
.pyenv/bin/activate.csh
-32
View File
@@ -1,32 +0,0 @@
# This file must be used with "source bin/activate.csh" *from csh*.
# You cannot run it directly.
# Created by Davide Di Blasi <davidedb@gmail.com>.
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
# Unset irrelavent variables.
deactivate nondestructive
setenv VIRTUAL_ENV "/Users/fawce/projects/quantopian/qsim/.pyenv"
set _OLD_VIRTUAL_PATH="$PATH"
setenv PATH "$VIRTUAL_ENV/bin:$PATH"
set _OLD_VIRTUAL_PROMPT="$prompt"
if ("" != "") then
set env_name = ""
else
if (`basename "$VIRTUAL_ENV"` == "__") then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
set env_name = `basename \`dirname "$VIRTUAL_ENV"\``
else
set env_name = `basename "$VIRTUAL_ENV"`
endif
endif
set prompt = "[$env_name] $prompt"
unset env_name
rehash
.pyenv/bin/activate.fish
-79
View File
@@ -1,79 +0,0 @@
# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org)
# you cannot run it directly
function deactivate -d "Exit virtualenv and return to normal shell environment"
# reset old environment variables
if test -n "$_OLD_VIRTUAL_PATH"
set -gx PATH $_OLD_VIRTUAL_PATH
set -e _OLD_VIRTUAL_PATH
end
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
set -e _OLD_VIRTUAL_PYTHONHOME
end
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
functions -e fish_prompt
set -e _OLD_FISH_PROMPT_OVERRIDE
end
set -e VIRTUAL_ENV
if test "$argv[1]" != "nondestructive"
# Self destruct!
functions -e deactivate
end
end
# unset irrelavent variables
deactivate nondestructive
set -gx VIRTUAL_ENV "/Users/fawce/projects/quantopian/qsim/.pyenv"
set -gx _OLD_VIRTUAL_PATH $PATH
set -gx PATH "$VIRTUAL_ENV/bin" $PATH
# unset PYTHONHOME if set
if set -q PYTHONHOME
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
set -e PYTHONHOME
end
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
# fish shell uses a function, instead of env vars,
# to produce the prompt. Overriding the existing function is easy.
# However, adding to the current prompt, instead of clobbering it,
# is a little more work.
set -l oldpromptfile (tempfile)
if test $status
# save the current fish_prompt function...
echo "function _old_fish_prompt" >> $oldpromptfile
echo -n \# >> $oldpromptfile
functions fish_prompt >> $oldpromptfile
# we've made the "_old_fish_prompt" file, source it.
. $oldpromptfile
rm -f $oldpromptfile
if test -n ""
# We've been given us a prompt override.
#
# FIXME: Unsure how to handle this *safely*. We could just eval()
# whatever is given, but the risk is a bit much.
echo "activate.fish: Alternative prompt prefix is not supported under fish-shell." 1>&2
echo "activate.fish: Alter the fish_prompt in this file as needed." 1>&2
end
# with the original prompt function renamed, we can override with our own.
function fish_prompt
set -l _checkbase (basename "$VIRTUAL_ENV")
if test $_checkbase = "__"
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt)
else
printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt)
end
end
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
end
end
.pyenv/bin/activate_this.py
-32
View File
@@ -1,32 +0,0 @@
"""By using execfile(this_file, dict(__file__=this_file)) you will
activate this virtualenv environment.
This can be used when you must use an existing Python interpreter, not
the virtualenv bin/python
"""
try:
__file__
except NameError:
raise AssertionError(
"You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))")
import sys
import os
base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if sys.platform == 'win32':
site_packages = os.path.join(base, 'Lib', 'site-packages')
else:
site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages')
prev_sys_path = list(sys.path)
import site
site.addsitedir(site_packages)
sys.real_prefix = sys.prefix
sys.prefix = base
# Move the added items to the front of the path:
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
.pyenv/bin/easy_install
-9
View File
@@ -1,9 +0,0 @@
#!/Users/fawce/projects/quantopian/qsim/.pyenv/bin/python
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install'
__requires__ = 'setuptools==0.6c11'
import sys
from pkg_resources import load_entry_point
sys.exit(
load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')()
)
.pyenv/bin/easy_install-2.6
-9
View File
@@ -1,9 +0,0 @@
#!/Users/fawce/projects/quantopian/qsim/.pyenv/bin/python
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install-2.6'
__requires__ = 'setuptools==0.6c11'
import sys
from pkg_resources import load_entry_point
sys.exit(
load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install-2.6')()
)
.pyenv/bin/pip
-9
View File
@@ -1,9 +0,0 @@
#!/Users/fawce/projects/quantopian/qsim/.pyenv/bin/python
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==1.0.2','console_scripts','pip'
__requires__ = 'pip==1.0.2'
import sys
from pkg_resources import load_entry_point
sys.exit(
load_entry_point('pip==1.0.2', 'console_scripts', 'pip')()
)
.pyenv/bin/pip-2.6
-9
View File
@@ -1,9 +0,0 @@
#!/Users/fawce/projects/quantopian/qsim/.pyenv/bin/python
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==1.0.2','console_scripts','pip-2.6'
__requires__ = 'pip==1.0.2'
import sys
from pkg_resources import load_entry_point
sys.exit(
load_entry_point('pip==1.0.2', 'console_scripts', 'pip-2.6')()
)
.pyenv/bin/python
BIN
View File
Binary file not shown.
.pyenv/bin/python2.6
-1
View File
@@ -1 +0,0 @@
python
.pyenv/include/python2.6
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/include/python2.6
.pyenv/lib/python2.6/UserDict.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/UserDict.py
.pyenv/lib/python2.6/_abcoll.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/_abcoll.py
.pyenv/lib/python2.6/abc.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/abc.py
.pyenv/lib/python2.6/codecs.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/codecs.py
.pyenv/lib/python2.6/config
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/config
.pyenv/lib/python2.6/copy_reg.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/copy_reg.py
.pyenv/lib/python2.6/distutils/__init__.py
-97
View File
@@ -1,97 +0,0 @@
import os
import sys
import warnings
import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib
# Important! To work on pypy, this must be a module that resides in the
# lib-python/modified-x.y.z directory
dirname = os.path.dirname
distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils')
if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)):
warnings.warn(
"The virtualenv distutils package at %s appears to be in the same location as the system distutils?")
else:
__path__.insert(0, distutils_path)
exec(open(os.path.join(distutils_path, '__init__.py')).read())
try:
import dist
import sysconfig
except ImportError:
from distutils import dist, sysconfig
try:
basestring
except NameError:
basestring = str
## patch build_ext (distutils doesn't know how to get the libs directory
## path on windows - it hardcodes the paths around the patched sys.prefix)
if sys.platform == 'win32':
from distutils.command.build_ext import build_ext as old_build_ext
class build_ext(old_build_ext):
def finalize_options (self):
if self.library_dirs is None:
self.library_dirs = []
elif isinstance(self.library_dirs, basestring):
self.library_dirs = self.library_dirs.split(os.pathsep)
self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs"))
old_build_ext.finalize_options(self)
from distutils.command import build_ext as build_ext_module
build_ext_module.build_ext = build_ext
## distutils.dist patches:
old_find_config_files = dist.Distribution.find_config_files
def find_config_files(self):
found = old_find_config_files(self)
system_distutils = os.path.join(distutils_path, 'distutils.cfg')
#if os.path.exists(system_distutils):
# found.insert(0, system_distutils)
# What to call the per-user config file
if os.name == 'posix':
user_filename = ".pydistutils.cfg"
else:
user_filename = "pydistutils.cfg"
user_filename = os.path.join(sys.prefix, user_filename)
if os.path.isfile(user_filename):
for item in list(found):
if item.endswith('pydistutils.cfg'):
found.remove(item)
found.append(user_filename)
return found
dist.Distribution.find_config_files = find_config_files
## distutils.sysconfig patches:
old_get_python_inc = sysconfig.get_python_inc
def sysconfig_get_python_inc(plat_specific=0, prefix=None):
if prefix is None:
prefix = sys.real_prefix
return old_get_python_inc(plat_specific, prefix)
sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__
sysconfig.get_python_inc = sysconfig_get_python_inc
old_get_python_lib = sysconfig.get_python_lib
def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
if standard_lib and prefix is None:
prefix = sys.real_prefix
return old_get_python_lib(plat_specific, standard_lib, prefix)
sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__
sysconfig.get_python_lib = sysconfig_get_python_lib
old_get_config_vars = sysconfig.get_config_vars
def sysconfig_get_config_vars(*args):
real_vars = old_get_config_vars(*args)
if sys.platform == 'win32':
lib_dir = os.path.join(sys.real_prefix, "libs")
if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars:
real_vars['LIBDIR'] = lib_dir # asked for all
elif isinstance(real_vars, list) and 'LIBDIR' in args:
real_vars = real_vars + [lib_dir] # asked for list
return real_vars
sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__
sysconfig.get_config_vars = sysconfig_get_config_vars
.pyenv/lib/python2.6/distutils/distutils.cfg
-6
View File
@@ -1,6 +0,0 @@
# This is a config file local to this virtualenv installation
# You may include options that will be used by all distutils commands,
# and by easy_install. For instance:
#
# [easy_install]
# find_links = http://mylocalsite
.pyenv/lib/python2.6/encodings
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/encodings
.pyenv/lib/python2.6/fnmatch.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/fnmatch.py
.pyenv/lib/python2.6/genericpath.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/genericpath.py
.pyenv/lib/python2.6/lib-dynload
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/lib-dynload
.pyenv/lib/python2.6/linecache.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/linecache.py
.pyenv/lib/python2.6/locale.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/locale.py
.pyenv/lib/python2.6/no-global-site-packages.txt
View File
.pyenv/lib/python2.6/ntpath.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/ntpath.py
.pyenv/lib/python2.6/orig-prefix.txt
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6
.pyenv/lib/python2.6/os.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/os.py
.pyenv/lib/python2.6/posixpath.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/posixpath.py
.pyenv/lib/python2.6/re.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py
.pyenv/lib/python2.6/site-packages/easy-install.pth
-4
View File
@@ -1,4 +0,0 @@
import sys; sys.__plen = len(sys.path)
./setuptools-0.6c11-py2.6.egg
./pip-1.0.2-py2.6.egg
import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/PKG-INFO
-860
View File
@@ -1,860 +0,0 @@
Metadata-Version: 1.0
Name: pip
Version: 1.0.2
Summary: pip installs packages. Python packages. An easy_install replacement
Home-page: http://www.pip-installer.org
Author: The pip developers
Author-email: python-virtualenv@groups.google.com
License: MIT
Description: pip
===
pip is a tool for installing and managing Python packages, such as those
found in the `Python Package Index <http://pypi.python.org/pypi>`_.
pip is a replacement for `easy_install
<http://peak.telecommunity.com/DevCenter/EasyInstall>`_. It mostly
uses the same techniques for finding packages, so packages that are
easy_installable should be pip-installable as well. This means that
you can use ``pip install SomePackage`` instead of ``easy_install
SomePackage``.
In order to use pip, you must first install `setuptools
<http://pypi.python.org/pypi/setuptools>`_ or `distribute
<http://pypi.python.org/pypi/distribute>`_. If you use `virtualenv
<http://www.virtualenv.org>`_, a copy of pip will be automatically be
installed in each virtual environment you create.
.. comment:
The main website for pip is `www.pip-installer.org
<http://www.pip-installer.org>`_. You can also install
the `in-development version <https://github.com/pypa/pip/tarball/develop#egg=pip-dev>`_
of pip with ``easy_install pip==dev``.
Usage
-----
Once you have pip, you can use it like this::
$ pip install SomePackage
SomePackage is some package you'll find on `PyPI
<http://pypi.python.org/pypi/>`_. This installs the package and all
its dependencies.
pip does other stuff too, with packages, but install is the biggest
one. You can ``pip uninstall`` too.
You can also install from a URL (that points to a tar or zip file),
install from some version control system (use URLs like
``hg+http://domain/repo`` -- or prefix ``git+``, ``svn+`` etc). pip
knows a bunch of stuff about revisions and stuff, so if you need to do
things like install a very specific revision from a repository pip can
do that too.
If you've ever used ``python setup.py develop``, you can do something
like that with ``pip install -e ./`` -- this works with packages that
use ``distutils`` too (usually this only works with Setuptools
projects).
You can use ``pip install --upgrade SomePackage`` to upgrade to a
newer version, or ``pip install SomePackage==1.0.4`` to install a very
specific version.
Pip Compared To easy_install
----------------------------
pip is meant to improve on easy_install. Some of the improvements:
* All packages are downloaded before installation. Partially-completed
installation doesn't occur as a result.
* Care is taken to present useful output on the console.
* The reasons for actions are kept track of. For instance, if a package is
being installed, pip keeps track of why that package was required.
* Error messages should be useful.
* The code is relatively concise and cohesive, making it easier to use
programmatically.
* Packages don't have to be installed as egg archives, they can be installed
flat (while keeping the egg metadata).
* Native support for other version control systems (Git, Mercurial and Bazaar)
* Uninstallation of packages.
* Simple to define fixed sets of requirements and reliably reproduce a
set of packages.
pip doesn't do everything that easy_install does. Specifically:
* It cannot install from eggs. It only installs from source. (In the
future it would be good if it could install binaries from Windows ``.exe``
or ``.msi`` -- binary install on other platforms is not a priority.)
* It doesn't understand Setuptools extras (like ``package[test]``). This should
be added eventually.
* It is incompatible with some packages that extensively customize distutils
or setuptools in their ``setup.py`` files.
pip is complementary with `virtualenv
<http://pypi.python.org/pypi/virtualenv>`__, and it is encouraged that you use
virtualenv to isolate your installation.
Community
---------
The homepage for pip is at `pip-installer.org <http://www.pip-installer.org/>`_.
Bugs can be filed in the `pip issue tracker
<https://github.com/pypa/pip/issues/>`_. Discussion happens on the
`virtualenv email group
<http://groups.google.com/group/python-virtualenv?hl=en>`_.
Uninstall
---------
pip is able to uninstall most installed packages with ``pip uninstall
package-name``.
Known exceptions include pure-distutils packages installed with
``python setup.py install`` (such packages leave behind no metadata allowing
determination of what files were installed), and script wrappers installed
by develop-installs (``python setup.py develop``).
pip also performs an automatic uninstall of an old version of a package
before upgrading to a newer version, so outdated files (and egg-info data)
from conflicting versions aren't left hanging around to cause trouble. The
old version of the package is automatically restored if the new version
fails to download or install.
.. _`requirements file`:
Requirements Files
------------------
When installing software, and Python packages in particular, it's common that
you get a lot of libraries installed. You just did ``easy_install MyPackage``
and you get a dozen packages. Each of these packages has its own version.
Maybe you ran that installation and it works. Great! Will it keep working?
Did you have to provide special options to get it to find everything? Did you
have to install a bunch of other optional pieces? Most of all, will you be able
to do it again? Requirements files give you a way to create an *environment*:
a *set* of packages that work together.
If you've ever tried to setup an application on a new system, or with slightly
updated pieces, and had it fail, pip requirements are for you. If you
haven't had this problem then you will eventually, so pip requirements are
for you too -- requirements make explicit, repeatable installation of packages.
So what are requirements files? They are very simple: lists of packages to
install. Instead of running something like ``pip MyApp`` and getting
whatever libraries come along, you can create a requirements file something like::
MyApp
Framework==0.9.4
Library>=0.2
Then, regardless of what MyApp lists in ``setup.py``, you'll get a
specific version of Framework (0.9.4) and at least the 0.2 version of
Library. (You might think you could list these specific versions in
MyApp's ``setup.py`` -- but if you do that you'll have to edit MyApp
if you want to try a new version of Framework, or release a new
version of MyApp if you determine that Library 0.3 doesn't work with
your application.) You can also add optional libraries and support
tools that MyApp doesn't strictly require, giving people a set of
recommended libraries.
You can also include "editable" packages -- packages that are checked out from
Subversion, Git, Mercurial and Bazaar. These are just like using the ``-e``
option to pip. They look like::
-e svn+http://myrepo/svn/MyApp#egg=MyApp
You have to start the URL with ``svn+`` (``git+``, ``hg+`` or ``bzr+``), and
you have to include ``#egg=Package`` so pip knows what to expect at that URL.
You can also include ``@rev`` in the URL, e.g., ``@275`` to check out
revision 275.
Requirement files are mostly *flat*. Maybe ``MyApp`` requires
``Framework``, and ``Framework`` requires ``Library``. I encourage
you to still list all these in a single requirement file; it is the
nature of Python programs that there are implicit bindings *directly*
between MyApp and Library. For instance, Framework might expose one
of Library's objects, and so if Library is updated it might directly
break MyApp. If that happens you can update the requirements file to
force an earlier version of Library, and you can do that without
having to re-release MyApp at all.
Read the `requirements file format <http://pip.openplans.org/requirement-format.html>`_ to
learn about other features.
Freezing Requirements
---------------------
So you have a working set of packages, and you want to be able to install them
elsewhere. `Requirements files`_ let you install exact versions, but it won't
tell you what all the exact versions are.
To create a new requirements file from a known working environment, use::
$ pip freeze > stable-req.txt
This will write a listing of *all* installed libraries to ``stable-req.txt``
with exact versions for every library. You may want to edit the file down after
generating (e.g., to eliminate unnecessary libraries), but it'll give you a
stable starting point for constructing your requirements file.
You can also give it an existing requirements file, and it will use that as a
sort of template for the new file. So if you do::
$ pip freeze -r devel-req.txt > stable-req.txt
it will keep the packages listed in ``devel-req.txt`` in order and preserve
comments.
Bundles
-------
Another way to distribute a set of libraries is a bundle format (specific to
pip). This format is not stable at this time (there simply hasn't been
any feedback, nor a great deal of thought). A bundle file contains all the
source for your package, and you can have pip install them all together.
Once you have the bundle file further network access won't be necessary. To
build a bundle file, do::
$ pip bundle MyApp.pybundle MyApp
(Using a `requirements file`_ would be wise.) Then someone else can get the
file ``MyApp.pybundle`` and run::
$ pip install MyApp.pybundle
This is *not* a binary format. This only packages source. If you have binary
packages, then the person who installs the files will have to have a compiler,
any necessary headers installed, etc. Binary packages are hard, this is
relatively easy.
Using pip with virtualenv
-------------------------
pip is most nutritious when used with `virtualenv
<http://pypi.python.org/pypi/virtualenv>`__. One of the reasons pip
doesn't install "multi-version" eggs is that virtualenv removes much of the need
for it. Because pip is installed by virtualenv, just use
``path/to/my/environment/bin/pip`` to install things into that
specific environment.
To tell pip to only run if there is a virtualenv currently activated,
and to bail if not, use::
export PIP_REQUIRE_VIRTUALENV=true
To tell pip to automatically use the currently active virtualenv::
export PIP_RESPECT_VIRTUALENV=true
Providing an environment with ``-E`` will be ignored.
Using pip with virtualenvwrapper
---------------------------------
If you are using `virtualenvwrapper
<http://www.doughellmann.com/projects/virtualenvwrapper/>`_, you might
want pip to automatically create its virtualenvs in your
``$WORKON_HOME``.
You can tell pip to do so by defining ``PIP_VIRTUALENV_BASE`` in your
environment and setting it to the same value as that of
``$WORKON_HOME``.
Do so by adding the line::
export PIP_VIRTUALENV_BASE=$WORKON_HOME
in your .bashrc under the line starting with ``export WORKON_HOME``.
Using pip with buildout
-----------------------
If you are using `zc.buildout
<http://pypi.python.org/pypi/zc.buildout>`_ you should look at
`gp.recipe.pip <http://pypi.python.org/pypi/gp.recipe.pip>`_ as an
option to use pip and virtualenv in your buildouts.
Command line completion
-----------------------
pip comes with support for command line completion in bash and zsh and
allows you tab complete commands and options. To enable it you simply
need copy the required shell script to the your shell startup file
(e.g. ``.profile`` or ``.zprofile``) by running the special ``completion``
command, e.g. for bash::
$ pip completion --bash >> ~/.profile
And for zsh::
$ pip completion --zsh >> ~/.zprofile
Alternatively, you can use the result of the ``completion`` command
directly with the eval function of you shell, e.g. by adding::
eval "`pip completion --bash`"
to your startup file.
Searching for packages
----------------------
pip can search the `Python Package Index <http://pypi.python.org/pypi>`_ (PyPI)
for packages using the ``pip search`` command. To search, run::
$ pip search "query"
The query will be used to search the names and summaries of all packages
indexed.
pip searches http://pypi.python.org/pypi by default but alternative indexes
can be searched by using the ``--index`` flag.
Mirror support
--------------
The `PyPI mirroring infrastructure <http://pypi.python.org/mirrors>`_ as
described in `PEP 381 <http://www.python.org/dev/peps/pep-0381/>`_ can be
used by passing the ``--use-mirrors`` option to the install command.
Alternatively, you can use the other ways to configure pip, e.g.::
$ export PIP_USE_MIRRORS=true
If enabled, pip will automatically query the DNS entry of the mirror index URL
to find the list of mirrors to use. In case you want to override this list,
please use the ``--mirrors`` option of the install command, or add to your pip
configuration file::
[install]
use-mirrors = true
mirrors =
http://d.pypi.python.org
http://b.pypi.python.org
News / Changelog
================
Next release (1.1) schedule
---------------------------
Beta release mid-July 2011, final release early August.
1.0.2 (2011-07-16)
------------------
* Fixed docs issues.
* Fixed issue #295 - Reinstall a package when using the ``install -I`` option
* Fixed issue #283 - Finds a Git tag pointing to same commit as origin/master
* Fixed issue #279 - Use absolute path for path to docs in setup.py
* Fixed issue #320 - Correctly handle exceptions on Python3.
* Fixed issue #314 - Correctly parse ``--editable`` lines in requirements files
1.0.1 (2011-04-30)
------------------
* Start to use git-flow.
* Fixed issue #274 - `find_command` should not raise AttributeError
* Fixed issue #273 - respect Content-Disposition header. Thanks Bradley Ayers.
* Fixed issue #233 - pathext handling on Windows.
* Fixed issue #252 - svn+svn protocol.
* Fixed issue #44 - multiple CLI searches.
* Fixed issue #266 - current working directory when running setup.py clean.
1.0 (2011-04-04)
----------------
* Added Python 3 support! Huge thanks to Vinay Sajip, Vitaly Babiy, Kelsey
Hightower, and Alex Gronholm, among others.
* Download progress only shown on a real TTY. Thanks Alex Morega.
* Fixed finding of VCS binaries to not be fooled by same-named directories.
Thanks Alex Morega.
* Fixed uninstall of packages from system Python for users of Debian/Ubuntu
python-setuptools package (workaround until fixed in Debian and Ubuntu).
* Added `get-pip.py <https://raw.github.com/pypa/pip/master/contrib/get-pip.py>`_
installer. Simply download and execute it, using the Python interpreter of
your choice::
$ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
$ python get-pip.py
This may have to be run as root.
.. note::
Make sure you have `distribute <http://pypi.python.org/pypi/distribute>`_
installed before using the installer!
0.8.3
-----
* Moved main repository to Github: https://github.com/pypa/pip
* Transferred primary maintenance from Ian to Jannis Leidel, Carl Meyer, Brian Rosner
* Fixed issue #14 - No uninstall-on-upgrade with URL package. Thanks Oliver Tonnhofer
* Fixed issue #163 - Egg name not properly resolved. Thanks Igor Sobreira
* Fixed issue #178 - Non-alphabetical installation of requirements. Thanks Igor Sobreira
* Fixed issue #199 - Documentation mentions --index instead of --index-url. Thanks Kelsey Hightower
* Fixed issue #204 - rmtree undefined in mercurial.py. Thanks Kelsey Hightower
* Fixed bug in Git vcs backend that would break during reinstallation.
* Fixed bug in Mercurial vcs backend related to pip freeze and branch/tag resolution.
* Fixed bug in version string parsing related to the suffix "-dev".
0.8.2
-----
* Avoid redundant unpacking of bundles (from pwaller)
* Fixed issue #32, #150, #161 - Fixed checking out the correct
tag/branch/commit when updating an editable Git requirement.
* Fixed issue #49 - Added ability to install version control requirements
without making them editable, e.g.::
pip install git+https://github.com/pypa/pip/
* Fixed issue #175 - Correctly locate build and source directory on Mac OS X.
* Added ``git+https://`` scheme to Git VCS backend.
0.8.1
-----
* Added global --user flag as shortcut for --install-option="--user". From
Ronny Pfannschmidt.
* Added support for `PyPI mirrors <http://pypi.python.org/mirrors>`_ as
defined in `PEP 381 <http://www.python.org/dev/peps/pep-0381/>`_, from
Jannis Leidel.
* Fixed issue #138 - Git revisions ignored. Thanks John-Scott Atlakson.
* Fixed issue #95 - Initial editable install of github package from a tag fails. Thanks John-Scott Atlakson.
* Fixed issue #107 - Can't install if a directory in cwd has the same name as the package you're installing.
* Fixed issue #39 - --install-option="--prefix=~/.local" ignored with -e.
Thanks Ronny Pfannschmidt and Wil Tan.
0.8
---
* Track which ``build/`` directories pip creates, never remove directories
it doesn't create. From Hugo Lopes Tavares.
* Pip now accepts file:// index URLs. Thanks Dave Abrahams.
* Various cleanup to make test-running more consistent and less fragile.
Thanks Dave Abrahams.
* Real Windows support (with passing tests). Thanks Dave Abrahams.
* ``pip-2.7`` etc. scripts are created (Python-version specific scripts)
* ``contrib/build-standalone`` script creates a runnable ``.zip`` form of
pip, from Jannis Leidel
* Editable git repos are updated when reinstalled
* Fix problem with ``--editable`` when multiple ``.egg-info/`` directories
are found.
* A number of VCS-related fixes for ``pip freeze``, from Hugo Lopes Tavares.
* Significant test framework changes, from Hugo Lopes Tavares.
0.7.2
-----
* Set zip_safe=False to avoid problems some people are encountering where
pip is installed as a zip file.
0.7.1
-----
* Fixed opening of logfile with no directory name. Thanks Alexandre Conrad.
* Temporary files are consistently cleaned up, especially after
installing bundles, also from Alex Conrad.
* Tests now require at least ScriptTest 1.0.3.
0.7
---
* Fixed uninstallation on Windows
* Added ``pip search`` command.
* Tab-complete names of installed distributions for ``pip uninstall``.
* Support tab-completion when there is a global-option before the
subcommand.
* Install header files in standard (scheme-default) location when installing
outside a virtualenv. Install them to a slightly more consistent
non-standard location inside a virtualenv (since the standard location is
a non-writable symlink to the global location).
* pip now logs to a central location by default (instead of creating
``pip-log.txt`` all over the place) and constantly overwrites the
file in question. On Unix and Mac OS X this is ``'$HOME/.pip/pip.log'``
and on Windows it's ``'%HOME%\\pip\\pip.log'``. You are still able to
override this location with the ``$PIP_LOG_FILE`` environment variable.
For a complete (appended) logfile use the separate ``'--log'`` command line
option.
* Fixed an issue with Git that left an editable packge as a checkout of a
remote branch, even if the default behaviour would have been fine, too.
* Fixed installing from a Git tag with older versions of Git.
* Expand "~" in logfile and download cache paths.
* Speed up installing from Mercurial repositories by cloning without
updating the working copy multiple times.
* Fixed installing directly from directories (e.g.
``pip install path/to/dir/``).
* Fixed installing editable packages with ``svn+ssh`` URLs.
* Don't print unwanted debug information when running the freeze command.
* Create log file directory automatically. Thanks Alexandre Conrad.
* Make test suite easier to run successfully. Thanks Dave Abrahams.
* Fixed "pip install ." and "pip install .."; better error for directory
without setup.py. Thanks Alexandre Conrad.
* Support Debian/Ubuntu "dist-packages" in zip command. Thanks duckx.
* Fix relative --src folder. Thanks Simon Cross.
* Handle missing VCS with an error message. Thanks Alexandre Conrad.
* Added --no-download option to install; pairs with --no-install to separate
download and installation into two steps. Thanks Simon Cross.
* Fix uninstalling from requirements file containing -f, -i, or
--extra-index-url.
* Leftover build directories are now removed. Thanks Alexandre Conrad.
0.6.3
-----
* Fixed import error on Windows with regard to the backwards compatibility
package
0.6.2
-----
* Fixed uninstall when /tmp is on a different filesystem.
* Fixed uninstallation of distributions with namespace packages.
0.6.1
-----
* Added support for the ``https`` and ``http-static`` schemes to the
Mercurial and ``ftp`` scheme to the Bazaar backend.
* Fixed uninstallation of scripts installed with easy_install.
* Fixed an issue in the package finder that could result in an
infinite loop while looking for links.
* Fixed issue with ``pip bundle`` and local files (which weren't being
copied into the bundle), from Whit Morriss.
0.6
---
* Add ``pip uninstall`` and uninstall-before upgrade (from Carl
Meyer).
* Extended configurability with config files and environment variables.
* Allow packages to be upgraded, e.g., ``pip install Package==0.1``
then ``pip install Package==0.2``.
* Allow installing/upgrading to Package==dev (fix "Source version does not
match target version" errors).
* Added command and option completion for bash and zsh.
* Extended integration with virtualenv by providing an option to
automatically use an active virtualenv and an option to warn if no active
virtualenv is found.
* Fixed a bug with pip install --download and editable packages, where
directories were being set with 0000 permissions, now defaults to 755.
* Fixed uninstallation of easy_installed console_scripts.
* Fixed uninstallation on Mac OS X Framework layout installs
* Fixed bug preventing uninstall of editables with source outside venv.
* Creates download cache directory if not existing.
0.5.1
-----
* Fixed a couple little bugs, with git and with extensions.
0.5
---
* Added ability to override the default log file name (``pip-log.txt``)
with the environmental variable ``$PIP_LOG_FILE``.
* Made the freeze command print installed packages to stdout instead of
writing them to a file. Use simple redirection (e.g.
``pip freeze > stable-req.txt``) to get a file with requirements.
* Fixed problem with freezing editable packages from a Git repository.
* Added support for base URLs using ``<base href='...'>`` when parsing
HTML pages.
* Fixed installing of non-editable packages from version control systems.
* Fixed issue with Bazaar's bzr+ssh scheme.
* Added --download-dir option to the install command to retrieve package
archives. If given an editable package it will create an archive of it.
* Added ability to pass local file and directory paths to ``--find-links``,
e.g. ``--find-links=file:///path/to/my/private/archive``
* Reduced the amount of console log messages when fetching a page to find a
distribution was problematic. The full messages can be found in pip-log.txt.
* Added ``--no-deps`` option to install ignore package dependencies
* Added ``--no-index`` option to ignore the package index (PyPI) temporarily
* Fixed installing editable packages from Git branches.
* Fixes freezing of editable packages from Mercurial repositories.
* Fixed handling read-only attributes of build files, e.g. of Subversion and
Bazaar on Windows.
* When downloading a file from a redirect, use the redirected
location's extension to guess the compression (happens specifically
when redirecting to a bitbucket.org tip.gz file).
* Editable freeze URLs now always use revision hash/id rather than tip or
branch names which could move.
* Fixed comparison of repo URLs so incidental differences such as
presence/absence of final slashes or quoted/unquoted special
characters don't trigger "ignore/switch/wipe/backup" choice.
* Fixed handling of attempt to checkout editable install to a
non-empty, non-repo directory.
0.4
---
* Make ``-e`` work better with local hg repositories
* Construct PyPI URLs the exact way easy_install constructs URLs (you
might notice this if you use a custom index that is
slash-sensitive).
* Improvements on Windows (from `Ionel Maries Cristian
<http://ionelmc.wordpress.com/>`_).
* Fixed problem with not being able to install private git repositories.
* Make ``pip zip`` zip all its arguments, not just the first.
* Fix some filename issues on Windows.
* Allow the ``-i`` and ``--extra-index-url`` options in requirements
files.
* Fix the way bundle components are unpacked and moved around, to make
bundles work.
* Adds ``-s`` option to allow the access to the global site-packages if a
virtualenv is to be created.
* Fixed support for Subversion 1.6.
0.3.1
-----
* Improved virtualenv restart and various path/cleanup problems on win32.
* Fixed a regression with installing from svn repositories (when not
using ``-e``).
* Fixes when installing editable packages that put their source in a
subdirectory (like ``src/``).
* Improve ``pip -h``
0.3
---
* Added support for editable packages created from Git, Mercurial and Bazaar
repositories and ability to freeze them. Refactored support for version
control systems.
* Do not use ``sys.exit()`` from inside the code, instead use a
return. This will make it easier to invoke programmatically.
* Put the install record in ``Package.egg-info/installed-files.txt``
(previously they went in
``site-packages/install-record-Package.txt``).
* Fix a problem with ``pip freeze`` not including ``-e svn+`` when an
svn structure is peculiar.
* Allow ``pip -E`` to work with a virtualenv that uses a different
version of Python than the parent environment.
* Fixed Win32 virtualenv (``-E``) option.
* Search the links passed in with ``-f`` for packages.
* Detect zip files, even when the file doesn't have a ``.zip``
extension and it is served with the wrong Content-Type.
* Installing editable from existing source now works, like ``pip
install -e some/path/`` will install the package in ``some/path/``.
Most importantly, anything that package requires will also be
installed by pip.
* Add a ``--path`` option to ``pip un/zip``, so you can avoid zipping
files that are outside of where you expect.
* Add ``--simulate`` option to ``pip zip``.
0.2.1
-----
* Fixed small problem that prevented using ``pip.py`` without actually
installing pip.
* Fixed ``--upgrade``, which would download and appear to install
upgraded packages, but actually just reinstall the existing package.
* Fixed Windows problem with putting the install record in the right
place, and generating the ``pip`` script with Setuptools.
* Download links that include embedded spaces or other unsafe
characters (those characters get %-encoded).
* Fixed use of URLs in requirement files, and problems with some blank
lines.
* Turn some tar file errors into warnings.
0.2
---
* Renamed to ``pip``, and to install you now do ``pip install
PACKAGE``
* Added command ``pip zip PACKAGE`` and ``pip unzip PACKAGE``. This
is particularly intended for Google App Engine to manage libraries
to stay under the 1000-file limit.
* Some fixes to bundles, especially editable packages and when
creating a bundle using unnamed packages (like just an svn
repository without ``#egg=Package``).
0.1.4
-----
* Added an option ``--install-option`` to pass options to pass
arguments to ``setup.py install``
* ``.svn/`` directories are no longer included in bundles, as these
directories are specific to a version of svn -- if you build a
bundle on a system with svn 1.5, you can't use the checkout on a
system with svn 1.4. Instead a file ``svn-checkout.txt`` is
included that notes the original location and revision, and the
command you can use to turn it back into an svn checkout. (Probably
unpacking the bundle should, maybe optionally, recreate this
information -- but that is not currently implemented, and it would
require network access.)
* Avoid ambiguities over project name case, where for instance
MyPackage and mypackage would be considered different packages.
This in particular caused problems on Macs, where ``MyPackage/`` and
``mypackage/`` are the same directory.
* Added support for an environmental variable
``$PIP_DOWNLOAD_CACHE`` which will cache package downloads, so
future installations won't require large downloads. Network access
is still required, but just some downloads will be avoided when
using this.
0.1.3
-----
* Always use ``svn checkout`` (not ``export``) so that
``tag_svn_revision`` settings give the revision of the package.
* Don't update checkouts that came from ``.pybundle`` files.
0.1.2
-----
* Improve error text when there are errors fetching HTML pages when
seeking packages.
* Improve bundles: include empty directories, make them work with
editable packages.
* If you use ``-E env`` and the environment ``env/`` doesn't exist, a
new virtual environment will be created.
* Fix ``dependency_links`` for finding packages.
0.1.1
-----
* Fixed a NameError exception when running pip outside of a
virtualenv environment.
* Added HTTP proxy support (from Prabhu Ramachandran)
* Fixed use of ``hashlib.md5`` on python2.5+ (also from Prabhu
Ramachandran)
0.1
---
* Initial release
Keywords: easy_install distutils setuptools egg virtualenv
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Topic :: Software Development :: Build Tools
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.4
Classifier: Programming Language :: Python :: 2.5
Classifier: Programming Language :: Python :: 2.6
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.1
Classifier: Programming Language :: Python :: 3.2
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/SOURCES.txt
-48
View File
@@ -1,48 +0,0 @@
AUTHORS.txt
LICENSE.txt
MANIFEST.in
setup.cfg
setup.py
docs/ci-server-step-by-step.txt
docs/configuration.txt
docs/how-to-contribute.txt
docs/index.txt
docs/installing.txt
docs/news.txt
docs/requirement-format.txt
docs/running-tests.txt
pip/__init__.py
pip/_pkgutil.py
pip/backwardcompat.py
pip/basecommand.py
pip/baseparser.py
pip/download.py
pip/exceptions.py
pip/index.py
pip/locations.py
pip/log.py
pip/req.py
pip/runner.py
pip/util.py
pip/venv.py
pip.egg-info/PKG-INFO
pip.egg-info/SOURCES.txt
pip.egg-info/dependency_links.txt
pip.egg-info/entry_points.txt
pip.egg-info/not-zip-safe
pip.egg-info/top_level.txt
pip/commands/__init__.py
pip/commands/bundle.py
pip/commands/completion.py
pip/commands/freeze.py
pip/commands/help.py
pip/commands/install.py
pip/commands/search.py
pip/commands/uninstall.py
pip/commands/unzip.py
pip/commands/zip.py
pip/vcs/__init__.py
pip/vcs/bazaar.py
pip/vcs/git.py
pip/vcs/mercurial.py
pip/vcs/subversion.py
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/dependency_links.txt
-1
View File
@@ -1 +0,0 @@
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/entry_points.txt
-4
View File
@@ -1,4 +0,0 @@
[console_scripts]
pip = pip:main
pip-2.6 = pip:main
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/not-zip-safe
-1
View File
@@ -1 +0,0 @@
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/EGG-INFO/top_level.txt
-1
View File
@@ -1 +0,0 @@
pip
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/__init__.py
-267
View File
@@ -1,267 +0,0 @@
#!/usr/bin/env python
import os
import optparse
import subprocess
import sys
import re
import difflib
from pip.backwardcompat import u, walk_packages, console_to_str
from pip.basecommand import command_dict, load_command, load_all_commands, command_names
from pip.baseparser import parser
from pip.exceptions import InstallationError
from pip.log import logger
from pip.util import get_installed_distributions
def autocomplete():
"""Command and option completion for the main option parser (and options)
and its subcommands (and options).
Enable by sourcing one of the completion shell scripts (bash or zsh).
"""
# Don't complete if user hasn't sourced bash_completion file.
if 'PIP_AUTO_COMPLETE' not in os.environ:
return
cwords = os.environ['COMP_WORDS'].split()[1:]
cword = int(os.environ['COMP_CWORD'])
try:
current = cwords[cword-1]
except IndexError:
current = ''
load_all_commands()
subcommands = [cmd for cmd, cls in command_dict.items() if not cls.hidden]
options = []
# subcommand
try:
subcommand_name = [w for w in cwords if w in subcommands][0]
except IndexError:
subcommand_name = None
# subcommand options
if subcommand_name:
# special case: 'help' subcommand has no options
if subcommand_name == 'help':
sys.exit(1)
# special case: list locally installed dists for uninstall command
if subcommand_name == 'uninstall' and not current.startswith('-'):
installed = []
lc = current.lower()
for dist in get_installed_distributions(local_only=True):
if dist.key.startswith(lc) and dist.key not in cwords[1:]:
installed.append(dist.key)
# if there are no dists installed, fall back to option completion
if installed:
for dist in installed:
print(dist)
sys.exit(1)
subcommand = command_dict.get(subcommand_name)
options += [(opt.get_opt_string(), opt.nargs)
for opt in subcommand.parser.option_list
if opt.help != optparse.SUPPRESS_HELP]
# filter out previously specified options from available options
prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]]
options = [(x, v) for (x, v) in options if x not in prev_opts]
# filter options by current input
options = [(k, v) for k, v in options if k.startswith(current)]
for option in options:
opt_label = option[0]
# append '=' to options which require args
if option[1]:
opt_label += '='
print(opt_label)
else:
# show options of main parser only when necessary
if current.startswith('-') or current.startswith('--'):
subcommands += [opt.get_opt_string()
for opt in parser.option_list
if opt.help != optparse.SUPPRESS_HELP]
print(' '.join([x for x in subcommands if x.startswith(current)]))
sys.exit(1)
def version_control():
# Import all the version control support modules:
from pip import vcs
for importer, modname, ispkg in \
walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'):
__import__(modname)
def main(initial_args=None):
if initial_args is None:
initial_args = sys.argv[1:]
autocomplete()
version_control()
options, args = parser.parse_args(initial_args)
if options.help and not args:
args = ['help']
if not args:
parser.error('You must give a command (use "pip help" to see a list of commands)')
command = args[0].lower()
load_command(command)
if command not in command_dict:
close_commands = difflib.get_close_matches(command, command_names())
if close_commands:
guess = close_commands[0]
if args[1:]:
guess = "%s %s" % (guess, " ".join(args[1:]))
else:
guess = 'install %s' % command
error_dict = {'arg': command, 'guess': guess,
'script': os.path.basename(sys.argv[0])}
parser.error('No command by the name %(script)s %(arg)s\n '
'(maybe you meant "%(script)s %(guess)s")' % error_dict)
command = command_dict[command]
return command.main(initial_args, args[1:], options)
def bootstrap():
"""
Bootstrapping function to be called from install-pip.py script.
"""
return main(['install', '--upgrade', 'pip'])
############################################################
## Writing freeze files
class FrozenRequirement(object):
def __init__(self, name, req, editable, comments=()):
self.name = name
self.req = req
self.editable = editable
self.comments = comments
_rev_re = re.compile(r'-r(\d+)$')
_date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
@classmethod
def from_dist(cls, dist, dependency_links, find_tags=False):
location = os.path.normcase(os.path.abspath(dist.location))
comments = []
from pip.vcs import vcs, get_src_requirement
if vcs.get_backend_name(location):
editable = True
req = get_src_requirement(dist, location, find_tags)
if req is None:
logger.warn('Could not determine repository location of %s' % location)
comments.append('## !! Could not determine repository location')
req = dist.as_requirement()
editable = False
else:
editable = False
req = dist.as_requirement()
specs = req.specs
assert len(specs) == 1 and specs[0][0] == '=='
version = specs[0][1]
ver_match = cls._rev_re.search(version)
date_match = cls._date_re.search(version)
if ver_match or date_match:
svn_backend = vcs.get_backend('svn')
if svn_backend:
svn_location = svn_backend(
).get_location(dist, dependency_links)
if not svn_location:
logger.warn(
'Warning: cannot find svn location for %s' % req)
comments.append('## FIXME: could not find svn URL in dependency_links for this package:')
else:
comments.append('# Installing as editable to satisfy requirement %s:' % req)
if ver_match:
rev = ver_match.group(1)
else:
rev = '{%s}' % date_match.group(1)
editable = True
req = '%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist))
return cls(dist.project_name, req, editable, comments)
@staticmethod
def egg_name(dist):
name = dist.egg_name()
match = re.search(r'-py\d\.\d$', name)
if match:
name = name[:match.start()]
return name
def __str__(self):
req = self.req
if self.editable:
req = '-e %s' % req
return '\n'.join(list(self.comments)+[str(req)])+'\n'
############################################################
## Requirement files
def call_subprocess(cmd, show_stdout=True,
filter_stdout=None, cwd=None,
raise_on_returncode=True,
command_level=logger.DEBUG, command_desc=None,
extra_environ=None):
if command_desc is None:
cmd_parts = []
for part in cmd:
if ' ' in part or '\n' in part or '"' in part or "'" in part:
part = '"%s"' % part.replace('"', '\\"')
cmd_parts.append(part)
command_desc = ' '.join(cmd_parts)
if show_stdout:
stdout = None
else:
stdout = subprocess.PIPE
logger.log(command_level, "Running command %s" % command_desc)
env = os.environ.copy()
if extra_environ:
env.update(extra_environ)
try:
proc = subprocess.Popen(
cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
cwd=cwd, env=env)
except Exception:
e = sys.exc_info()[1]
logger.fatal(
"Error %s while executing command %s" % (e, command_desc))
raise
all_output = []
if stdout is not None:
stdout = proc.stdout
while 1:
line = console_to_str(stdout.readline())
if not line:
break
line = line.rstrip()
all_output.append(line + '\n')
if filter_stdout:
level = filter_stdout(line)
if isinstance(level, tuple):
level, line = level
logger.log(level, line)
if not logger.stdout_level_matches(level):
logger.show_progress()
else:
logger.info(line)
else:
returned_stdout, returned_stderr = proc.communicate()
all_output = [returned_stdout or '']
proc.wait()
if proc.returncode:
if raise_on_returncode:
if all_output:
logger.notify('Complete output from command %s:' % command_desc)
logger.notify('\n'.join(all_output) + '\n----------------------------------------')
raise InstallationError(
"Command %s failed with error code %s"
% (command_desc, proc.returncode))
else:
logger.warn(
"Command %s had error code %s"
% (command_desc, proc.returncode))
if stdout is not None:
return ''.join(all_output)
if __name__ == '__main__':
exit = main()
if exit:
sys.exit(exit)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/_pkgutil.py
-592
View File
@@ -1,592 +0,0 @@
"""Utilities to support packages."""
# NOTE: This module must remain compatible with Python 2.3, as it is shared
# by setuptools for distribution with Python 2.3 and up.
import os
import sys
import imp
import os.path
from types import ModuleType
__all__ = [
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
'walk_packages', 'iter_modules',
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]
def read_code(stream):
# This helper is needed in order for the PEP 302 emulation to
# correctly handle compiled files
import marshal
magic = stream.read(4)
if magic != imp.get_magic():
return None
stream.read(4) # Skip timestamp
return marshal.load(stream)
def simplegeneric(func):
"""Make a trivial single-dispatch generic function"""
registry = {}
def wrapper(*args, **kw):
ob = args[0]
try:
cls = ob.__class__
except AttributeError:
cls = type(ob)
try:
mro = cls.__mro__
except AttributeError:
try:
class cls(cls, object):
pass
mro = cls.__mro__[1:]
except TypeError:
mro = object, # must be an ExtensionClass or some such :(
for t in mro:
if t in registry:
return registry[t](*args, **kw)
else:
return func(*args, **kw)
try:
wrapper.__name__ = func.__name__
except (TypeError, AttributeError):
pass # Python 2.3 doesn't allow functions to be renamed
def register(typ, func=None):
if func is None:
return lambda f: register(typ, f)
registry[typ] = func
return func
wrapper.__dict__ = func.__dict__
wrapper.__doc__ = func.__doc__
wrapper.register = register
return wrapper
def walk_packages(path=None, prefix='', onerror=None):
"""Yields (module_loader, name, ispkg) for all modules recursively
on path, or, if path is None, all accessible modules.
'path' should be either None or a list of paths to look for
modules in.
'prefix' is a string to output on the front of every module name
on output.
Note that this function must import all *packages* (NOT all
modules!) on the given path, in order to access the __path__
attribute to find submodules.
'onerror' is a function which gets called with one argument (the
name of the package which was being imported) if any exception
occurs while trying to import a package. If no onerror function is
supplied, ImportErrors are caught and ignored, while all other
exceptions are propagated, terminating the search.
Examples:
# list all modules python can access
walk_packages()
# list all submodules of ctypes
walk_packages(ctypes.__path__, ctypes.__name__+'.')
"""
def seen(p, m={}):
if p in m:
return True
m[p] = True
for importer, name, ispkg in iter_modules(path, prefix):
yield importer, name, ispkg
if ispkg:
try:
__import__(name)
except ImportError:
if onerror is not None:
onerror(name)
except Exception:
if onerror is not None:
onerror(name)
else:
raise
else:
path = getattr(sys.modules[name], '__path__', None) or []
# don't traverse path items we've seen before
path = [p for p in path if not seen(p)]
for item in walk_packages(path, name+'.', onerror):
yield item
def iter_modules(path=None, prefix=''):
"""Yields (module_loader, name, ispkg) for all submodules on path,
or, if path is None, all top-level modules on sys.path.
'path' should be either None or a list of paths to look for
modules in.
'prefix' is a string to output on the front of every module name
on output.
"""
if path is None:
importers = iter_importers()
else:
importers = map(get_importer, path)
yielded = {}
for i in importers:
for name, ispkg in iter_importer_modules(i, prefix):
if name not in yielded:
yielded[name] = 1
yield i, name, ispkg
#@simplegeneric
def iter_importer_modules(importer, prefix=''):
if not hasattr(importer, 'iter_modules'):
return []
return importer.iter_modules(prefix)
iter_importer_modules = simplegeneric(iter_importer_modules)
class ImpImporter:
"""PEP 302 Importer that wraps Python's "classic" import algorithm
ImpImporter(dirname) produces a PEP 302 importer that searches that
directory. ImpImporter(None) produces a PEP 302 importer that searches
the current sys.path, plus any modules that are frozen or built-in.
Note that ImpImporter does not currently support being used by placement
on sys.meta_path.
"""
def __init__(self, path=None):
self.path = path
def find_module(self, fullname, path=None):
# Note: we ignore 'path' argument since it is only used via meta_path
subname = fullname.split(".")[-1]
if subname != fullname and self.path is None:
return None
if self.path is None:
path = None
else:
path = [os.path.realpath(self.path)]
try:
file, filename, etc = imp.find_module(subname, path)
except ImportError:
return None
return ImpLoader(fullname, file, filename, etc)
def iter_modules(self, prefix=''):
if self.path is None or not os.path.isdir(self.path):
return
yielded = {}
import inspect
filenames = os.listdir(self.path)
filenames.sort() # handle packages before same-named modules
for fn in filenames:
modname = inspect.getmodulename(fn)
if modname=='__init__' or modname in yielded:
continue
path = os.path.join(self.path, fn)
ispkg = False
if not modname and os.path.isdir(path) and '.' not in fn:
modname = fn
for fn in os.listdir(path):
subname = inspect.getmodulename(fn)
if subname=='__init__':
ispkg = True
break
else:
continue # not a package
if modname and '.' not in modname:
yielded[modname] = 1
yield prefix + modname, ispkg
class ImpLoader:
"""PEP 302 Loader that wraps Python's "classic" import algorithm
"""
code = source = None
def __init__(self, fullname, file, filename, etc):
self.file = file
self.filename = filename
self.fullname = fullname
self.etc = etc
def load_module(self, fullname):
self._reopen()
try:
mod = imp.load_module(fullname, self.file, self.filename, self.etc)
finally:
if self.file:
self.file.close()
# Note: we don't set __loader__ because we want the module to look
# normal; i.e. this is just a wrapper for standard import machinery
return mod
def get_data(self, pathname):
return open(pathname, "rb").read()
def _reopen(self):
if self.file and self.file.closed:
mod_type = self.etc[2]
if mod_type==imp.PY_SOURCE:
self.file = open(self.filename, 'rU')
elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
self.file = open(self.filename, 'rb')
def _fix_name(self, fullname):
if fullname is None:
fullname = self.fullname
elif fullname != self.fullname:
raise ImportError("Loader for module %s cannot handle "
"module %s" % (self.fullname, fullname))
return fullname
def is_package(self, fullname):
fullname = self._fix_name(fullname)
return self.etc[2]==imp.PKG_DIRECTORY
def get_code(self, fullname=None):
fullname = self._fix_name(fullname)
if self.code is None:
mod_type = self.etc[2]
if mod_type==imp.PY_SOURCE:
source = self.get_source(fullname)
self.code = compile(source, self.filename, 'exec')
elif mod_type==imp.PY_COMPILED:
self._reopen()
try:
self.code = read_code(self.file)
finally:
self.file.close()
elif mod_type==imp.PKG_DIRECTORY:
self.code = self._get_delegate().get_code()
return self.code
def get_source(self, fullname=None):
fullname = self._fix_name(fullname)
if self.source is None:
mod_type = self.etc[2]
if mod_type==imp.PY_SOURCE:
self._reopen()
try:
self.source = self.file.read()
finally:
self.file.close()
elif mod_type==imp.PY_COMPILED:
if os.path.exists(self.filename[:-1]):
f = open(self.filename[:-1], 'rU')
self.source = f.read()
f.close()
elif mod_type==imp.PKG_DIRECTORY:
self.source = self._get_delegate().get_source()
return self.source
def _get_delegate(self):
return ImpImporter(self.filename).find_module('__init__')
def get_filename(self, fullname=None):
fullname = self._fix_name(fullname)
mod_type = self.etc[2]
if self.etc[2]==imp.PKG_DIRECTORY:
return self._get_delegate().get_filename()
elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
return self.filename
return None
try:
import zipimport
from zipimport import zipimporter
def iter_zipimport_modules(importer, prefix=''):
dirlist = list(zipimport._zip_directory_cache[importer.archive].keys())
dirlist.sort()
_prefix = importer.prefix
plen = len(_prefix)
yielded = {}
import inspect
for fn in dirlist:
if not fn.startswith(_prefix):
continue
fn = fn[plen:].split(os.sep)
if len(fn)==2 and fn[1].startswith('__init__.py'):
if fn[0] not in yielded:
yielded[fn[0]] = 1
yield fn[0], True
if len(fn)!=1:
continue
modname = inspect.getmodulename(fn[0])
if modname=='__init__':
continue
if modname and '.' not in modname and modname not in yielded:
yielded[modname] = 1
yield prefix + modname, False
iter_importer_modules.register(zipimporter, iter_zipimport_modules)
except ImportError:
pass
def get_importer(path_item):
"""Retrieve a PEP 302 importer for the given path item
The returned importer is cached in sys.path_importer_cache
if it was newly created by a path hook.
If there is no importer, a wrapper around the basic import
machinery is returned. This wrapper is never inserted into
the importer cache (None is inserted instead).
The cache (or part of it) can be cleared manually if a
rescan of sys.path_hooks is necessary.
"""
try:
importer = sys.path_importer_cache[path_item]
except KeyError:
for path_hook in sys.path_hooks:
try:
importer = path_hook(path_item)
break
except ImportError:
pass
else:
importer = None
sys.path_importer_cache.setdefault(path_item, importer)
if importer is None:
try:
importer = ImpImporter(path_item)
except ImportError:
importer = None
return importer
def iter_importers(fullname=""):
"""Yield PEP 302 importers for the given module name
If fullname contains a '.', the importers will be for the package
containing fullname, otherwise they will be importers for sys.meta_path,
sys.path, and Python's "classic" import machinery, in that order. If
the named module is in a package, that package is imported as a side
effect of invoking this function.
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
standard import machinery to find files in alternative locations
are partially supported, but are searched AFTER sys.path. Normally,
these locations are searched BEFORE sys.path, preventing sys.path
entries from shadowing them.
For this to cause a visible difference in behaviour, there must
be a module or package name that is accessible via both sys.path
and one of the non PEP 302 file system mechanisms. In this case,
the emulation will find the former version, while the builtin
import mechanism will find the latter.
Items of the following types can be affected by this discrepancy:
imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
"""
if fullname.startswith('.'):
raise ImportError("Relative module names not supported")
if '.' in fullname:
# Get the containing package's __path__
pkg = '.'.join(fullname.split('.')[:-1])
if pkg not in sys.modules:
__import__(pkg)
path = getattr(sys.modules[pkg], '__path__', None) or []
else:
for importer in sys.meta_path:
yield importer
path = sys.path
for item in path:
yield get_importer(item)
if '.' not in fullname:
yield ImpImporter()
def get_loader(module_or_name):
"""Get a PEP 302 "loader" object for module_or_name
If the module or package is accessible via the normal import
mechanism, a wrapper around the relevant part of that machinery
is returned. Returns None if the module cannot be found or imported.
If the named module is not already imported, its containing package
(if any) is imported, in order to establish the package __path__.
This function uses iter_importers(), and is thus subject to the same
limitations regarding platform-specific special import locations such
as the Windows registry.
"""
if module_or_name in sys.modules:
module_or_name = sys.modules[module_or_name]
if isinstance(module_or_name, ModuleType):
module = module_or_name
loader = getattr(module, '__loader__', None)
if loader is not None:
return loader
fullname = module.__name__
else:
fullname = module_or_name
return find_loader(fullname)
def find_loader(fullname):
"""Find a PEP 302 "loader" object for fullname
If fullname contains dots, path must be the containing package's __path__.
Returns None if the module cannot be found or imported. This function uses
iter_importers(), and is thus subject to the same limitations regarding
platform-specific special import locations such as the Windows registry.
"""
for importer in iter_importers(fullname):
loader = importer.find_module(fullname)
if loader is not None:
return loader
return None
def extend_path(path, name):
"""Extend a package's path.
Intended use is to place the following code in a package's __init__.py:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
This will add to the package's __path__ all subdirectories of
directories on sys.path named after the package. This is useful
if one wants to distribute different parts of a single logical
package as multiple directories.
It also looks for *.pkg files beginning where * matches the name
argument. This feature is similar to *.pth files (see site.py),
except that it doesn't special-case lines starting with 'import'.
A *.pkg file is trusted at face value: apart from checking for
duplicates, all entries found in a *.pkg file are added to the
path, regardless of whether they are exist the filesystem. (This
is a feature.)
If the input path is not a list (as is the case for frozen
packages) it is returned unchanged. The input path is not
modified; an extended copy is returned. Items are only appended
to the copy at the end.
It is assumed that sys.path is a sequence. Items of sys.path that
are not (unicode or 8-bit) strings referring to existing
directories are ignored. Unicode items of sys.path that cause
errors when used as filenames may cause this function to raise an
exception (in line with os.path.isdir() behavior).
"""
if not isinstance(path, list):
# This could happen e.g. when this is called from inside a
# frozen package. Return the path unchanged in that case.
return path
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
# Just in case os.extsep != '.'
sname = os.extsep.join(name.split('.'))
sname_pkg = sname + os.extsep + "pkg"
init_py = "__init__" + os.extsep + "py"
path = path[:] # Start with a copy of the existing path
from pip.backwardcompat import string_types
for dir in sys.path:
if not isinstance(dir, string_types) or not os.path.isdir(dir):
continue
subdir = os.path.join(dir, pname)
# XXX This may still add duplicate entries to path on
# case-insensitive filesystems
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
# XXX Is this the right thing for subpackages like zope.app?
# It looks for a file named "zope.app.pkg"
pkgfile = os.path.join(dir, sname_pkg)
if os.path.isfile(pkgfile):
try:
f = open(pkgfile)
except IOError:
msg = sys.exc_info()[1]
sys.stderr.write("Can't open %s: %s\n" %
(pkgfile, msg))
else:
for line in f:
line = line.rstrip('\n')
if not line or line.startswith('#'):
continue
path.append(line) # Don't check for existence!
f.close()
return path
def get_data(package, resource):
"""Get a resource from a package.
This is a wrapper round the PEP 302 loader get_data API. The package
argument should be the name of a package, in standard module format
(foo.bar). The resource argument should be in the form of a relative
filename, using '/' as the path separator. The parent directory name '..'
is not allowed, and nor is a rooted name (starting with a '/').
The function returns a binary string, which is the contents of the
specified resource.
For packages located in the filesystem, which have already been imported,
this is the rough equivalent of
d = os.path.dirname(sys.modules[package].__file__)
data = open(os.path.join(d, resource), 'rb').read()
If the package cannot be located or loaded, or it uses a PEP 302 loader
which does not support get_data(), then None is returned.
"""
loader = get_loader(package)
if loader is None or not hasattr(loader, 'get_data'):
return None
mod = sys.modules.get(package) or loader.load_module(package)
if mod is None or not hasattr(mod, '__file__'):
return None
# Modify the resource name to be compatible with the loader.get_data
# signature - an os.path format "filename" starting with the dirname of
# the package's __file__
parts = resource.split('/')
parts.insert(0, os.path.dirname(mod.__file__))
resource_name = os.path.join(*parts)
return loader.get_data(resource_name)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/backwardcompat.py
-115
View File
@@ -1,115 +0,0 @@
"""Stuff that differs in different Python versions"""
import sys
import os
import shutil
__all__ = ['any', 'WindowsError', 'md5', 'copytree']
try:
WindowsError = WindowsError
except NameError:
class NeverUsedException(Exception):
"""this exception should never be raised"""
WindowsError = NeverUsedException
try:
from hashlib import md5
except ImportError:
import md5 as md5_module
md5 = md5_module.new
try:
from pkgutil import walk_packages
except ImportError:
# let's fall back as long as we can
from pip._pkgutil import walk_packages
try:
any = any
except NameError:
def any(seq):
for item in seq:
if item:
return True
return False
console_encoding = sys.__stdout__.encoding
if sys.version_info >= (3,):
from io import StringIO
from functools import reduce
from urllib.error import URLError, HTTPError
from queue import Queue, Empty
from urllib.request import url2pathname
from urllib.request import urlretrieve
from email import message as emailmessage
import urllib.parse as urllib
import urllib.request as urllib2
import configparser as ConfigParser
import xmlrpc.client as xmlrpclib
import urllib.parse as urlparse
import http.client as httplib
def cmp(a, b):
return (a > b) - (a < b)
def b(s):
return s.encode('utf-8')
def u(s):
return s.decode('utf-8')
def console_to_str(s):
return s.decode(console_encoding)
bytes = bytes
string_types = (str,)
raw_input = input
else:
from cStringIO import StringIO
from urllib2 import URLError, HTTPError
from Queue import Queue, Empty
from urllib import url2pathname, urlretrieve
from email import Message as emailmessage
import urllib
import urllib2
import urlparse
import ConfigParser
import xmlrpclib
import httplib
def b(s):
return s
def u(s):
return s
def console_to_str(s):
return s
bytes = str
string_types = (basestring,)
reduce = reduce
cmp = cmp
raw_input = raw_input
try:
from email.parser import FeedParser
except ImportError:
# python lesser than 2.5
from email.FeedParser import FeedParser
from distutils.sysconfig import get_python_lib, get_python_version
def copytree(src, dst):
if sys.version_info < (2, 5):
before_last_dir = os.path.dirname(dst)
if not os.path.exists(before_last_dir):
os.makedirs(before_last_dir)
shutil.copytree(src, dst)
shutil.copymode(src, dst)
else:
shutil.copytree(src, dst)
def product(*args, **kwds):
# product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
# product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
pools = list(map(tuple, args)) * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
result = [x+[y] for x in result for y in pool]
for prod in result:
yield tuple(prod)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/basecommand.py
-205
View File
@@ -1,205 +0,0 @@
"""Base Command class, and related routines"""
import os
import socket
import sys
import traceback
import time
from pip import commands
from pip.log import logger
from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip.download import urlopen
from pip.exceptions import BadCommand, InstallationError, UninstallationError
from pip.venv import restart_in_venv
from pip.backwardcompat import StringIO, urllib, urllib2, walk_packages
__all__ = ['command_dict', 'Command', 'load_all_commands',
'load_command', 'command_names']
command_dict = {}
# for backwards compatibiliy
get_proxy = urlopen.get_proxy
class Command(object):
name = None
usage = None
hidden = False
def __init__(self):
assert self.name
self.parser = ConfigOptionParser(
usage=self.usage,
prog='%s %s' % (sys.argv[0], self.name),
version=parser.version,
formatter=UpdatingDefaultsHelpFormatter(),
name=self.name)
for option in parser.option_list:
if not option.dest or option.dest == 'help':
# -h, --version, etc
continue
self.parser.add_option(option)
command_dict[self.name] = self
def merge_options(self, initial_options, options):
# Make sure we have all global options carried over
for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv',
'respect_venv', 'log_explicit_levels', 'log_file',
'timeout', 'default_vcs', 'skip_requirements_regex',
'no_input']:
setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr))
options.quiet += initial_options.quiet
options.verbose += initial_options.verbose
def setup_logging(self):
pass
def main(self, complete_args, args, initial_options):
options, args = self.parser.parse_args(args)
self.merge_options(initial_options, options)
level = 1 # Notify
level += options.verbose
level -= options.quiet
level = logger.level_for_integer(4-level)
complete_log = []
logger.consumers.extend(
[(level, sys.stdout),
(logger.DEBUG, complete_log.append)])
if options.log_explicit_levels:
logger.explicit_levels = True
self.setup_logging()
if options.require_venv and not options.venv:
# If a venv is required check if it can really be found
if not os.environ.get('VIRTUAL_ENV'):
logger.fatal('Could not find an activated virtualenv (required).')
sys.exit(3)
# Automatically install in currently activated venv if required
options.respect_venv = True
if args and args[-1] == '___VENV_RESTART___':
## FIXME: We don't do anything this this value yet:
args = args[:-2]
options.venv = None
else:
# If given the option to respect the activated environment
# check if no venv is given as a command line parameter
if options.respect_venv and os.environ.get('VIRTUAL_ENV'):
if options.venv and os.path.exists(options.venv):
# Make sure command line venv and environmental are the same
if (os.path.realpath(os.path.expanduser(options.venv)) !=
os.path.realpath(os.environ.get('VIRTUAL_ENV'))):
logger.fatal("Given virtualenv (%s) doesn't match "
"currently activated virtualenv (%s)."
% (options.venv, os.environ.get('VIRTUAL_ENV')))
sys.exit(3)
else:
options.venv = os.environ.get('VIRTUAL_ENV')
logger.info('Using already activated environment %s' % options.venv)
if options.venv:
logger.info('Running in environment %s' % options.venv)
site_packages=False
if options.site_packages:
site_packages=True
restart_in_venv(options.venv, options.venv_base, site_packages,
complete_args)
# restart_in_venv should actually never return, but for clarity...
return
## FIXME: not sure if this sure come before or after venv restart
if options.log:
log_fp = open_logfile(options.log, 'a')
logger.consumers.append((logger.DEBUG, log_fp))
else:
log_fp = None
socket.setdefaulttimeout(options.timeout or None)
urlopen.setup(proxystr=options.proxy, prompting=not options.no_input)
exit = 0
try:
self.run(options, args)
except (InstallationError, UninstallationError):
e = sys.exc_info()[1]
logger.fatal(str(e))
logger.info('Exception information:\n%s' % format_exc())
exit = 1
except BadCommand:
e = sys.exc_info()[1]
logger.fatal(str(e))
logger.info('Exception information:\n%s' % format_exc())
exit = 1
except KeyboardInterrupt:
logger.fatal('Operation cancelled by user')
logger.info('Exception information:\n%s' % format_exc())
exit = 1
except:
logger.fatal('Exception:\n%s' % format_exc())
exit = 2
if log_fp is not None:
log_fp.close()
if exit:
log_fn = options.log_file
text = '\n'.join(complete_log)
logger.fatal('Storing complete log in %s' % log_fn)
log_fp = open_logfile(log_fn, 'w')
log_fp.write(text)
log_fp.close()
return exit
def format_exc(exc_info=None):
if exc_info is None:
exc_info = sys.exc_info()
out = StringIO()
traceback.print_exception(*exc_info, **dict(file=out))
return out.getvalue()
def open_logfile(filename, mode='a'):
"""Open the named log file in append mode.
If the file already exists, a separator will also be printed to
the file to separate past activity from current activity.
"""
filename = os.path.expanduser(filename)
filename = os.path.abspath(filename)
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
exists = os.path.exists(filename)
log_fp = open(filename, mode)
if exists:
log_fp.write('%s\n' % ('-'*60))
log_fp.write('%s run on %s\n' % (sys.argv[0], time.strftime('%c')))
return log_fp
def load_command(name):
full_name = 'pip.commands.%s' % name
if full_name in sys.modules:
return
try:
__import__(full_name)
except ImportError:
pass
def load_all_commands():
for name in command_names():
load_command(name)
def command_names():
names = set((pkg[1] for pkg in walk_packages(path=commands.__path__)))
return list(names)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/baseparser.py
-232
View File
@@ -1,232 +0,0 @@
"""Base option parser setup"""
import sys
import optparse
import pkg_resources
import os
from distutils.util import strtobool
from pip.backwardcompat import ConfigParser, string_types
from pip.locations import default_config_file, default_log_file
class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
"""Custom help formatter for use in ConfigOptionParser that updates
the defaults before expanding them, allowing them to show up correctly
in the help listing"""
def expand_default(self, option):
if self.parser is not None:
self.parser.update_defaults(self.parser.defaults)
return optparse.IndentedHelpFormatter.expand_default(self, option)
class ConfigOptionParser(optparse.OptionParser):
"""Custom option parser which updates its defaults by by checking the
configuration files and environmental variables"""
def __init__(self, *args, **kwargs):
self.config = ConfigParser.RawConfigParser()
self.name = kwargs.pop('name')
self.files = self.get_config_files()
self.config.read(self.files)
assert self.name
optparse.OptionParser.__init__(self, *args, **kwargs)
def get_config_files(self):
config_file = os.environ.get('PIP_CONFIG_FILE', False)
if config_file and os.path.exists(config_file):
return [config_file]
return [default_config_file]
def update_defaults(self, defaults):
"""Updates the given defaults with values from the config files and
the environ. Does a little special handling for certain types of
options (lists)."""
# Then go and look for the other sources of configuration:
config = {}
# 1. config files
for section in ('global', self.name):
config.update(dict(self.get_config_section(section)))
# 2. environmental variables
config.update(dict(self.get_environ_vars()))
# Then set the options with those values
for key, val in config.items():
key = key.replace('_', '-')
if not key.startswith('--'):
key = '--%s' % key # only prefer long opts
option = self.get_option(key)
if option is not None:
# ignore empty values
if not val:
continue
# handle multiline configs
if option.action == 'append':
val = val.split()
else:
option.nargs = 1
if option.action in ('store_true', 'store_false', 'count'):
val = strtobool(val)
try:
val = option.convert_value(key, val)
except optparse.OptionValueError:
e = sys.exc_info()[1]
print("An error occured during configuration: %s" % e)
sys.exit(3)
defaults[option.dest] = val
return defaults
def get_config_section(self, name):
"""Get a section of a configuration"""
if self.config.has_section(name):
return self.config.items(name)
return []
def get_environ_vars(self, prefix='PIP_'):
"""Returns a generator with all environmental vars with prefix PIP_"""
for key, val in os.environ.items():
if key.startswith(prefix):
yield (key.replace(prefix, '').lower(), val)
def get_default_values(self):
"""Overridding to make updating the defaults after instantiation of
the option parser possible, update_defaults() does the dirty work."""
if not self.process_default_values:
# Old, pre-Optik 1.5 behaviour.
return optparse.Values(self.defaults)
defaults = self.update_defaults(self.defaults.copy()) # ours
for option in self._get_all_options():
default = defaults.get(option.dest)
if isinstance(default, string_types):
opt_str = option.get_opt_string()
defaults[option.dest] = option.check_value(opt_str, default)
return optparse.Values(defaults)
try:
pip_dist = pkg_resources.get_distribution('pip')
version = '%s from %s (python %s)' % (
pip_dist, pip_dist.location, sys.version[:3])
except pkg_resources.DistributionNotFound:
# when running pip.py without installing
version=None
parser = ConfigOptionParser(
usage='%prog COMMAND [OPTIONS]',
version=version,
add_help_option=False,
formatter=UpdatingDefaultsHelpFormatter(),
name='global')
parser.add_option(
'-h', '--help',
dest='help',
action='store_true',
help='Show help')
parser.add_option(
'-E', '--environment',
dest='venv',
metavar='DIR',
help='virtualenv environment to run pip in (either give the '
'interpreter or the environment base directory)')
parser.add_option(
'-s', '--enable-site-packages',
dest='site_packages',
action='store_true',
help='Include site-packages in virtualenv if one is to be '
'created. Ignored if --environment is not used or '
'the virtualenv already exists.')
parser.add_option(
# Defines a default root directory for virtualenvs, relative
# virtualenvs names/paths are considered relative to it.
'--virtualenv-base',
dest='venv_base',
type='str',
default='',
help=optparse.SUPPRESS_HELP)
parser.add_option(
# Run only if inside a virtualenv, bail if not.
'--require-virtualenv', '--require-venv',
dest='require_venv',
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option(
# Use automatically an activated virtualenv instead of installing
# globally. -E will be ignored if used.
'--respect-virtualenv', '--respect-venv',
dest='respect_venv',
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option(
'-v', '--verbose',
dest='verbose',
action='count',
default=0,
help='Give more output')
parser.add_option(
'-q', '--quiet',
dest='quiet',
action='count',
default=0,
help='Give less output')
parser.add_option(
'--log',
dest='log',
metavar='FILENAME',
help='Log file where a complete (maximum verbosity) record will be kept')
parser.add_option(
# Writes the log levels explicitely to the log'
'--log-explicit-levels',
dest='log_explicit_levels',
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option(
# The default log file
'--local-log', '--log-file',
dest='log_file',
metavar='FILENAME',
default=default_log_file,
help=optparse.SUPPRESS_HELP)
parser.add_option(
# Don't ask for input
'--no-input',
dest='no_input',
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option(
'--proxy',
dest='proxy',
type='str',
default='',
help="Specify a proxy in the form user:passwd@proxy.server:port. "
"Note that the user:password@ is optional and required only if you "
"are behind an authenticated proxy. If you provide "
"user@proxy.server:port then you will be prompted for a password.")
parser.add_option(
'--timeout', '--default-timeout',
metavar='SECONDS',
dest='timeout',
type='float',
default=15,
help='Set the socket timeout (default %default seconds)')
parser.add_option(
# The default version control system for editables, e.g. 'svn'
'--default-vcs',
dest='default_vcs',
type='str',
default='',
help=optparse.SUPPRESS_HELP)
parser.add_option(
# A regex to be used to skip requirements
'--skip-requirements-regex',
dest='skip_requirements_regex',
type='str',
default='',
help=optparse.SUPPRESS_HELP)
parser.disable_interspersed_args()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/__init__.py
-1
View File
@@ -1 +0,0 @@
#
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/bundle.py
-33
View File
@@ -1,33 +0,0 @@
from pip.locations import build_prefix, src_prefix
from pip.util import display_path, backup_dir
from pip.log import logger
from pip.exceptions import InstallationError
from pip.commands.install import InstallCommand
class BundleCommand(InstallCommand):
name = 'bundle'
usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...'
summary = 'Create pybundles (archives containing multiple packages)'
bundle = True
def __init__(self):
super(BundleCommand, self).__init__()
def run(self, options, args):
if not args:
raise InstallationError('You must give a bundle filename')
if not options.build_dir:
options.build_dir = backup_dir(build_prefix, '-bundle')
if not options.src_dir:
options.src_dir = backup_dir(src_prefix, '-bundle')
# We have to get everything when creating a bundle:
options.ignore_installed = True
logger.notify('Putting temporary build files in %s and source/develop files in %s'
% (display_path(options.build_dir), display_path(options.src_dir)))
self.bundle_filename = args.pop(0)
requirement_set = super(BundleCommand, self).run(options, args)
return requirement_set
BundleCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/completion.py
-60
View File
@@ -1,60 +0,0 @@
import sys
from pip.basecommand import Command
BASE_COMPLETION = """
# pip %(shell)s completion start%(script)s# pip %(shell)s completion end
"""
COMPLETION_SCRIPTS = {
'bash': """
_pip_completion()
{
COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\
COMP_CWORD=$COMP_CWORD \\
PIP_AUTO_COMPLETE=1 $1 ) )
}
complete -o default -F _pip_completion pip
""", 'zsh': """
function _pip_completion {
local words cword
read -Ac words
read -cn cword
reply=( $( COMP_WORDS="$words[*]" \\
COMP_CWORD=$(( cword-1 )) \\
PIP_AUTO_COMPLETE=1 $words[1] ) )
}
compctl -K _pip_completion pip
"""}
class CompletionCommand(Command):
name = 'completion'
summary = 'A helper command to be used for command completion'
hidden = True
def __init__(self):
super(CompletionCommand, self).__init__()
self.parser.add_option(
'--bash', '-b',
action='store_const',
const='bash',
dest='shell',
help='Emit completion code for bash')
self.parser.add_option(
'--zsh', '-z',
action='store_const',
const='zsh',
dest='shell',
help='Emit completion code for zsh')
def run(self, options, args):
"""Prints the completion code of the given shell"""
shells = COMPLETION_SCRIPTS.keys()
shell_options = ['--'+shell for shell in sorted(shells)]
if options.shell in shells:
script = COMPLETION_SCRIPTS.get(options.shell, '')
print(BASE_COMPLETION % {'script': script, 'shell': options.shell})
else:
sys.stderr.write('ERROR: You must pass %s\n' % ' or '.join(shell_options))
CompletionCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/freeze.py
-109
View File
@@ -1,109 +0,0 @@
import re
import sys
import pkg_resources
import pip
from pip.req import InstallRequirement
from pip.log import logger
from pip.basecommand import Command
from pip.util import get_installed_distributions
class FreezeCommand(Command):
name = 'freeze'
usage = '%prog [OPTIONS]'
summary = 'Output all currently installed packages (exact versions) to stdout'
def __init__(self):
super(FreezeCommand, self).__init__()
self.parser.add_option(
'-r', '--requirement',
dest='requirement',
action='store',
default=None,
metavar='FILENAME',
help='Use the given requirements file as a hint about how to generate the new frozen requirements')
self.parser.add_option(
'-f', '--find-links',
dest='find_links',
action='append',
default=[],
metavar='URL',
help='URL for finding packages, which will be added to the frozen requirements file')
self.parser.add_option(
'-l', '--local',
dest='local',
action='store_true',
default=False,
help='If in a virtualenv, do not report globally-installed packages')
def setup_logging(self):
logger.move_stdout_to_stderr()
def run(self, options, args):
requirement = options.requirement
find_links = options.find_links or []
local_only = options.local
## FIXME: Obviously this should be settable:
find_tags = False
skip_match = None
skip_regex = options.skip_requirements_regex
if skip_regex:
skip_match = re.compile(skip_regex)
dependency_links = []
f = sys.stdout
for dist in pkg_resources.working_set:
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
for link in find_links:
if '#egg=' in link:
dependency_links.append(link)
for link in find_links:
f.write('-f %s\n' % link)
installations = {}
for dist in get_installed_distributions(local_only=local_only):
req = pip.FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags)
installations[req.name] = req
if requirement:
req_f = open(requirement)
for line in req_f:
if not line.strip() or line.strip().startswith('#'):
f.write(line)
continue
if skip_match and skip_match.search(line):
f.write(line)
continue
elif line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs)
elif (line.startswith('-r') or line.startswith('--requirement')
or line.startswith('-Z') or line.startswith('--always-unzip')
or line.startswith('-f') or line.startswith('-i')
or line.startswith('--extra-index-url')):
f.write(line)
continue
else:
line_req = InstallRequirement.from_line(line)
if not line_req.name:
logger.notify("Skipping line because it's not clear what it would install: %s"
% line.strip())
logger.notify(" (add #egg=PackageName to the URL to avoid this warning)")
continue
if line_req.name not in installations:
logger.warn("Requirement file contains %s, but that package is not installed"
% line.strip())
continue
f.write(str(installations[line_req.name]))
del installations[line_req.name]
f.write('## The following requirements were added by pip --freeze:\n')
for installation in sorted(installations.values(), key=lambda x: x.name):
f.write(str(installation))
FreezeCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/help.py
-31
View File
@@ -1,31 +0,0 @@
from pip.basecommand import Command, command_dict, load_all_commands
from pip.exceptions import InstallationError
from pip.baseparser import parser
class HelpCommand(Command):
name = 'help'
usage = '%prog'
summary = 'Show available commands'
def run(self, options, args):
load_all_commands()
if args:
## FIXME: handle errors better here
command = args[0]
if command not in command_dict:
raise InstallationError('No command with the name: %s' % command)
command = command_dict[command]
command.parser.print_help()
return
parser.print_help()
print('\nCommands available:')
commands = list(set(command_dict.values()))
commands.sort(key=lambda x: x.name)
for command in commands:
if command.hidden:
continue
print(' %s: %s' % (command.name, command.summary))
HelpCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/install.py
-247
View File
@@ -1,247 +0,0 @@
import os, sys
from pip.req import InstallRequirement, RequirementSet
from pip.req import parse_requirements
from pip.log import logger
from pip.locations import build_prefix, src_prefix
from pip.basecommand import Command
from pip.index import PackageFinder
from pip.exceptions import InstallationError
class InstallCommand(Command):
name = 'install'
usage = '%prog [OPTIONS] PACKAGE_NAMES...'
summary = 'Install packages'
bundle = False
def __init__(self):
super(InstallCommand, self).__init__()
self.parser.add_option(
'-e', '--editable',
dest='editables',
action='append',
default=[],
metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE',
help='Install a package directly from a checkout. Source will be checked '
'out into src/PACKAGE (lower-case) and installed in-place (using '
'setup.py develop). You can run this on an existing directory/checkout (like '
'pip install -e src/mycheckout). This option may be provided multiple times. '
'Possible values for VCS are: svn, git, hg and bzr.')
self.parser.add_option(
'-r', '--requirement',
dest='requirements',
action='append',
default=[],
metavar='FILENAME',
help='Install all the packages listed in the given requirements file. '
'This option can be used multiple times.')
self.parser.add_option(
'-f', '--find-links',
dest='find_links',
action='append',
default=[],
metavar='URL',
help='URL to look for packages at')
self.parser.add_option(
'-i', '--index-url', '--pypi-url',
dest='index_url',
metavar='URL',
default='http://pypi.python.org/simple/',
help='Base URL of Python Package Index (default %default)')
self.parser.add_option(
'--extra-index-url',
dest='extra_index_urls',
metavar='URL',
action='append',
default=[],
help='Extra URLs of package indexes to use in addition to --index-url')
self.parser.add_option(
'--no-index',
dest='no_index',
action='store_true',
default=False,
help='Ignore package index (only looking at --find-links URLs instead)')
self.parser.add_option(
'-M', '--use-mirrors',
dest='use_mirrors',
action='store_true',
default=False,
help='Use the PyPI mirrors as a fallback in case the main index is down.')
self.parser.add_option(
'--mirrors',
dest='mirrors',
metavar='URL',
action='append',
default=[],
help='Specific mirror URLs to query when --use-mirrors is used')
self.parser.add_option(
'-b', '--build', '--build-dir', '--build-directory',
dest='build_dir',
metavar='DIR',
default=None,
help='Unpack packages into DIR (default %s) and build from there' % build_prefix)
self.parser.add_option(
'-d', '--download', '--download-dir', '--download-directory',
dest='download_dir',
metavar='DIR',
default=None,
help='Download packages into DIR instead of installing them')
self.parser.add_option(
'--download-cache',
dest='download_cache',
metavar='DIR',
default=None,
help='Cache downloaded packages in DIR')
self.parser.add_option(
'--src', '--source', '--source-dir', '--source-directory',
dest='src_dir',
metavar='DIR',
default=None,
help='Check out --editable packages into DIR (default %s)' % src_prefix)
self.parser.add_option(
'-U', '--upgrade',
dest='upgrade',
action='store_true',
help='Upgrade all packages to the newest available version')
self.parser.add_option(
'-I', '--ignore-installed',
dest='ignore_installed',
action='store_true',
help='Ignore the installed packages (reinstalling instead)')
self.parser.add_option(
'--no-deps', '--no-dependencies',
dest='ignore_dependencies',
action='store_true',
default=False,
help='Ignore package dependencies')
self.parser.add_option(
'--no-install',
dest='no_install',
action='store_true',
help="Download and unpack all packages, but don't actually install them")
self.parser.add_option(
'--no-download',
dest='no_download',
action="store_true",
help="Don't download any packages, just install the ones already downloaded "
"(completes an install run with --no-install)")
self.parser.add_option(
'--install-option',
dest='install_options',
action='append',
help="Extra arguments to be supplied to the setup.py install "
"command (use like --install-option=\"--install-scripts=/usr/local/bin\"). "
"Use multiple --install-option options to pass multiple options to setup.py install. "
"If you are using an option with a directory path, be sure to use absolute path.")
self.parser.add_option(
'--global-option',
dest='global_options',
action='append',
help="Extra global options to be supplied to the setup.py"
"call before the install command")
self.parser.add_option(
'--user',
dest='use_user_site',
action='store_true',
help='Install to user-site')
def _build_package_finder(self, options, index_urls):
"""
Create a package finder appropriate to this install command.
This method is meant to be overridden by subclasses, not
called directly.
"""
return PackageFinder(find_links=options.find_links,
index_urls=index_urls,
use_mirrors=options.use_mirrors,
mirrors=options.mirrors)
def run(self, options, args):
if not options.build_dir:
options.build_dir = build_prefix
if not options.src_dir:
options.src_dir = src_prefix
if options.download_dir:
options.no_install = True
options.ignore_installed = True
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
install_options = options.install_options or []
if options.use_user_site:
install_options.append('--user')
global_options = options.global_options or []
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
index_urls = []
finder = self._build_package_finder(options, index_urls)
requirement_set = RequirementSet(
build_dir=options.build_dir,
src_dir=options.src_dir,
download_dir=options.download_dir,
download_cache=options.download_cache,
upgrade=options.upgrade,
ignore_installed=options.ignore_installed,
ignore_dependencies=options.ignore_dependencies)
for name in args:
requirement_set.add_requirement(
InstallRequirement.from_line(name, None))
for name in options.editables:
requirement_set.add_requirement(
InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
for filename in options.requirements:
for req in parse_requirements(filename, finder=finder, options=options):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
if options.find_links:
raise InstallationError('You must give at least one '
'requirement to %s (maybe you meant "pip install %s"?)'
% (self.name, " ".join(options.find_links)))
raise InstallationError('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' % dict(name=self.name))
if (options.use_user_site and
sys.version_info < (2, 6)):
raise InstallationError('--user is only supported in Python version 2.6 and newer')
import setuptools
if (options.use_user_site and
requirement_set.has_editables and
not getattr(setuptools, '_distribute', False)):
raise InstallationError('--user --editable not supported with setuptools, use distribute')
if not options.no_download:
requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle)
else:
requirement_set.locate_files()
if not options.no_install and not self.bundle:
requirement_set.install(install_options, global_options)
installed = ' '.join([req.name for req in
requirement_set.successfully_installed])
if installed:
logger.notify('Successfully installed %s' % installed)
elif not self.bundle:
downloaded = ' '.join([req.name for req in
requirement_set.successfully_downloaded])
if downloaded:
logger.notify('Successfully downloaded %s' % downloaded)
elif self.bundle:
requirement_set.create_bundle(self.bundle_filename)
logger.notify('Created bundle in %s' % self.bundle_filename)
# Clean up
if not options.no_install:
requirement_set.cleanup_files(bundle=self.bundle)
return requirement_set
InstallCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/search.py
-116
View File
@@ -1,116 +0,0 @@
import sys
import textwrap
import pkg_resources
import pip.download
from pip.basecommand import Command
from pip.util import get_terminal_size
from pip.log import logger
from pip.backwardcompat import xmlrpclib, reduce, cmp
from distutils.version import StrictVersion, LooseVersion
class SearchCommand(Command):
name = 'search'
usage = '%prog QUERY'
summary = 'Search PyPI'
def __init__(self):
super(SearchCommand, self).__init__()
self.parser.add_option(
'--index',
dest='index',
metavar='URL',
default='http://pypi.python.org/pypi',
help='Base URL of Python Package Index (default %default)')
def run(self, options, args):
if not args:
logger.warn('ERROR: Missing required argument (search query).')
return
query = args
index_url = options.index
pypi_hits = self.search(query, index_url)
hits = transform_hits(pypi_hits)
terminal_width = None
if sys.stdout.isatty():
terminal_width = get_terminal_size()[0]
print_results(hits, terminal_width=terminal_width)
def search(self, query, index_url):
pypi = xmlrpclib.ServerProxy(index_url, pip.download.xmlrpclib_transport)
hits = pypi.search({'name': query, 'summary': query}, 'or')
return hits
def transform_hits(hits):
"""
The list from pypi is really a list of versions. We want a list of
packages with the list of versions stored inline. This converts the
list from pypi into one we can use.
"""
packages = {}
for hit in hits:
name = hit['name']
summary = hit['summary']
version = hit['version']
score = hit['_pypi_ordering']
if name not in packages.keys():
packages[name] = {'name': name, 'summary': summary, 'versions': [version], 'score': score}
else:
packages[name]['versions'].append(version)
# if this is the highest version, replace summary and score
if version == highest_version(packages[name]['versions']):
packages[name]['summary'] = summary
packages[name]['score'] = score
# each record has a unique name now, so we will convert the dict into a list sorted by score
package_list = sorted(packages.values(), key=lambda x: x['score'], reverse=True)
return package_list
def print_results(hits, name_column_width=25, terminal_width=None):
installed_packages = [p.project_name for p in pkg_resources.working_set]
for hit in hits:
name = hit['name']
summary = hit['summary'] or ''
if terminal_width is not None:
# wrap and indent summary to fit terminal
summary = textwrap.wrap(summary, terminal_width - name_column_width - 5)
summary = ('\n' + ' ' * (name_column_width + 3)).join(summary)
line = '%s - %s' % (name.ljust(name_column_width), summary)
try:
logger.notify(line)
if name in installed_packages:
dist = pkg_resources.get_distribution(name)
logger.indent += 2
try:
latest = highest_version(hit['versions'])
if dist.version == latest:
logger.notify('INSTALLED: %s (latest)' % dist.version)
else:
logger.notify('INSTALLED: %s' % dist.version)
logger.notify('LATEST: %s' % latest)
finally:
logger.indent -= 2
except UnicodeEncodeError:
pass
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
def highest_version(versions):
return reduce((lambda v1, v2: compare_versions(v1, v2) == 1 and v1 or v2), versions)
SearchCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/uninstall.py
-42
View File
@@ -1,42 +0,0 @@
from pip.req import InstallRequirement, RequirementSet, parse_requirements
from pip.basecommand import Command
from pip.exceptions import InstallationError
class UninstallCommand(Command):
name = 'uninstall'
usage = '%prog [OPTIONS] PACKAGE_NAMES ...'
summary = 'Uninstall packages'
def __init__(self):
super(UninstallCommand, self).__init__()
self.parser.add_option(
'-r', '--requirement',
dest='requirements',
action='append',
default=[],
metavar='FILENAME',
help='Uninstall all the packages listed in the given requirements file. '
'This option can be used multiple times.')
self.parser.add_option(
'-y', '--yes',
dest='yes',
action='store_true',
help="Don't ask for confirmation of uninstall deletions.")
def run(self, options, args):
requirement_set = RequirementSet(
build_dir=None,
src_dir=None,
download_dir=None)
for name in args:
requirement_set.add_requirement(
InstallRequirement.from_line(name))
for filename in options.requirements:
for req in parse_requirements(filename, options=options):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
raise InstallationError('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' % dict(name=self.name))
requirement_set.uninstall(auto_confirm=options.yes)
UninstallCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/unzip.py
-9
View File
@@ -1,9 +0,0 @@
from pip.commands.zip import ZipCommand
class UnzipCommand(ZipCommand):
name = 'unzip'
summary = 'Unzip individual packages'
UnzipCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/commands/zip.py
-346
View File
@@ -1,346 +0,0 @@
import sys
import re
import fnmatch
import os
import shutil
import zipfile
from pip.util import display_path, backup_dir, rmtree
from pip.log import logger
from pip.exceptions import InstallationError
from pip.basecommand import Command
class ZipCommand(Command):
name = 'zip'
usage = '%prog [OPTIONS] PACKAGE_NAMES...'
summary = 'Zip individual packages'
def __init__(self):
super(ZipCommand, self).__init__()
if self.name == 'zip':
self.parser.add_option(
'--unzip',
action='store_true',
dest='unzip',
help='Unzip (rather than zip) a package')
else:
self.parser.add_option(
'--zip',
action='store_false',
dest='unzip',
default=True,
help='Zip (rather than unzip) a package')
self.parser.add_option(
'--no-pyc',
action='store_true',
dest='no_pyc',
help='Do not include .pyc files in zip files (useful on Google App Engine)')
self.parser.add_option(
'-l', '--list',
action='store_true',
dest='list',
help='List the packages available, and their zip status')
self.parser.add_option(
'--sort-files',
action='store_true',
dest='sort_files',
help='With --list, sort packages according to how many files they contain')
self.parser.add_option(
'--path',
action='append',
dest='paths',
help='Restrict operations to the given paths (may include wildcards)')
self.parser.add_option(
'-n', '--simulate',
action='store_true',
help='Do not actually perform the zip/unzip operation')
def paths(self):
"""All the entries of sys.path, possibly restricted by --path"""
if not self.select_paths:
return sys.path
result = []
match_any = set()
for path in sys.path:
path = os.path.normcase(os.path.abspath(path))
for match in self.select_paths:
match = os.path.normcase(os.path.abspath(match))
if '*' in match:
if re.search(fnmatch.translate(match+'*'), path):
result.append(path)
match_any.add(match)
break
else:
if path.startswith(match):
result.append(path)
match_any.add(match)
break
else:
logger.debug("Skipping path %s because it doesn't match %s"
% (path, ', '.join(self.select_paths)))
for match in self.select_paths:
if match not in match_any and '*' not in match:
result.append(match)
logger.debug("Adding path %s because it doesn't match anything already on sys.path"
% match)
return result
def run(self, options, args):
self.select_paths = options.paths
self.simulate = options.simulate
if options.list:
return self.list(options, args)
if not args:
raise InstallationError(
'You must give at least one package to zip or unzip')
packages = []
for arg in args:
module_name, filename = self.find_package(arg)
if options.unzip and os.path.isdir(filename):
raise InstallationError(
'The module %s (in %s) is not a zip file; cannot be unzipped'
% (module_name, filename))
elif not options.unzip and not os.path.isdir(filename):
raise InstallationError(
'The module %s (in %s) is not a directory; cannot be zipped'
% (module_name, filename))
packages.append((module_name, filename))
last_status = None
for module_name, filename in packages:
if options.unzip:
last_status = self.unzip_package(module_name, filename)
else:
last_status = self.zip_package(module_name, filename, options.no_pyc)
return last_status
def unzip_package(self, module_name, filename):
zip_filename = os.path.dirname(filename)
if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename):
raise InstallationError(
'Module %s (in %s) isn\'t located in a zip file in %s'
% (module_name, filename, zip_filename))
package_path = os.path.dirname(zip_filename)
if not package_path in self.paths():
logger.warn(
'Unpacking %s into %s, but %s is not on sys.path'
% (display_path(zip_filename), display_path(package_path),
display_path(package_path)))
logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename)))
if self.simulate:
logger.notify('Skipping remaining operations because of --simulate')
return
logger.indent += 2
try:
## FIXME: this should be undoable:
zip = zipfile.ZipFile(zip_filename)
to_save = []
for name in zip.namelist():
if name.startswith(module_name + os.path.sep):
content = zip.read(name)
dest = os.path.join(package_path, name)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
if not content and dest.endswith(os.path.sep):
if not os.path.exists(dest):
os.makedirs(dest)
else:
f = open(dest, 'wb')
f.write(content)
f.close()
else:
to_save.append((name, zip.read(name)))
zip.close()
if not to_save:
logger.info('Removing now-empty zip file %s' % display_path(zip_filename))
os.unlink(zip_filename)
self.remove_filename_from_pth(zip_filename)
else:
logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename)))
zip = zipfile.ZipFile(zip_filename, 'w')
for name, content in to_save:
zip.writestr(name, content)
zip.close()
finally:
logger.indent -= 2
def zip_package(self, module_name, filename, no_pyc):
orig_filename = filename
logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
logger.indent += 2
if filename.endswith('.egg'):
dest_filename = filename
else:
dest_filename = filename + '.zip'
try:
## FIXME: I think this needs to be undoable:
if filename == dest_filename:
filename = backup_dir(orig_filename)
logger.notify('Moving %s aside to %s' % (orig_filename, filename))
if not self.simulate:
shutil.move(orig_filename, filename)
try:
logger.info('Creating zip file in %s' % display_path(dest_filename))
if not self.simulate:
zip = zipfile.ZipFile(dest_filename, 'w')
zip.writestr(module_name + '/', '')
for dirpath, dirnames, filenames in os.walk(filename):
if no_pyc:
filenames = [f for f in filenames
if not f.lower().endswith('.pyc')]
for fns, is_dir in [(dirnames, True), (filenames, False)]:
for fn in fns:
full = os.path.join(dirpath, fn)
dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn)
if is_dir:
zip.writestr(dest+'/', '')
else:
zip.write(full, dest)
zip.close()
logger.info('Removing old directory %s' % display_path(filename))
if not self.simulate:
rmtree(filename)
except:
## FIXME: need to do an undo here
raise
## FIXME: should also be undone:
self.add_filename_to_pth(dest_filename)
finally:
logger.indent -= 2
def remove_filename_from_pth(self, filename):
for pth in self.pth_files():
f = open(pth, 'r')
lines = f.readlines()
f.close()
new_lines = [
l for l in lines if l.strip() != filename]
if lines != new_lines:
logger.info('Removing reference to %s from .pth file %s'
% (display_path(filename), display_path(pth)))
if not [line for line in new_lines if line]:
logger.info('%s file would be empty: deleting' % display_path(pth))
if not self.simulate:
os.unlink(pth)
else:
if not self.simulate:
f = open(pth, 'wb')
f.writelines(new_lines)
f.close()
return
logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename))
def add_filename_to_pth(self, filename):
path = os.path.dirname(filename)
dest = os.path.join(path, filename + '.pth')
if path not in self.paths():
logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest))
if not self.simulate:
if os.path.exists(dest):
f = open(dest)
lines = f.readlines()
f.close()
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
lines.append(filename+'\n')
else:
lines = [filename + '\n']
f = open(dest, 'wb')
f.writelines(lines)
f.close()
def pth_files(self):
for path in self.paths():
if not os.path.exists(path) or not os.path.isdir(path):
continue
for filename in os.listdir(path):
if filename.endswith('.pth'):
yield os.path.join(path, filename)
def find_package(self, package):
for path in self.paths():
full = os.path.join(path, package)
if os.path.exists(full):
return package, full
if not os.path.isdir(path) and zipfile.is_zipfile(path):
zip = zipfile.ZipFile(path, 'r')
try:
zip.read(os.path.join(package, '__init__.py'))
except KeyError:
pass
else:
zip.close()
return package, full
zip.close()
## FIXME: need special error for package.py case:
raise InstallationError(
'No package with the name %s found' % package)
def list(self, options, args):
if args:
raise InstallationError(
'You cannot give an argument with --list')
for path in sorted(self.paths()):
if not os.path.exists(path):
continue
basename = os.path.basename(path.rstrip(os.path.sep))
if os.path.isfile(path) and zipfile.is_zipfile(path):
if os.path.dirname(path) not in self.paths():
logger.notify('Zipped egg: %s' % display_path(path))
continue
if (basename != 'site-packages' and basename != 'dist-packages'
and not path.replace('\\', '/').endswith('lib/python')):
continue
logger.notify('In %s:' % display_path(path))
logger.indent += 2
zipped = []
unzipped = []
try:
for filename in sorted(os.listdir(path)):
ext = os.path.splitext(filename)[1].lower()
if ext in ('.pth', '.egg-info', '.egg-link'):
continue
if ext == '.py':
logger.info('Not displaying %s: not a package' % display_path(filename))
continue
full = os.path.join(path, filename)
if os.path.isdir(full):
unzipped.append((filename, self.count_package(full)))
elif zipfile.is_zipfile(full):
zipped.append(filename)
else:
logger.info('Unknown file: %s' % display_path(filename))
if zipped:
logger.notify('Zipped packages:')
logger.indent += 2
try:
for filename in zipped:
logger.notify(filename)
finally:
logger.indent -= 2
else:
logger.notify('No zipped packages.')
if unzipped:
if options.sort_files:
unzipped.sort(key=lambda x: -x[1])
logger.notify('Unzipped packages:')
logger.indent += 2
try:
for filename, count in unzipped:
logger.notify('%s (%i files)' % (filename, count))
finally:
logger.indent -= 2
else:
logger.notify('No unzipped packages.')
finally:
logger.indent -= 2
def count_package(self, path):
total = 0
for dirpath, dirnames, filenames in os.walk(path):
filenames = [f for f in filenames
if not f.lower().endswith('.pyc')]
total += len(filenames)
return total
ZipCommand()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/download.py
-480
View File
@@ -1,480 +0,0 @@
import cgi
import getpass
import mimetypes
import os
import re
import shutil
import sys
import tempfile
from pip.backwardcompat import (md5, copytree, xmlrpclib, urllib, urllib2,
urlparse, string_types, HTTPError)
from pip.exceptions import InstallationError
from pip.util import (splitext, rmtree,
format_size, display_path, backup_dir, ask,
unpack_file, create_download_cache_folder, cache_download)
from pip.vcs import vcs
from pip.log import logger
__all__ = ['xmlrpclib_transport', 'get_file_content', 'urlopen',
'is_url', 'url_to_path', 'path_to_url', 'path_to_url2',
'geturl', 'is_archive_file', 'unpack_vcs_link',
'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url']
xmlrpclib_transport = xmlrpclib.Transport()
def get_file_content(url, comes_from=None):
"""Gets the content of a file; it may be a filename, file: URL, or
http: URL. Returns (location, content)"""
match = _scheme_re.search(url)
if match:
scheme = match.group(1).lower()
if (scheme == 'file' and comes_from
and comes_from.startswith('http')):
raise InstallationError(
'Requirements file %s references URL %s, which is local'
% (comes_from, url))
if scheme == 'file':
path = url.split(':', 1)[1]
path = path.replace('\\', '/')
match = _url_slash_drive_re.match(path)
if match:
path = match.group(1) + ':' + path.split('|', 1)[1]
path = urllib.unquote(path)
if path.startswith('/'):
path = '/' + path.lstrip('/')
url = path
else:
## FIXME: catch some errors
resp = urlopen(url)
return geturl(resp), resp.read()
try:
f = open(url)
content = f.read()
except IOError:
e = sys.exc_info()[1]
raise InstallationError('Could not open requirements file: %s' % str(e))
else:
f.close()
return url, content
_scheme_re = re.compile(r'^(http|https|file):', re.I)
_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
class URLOpener(object):
"""
pip's own URL helper that adds HTTP auth and proxy support
"""
def __init__(self):
self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
def __call__(self, url):
"""
If the given url contains auth info or if a normal request gets a 401
response, an attempt is made to fetch the resource using basic HTTP
auth.
"""
url, username, password = self.extract_credentials(url)
if username is None:
try:
response = urllib2.urlopen(self.get_request(url))
except urllib2.HTTPError:
e = sys.exc_info()[1]
if e.code != 401:
raise
response = self.get_response(url)
else:
response = self.get_response(url, username, password)
return response
def get_request(self, url):
"""
Wraps the URL to retrieve to protects against "creative"
interpretation of the RFC: http://bugs.python.org/issue8732
"""
if isinstance(url, string_types):
url = urllib2.Request(url, headers={'Accept-encoding': 'identity'})
return url
def get_response(self, url, username=None, password=None):
"""
does the dirty work of actually getting the rsponse object using urllib2
and its HTTP auth builtins.
"""
scheme, netloc, path, query, frag = urlparse.urlsplit(url)
req = self.get_request(url)
stored_username, stored_password = self.passman.find_user_password(None, netloc)
# see if we have a password stored
if stored_username is None:
if username is None and self.prompting:
username = urllib.quote(raw_input('User for %s: ' % netloc))
password = urllib.quote(getpass.getpass('Password: '))
if username and password:
self.passman.add_password(None, netloc, username, password)
stored_username, stored_password = self.passman.find_user_password(None, netloc)
authhandler = urllib2.HTTPBasicAuthHandler(self.passman)
opener = urllib2.build_opener(authhandler)
# FIXME: should catch a 401 and offer to let the user reenter credentials
return opener.open(req)
def setup(self, proxystr='', prompting=True):
"""
Sets the proxy handler given the option passed on the command
line. If an empty string is passed it looks at the HTTP_PROXY
environment variable.
"""
self.prompting = prompting
proxy = self.get_proxy(proxystr)
if proxy:
proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy})
opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler)
urllib2.install_opener(opener)
def parse_credentials(self, netloc):
if "@" in netloc:
userinfo = netloc.rsplit("@", 1)[0]
if ":" in userinfo:
return userinfo.split(":", 1)
return userinfo, None
return None, None
def extract_credentials(self, url):
"""
Extracts user/password from a url.
Returns a tuple:
(url-without-auth, username, password)
"""
if isinstance(url, urllib2.Request):
result = urlparse.urlsplit(url.get_full_url())
else:
result = urlparse.urlsplit(url)
scheme, netloc, path, query, frag = result
username, password = self.parse_credentials(netloc)
if username is None:
return url, None, None
elif password is None and self.prompting:
# remove the auth credentials from the url part
netloc = netloc.replace('%s@' % username, '', 1)
# prompt for the password
prompt = 'Password for %s@%s: ' % (username, netloc)
password = urllib.quote(getpass.getpass(prompt))
else:
# remove the auth credentials from the url part
netloc = netloc.replace('%s:%s@' % (username, password), '', 1)
target_url = urlparse.urlunsplit((scheme, netloc, path, query, frag))
return target_url, username, password
def get_proxy(self, proxystr=''):
"""
Get the proxy given the option passed on the command line.
If an empty string is passed it looks at the HTTP_PROXY
environment variable.
"""
if not proxystr:
proxystr = os.environ.get('HTTP_PROXY', '')
if proxystr:
if '@' in proxystr:
user_password, server_port = proxystr.split('@', 1)
if ':' in user_password:
user, password = user_password.split(':', 1)
else:
user = user_password
prompt = 'Password for %s@%s: ' % (user, server_port)
password = urllib.quote(getpass.getpass(prompt))
return '%s:%s@%s' % (user, password, server_port)
else:
return proxystr
else:
return None
urlopen = URLOpener()
def is_url(name):
"""Returns true if the name looks like a URL"""
if ':' not in name:
return False
scheme = name.split(':', 1)[0].lower()
return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
def url_to_path(url):
"""
Convert a file: URL to a path.
"""
assert url.startswith('file:'), (
"You can only turn file: urls into filenames (not %r)" % url)
path = url[len('file:'):].lstrip('/')
path = urllib.unquote(path)
if _url_drive_re.match(path):
path = path[0] + ':' + path[2:]
else:
path = '/' + path
return path
_drive_re = re.compile('^([a-z]):', re.I)
_url_drive_re = re.compile('^([a-z])[:|]', re.I)
def path_to_url(path):
"""
Convert a path to a file: URL. The path will be made absolute.
"""
path = os.path.normcase(os.path.abspath(path))
if _drive_re.match(path):
path = path[0] + '|' + path[2:]
url = urllib.quote(path)
url = url.replace(os.path.sep, '/')
url = url.lstrip('/')
return 'file:///' + url
def path_to_url2(path):
"""
Convert a path to a file: URL. The path will be made absolute and have
quoted path parts.
"""
path = os.path.normpath(os.path.abspath(path))
drive, path = os.path.splitdrive(path)
filepath = path.split(os.path.sep)
url = '/'.join([urllib.quote(part) for part in filepath])
if not drive:
url = url.lstrip('/')
return 'file:///' + drive + url
def geturl(urllib2_resp):
"""
Use instead of urllib.addinfourl.geturl(), which appears to have
some issues with dropping the double slash for certain schemes
(e.g. file://). This implementation is probably over-eager, as it
always restores '://' if it is missing, and it appears some url
schemata aren't always followed by '//' after the colon, but as
far as I know pip doesn't need any of those.
The URI RFC can be found at: http://tools.ietf.org/html/rfc1630
This function assumes that
scheme:/foo/bar
is the same as
scheme:///foo/bar
"""
url = urllib2_resp.geturl()
scheme, rest = url.split(':', 1)
if rest.startswith('//'):
return url
else:
# FIXME: write a good test to cover it
return '%s://%s' % (scheme, rest)
def is_archive_file(name):
"""Return True if `name` is a considered as an archive file."""
archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle')
ext = splitext(name)[1].lower()
if ext in archives:
return True
return False
def unpack_vcs_link(link, location, only_download=False):
vcs_backend = _get_used_vcs_backend(link)
if only_download:
vcs_backend.export(location)
else:
vcs_backend.unpack(location)
def unpack_file_url(link, location):
source = url_to_path(link.url)
content_type = mimetypes.guess_type(source)[0]
if os.path.isdir(source):
# delete the location since shutil will create it again :(
if os.path.isdir(location):
rmtree(location)
copytree(source, location)
else:
unpack_file(source, location, content_type, link)
def _get_used_vcs_backend(link):
for backend in vcs.backends:
if link.scheme in backend.schemes:
vcs_backend = backend(link.url)
return vcs_backend
def is_vcs_url(link):
return bool(_get_used_vcs_backend(link))
def is_file_url(link):
return link.url.lower().startswith('file:')
def _check_md5(download_hash, link):
download_hash = download_hash.hexdigest()
if download_hash != link.md5_hash:
logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!"
% (link, download_hash, link.md5_hash))
raise InstallationError('Bad MD5 hash for package %s' % link)
def _get_md5_from_file(target_file, link):
download_hash = md5()
fp = open(target_file, 'rb')
while True:
chunk = fp.read(4096)
if not chunk:
break
download_hash.update(chunk)
fp.close()
return download_hash
def _download_url(resp, link, temp_location):
fp = open(temp_location, 'wb')
download_hash = None
if link.md5_hash:
download_hash = md5()
try:
total_length = int(resp.info()['content-length'])
except (ValueError, KeyError, TypeError):
total_length = 0
downloaded = 0
show_progress = total_length > 40*1000 or not total_length
show_url = link.show_url
try:
if show_progress:
## FIXME: the URL can get really long in this message:
if total_length:
logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length)))
else:
logger.start_progress('Downloading %s (unknown size): ' % show_url)
else:
logger.notify('Downloading %s' % show_url)
logger.debug('Downloading from URL %s' % link)
while True:
chunk = resp.read(4096)
if not chunk:
break
downloaded += len(chunk)
if show_progress:
if not total_length:
logger.show_progress('%s' % format_size(downloaded))
else:
logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded)))
if link.md5_hash:
download_hash.update(chunk)
fp.write(chunk)
fp.close()
finally:
if show_progress:
logger.end_progress('%s downloaded' % format_size(downloaded))
return download_hash
def _copy_file(filename, location, content_type, link):
copy = True
download_location = os.path.join(location, link.filename)
if os.path.exists(download_location):
response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup '
% display_path(download_location), ('i', 'w', 'b'))
if response == 'i':
copy = False
elif response == 'w':
logger.warn('Deleting %s' % display_path(download_location))
os.remove(download_location)
elif response == 'b':
dest_file = backup_dir(download_location)
logger.warn('Backing up %s to %s'
% (display_path(download_location), display_path(dest_file)))
shutil.move(download_location, dest_file)
if copy:
shutil.copy(filename, download_location)
logger.indent -= 2
logger.notify('Saved %s' % display_path(download_location))
def unpack_http_url(link, location, download_cache, only_download):
temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
target_url = link.url.split('#', 1)[0]
target_file = None
download_hash = None
if download_cache:
target_file = os.path.join(download_cache,
urllib.quote(target_url, ''))
if not os.path.isdir(download_cache):
create_download_cache_folder(download_cache)
if (target_file
and os.path.exists(target_file)
and os.path.exists(target_file + '.content-type')):
fp = open(target_file+'.content-type')
content_type = fp.read().strip()
fp.close()
if link.md5_hash:
download_hash = _get_md5_from_file(target_file, link)
temp_location = target_file
logger.notify('Using download cache from %s' % target_file)
else:
resp = _get_response_from_url(target_url, link)
content_type = resp.info()['content-type']
filename = link.filename # fallback
# Have a look at the Content-Disposition header for a better guess
content_disposition = resp.info().get('content-disposition')
if content_disposition:
type, params = cgi.parse_header(content_disposition)
# We use ``or`` here because we don't want to use an "empty" value
# from the filename param.
filename = params.get('filename') or filename
ext = splitext(filename)[1]
if not ext:
ext = mimetypes.guess_extension(content_type)
if ext:
filename += ext
if not ext and link.url != geturl(resp):
ext = os.path.splitext(geturl(resp))[1]
if ext:
filename += ext
temp_location = os.path.join(temp_dir, filename)
download_hash = _download_url(resp, link, temp_location)
if link.md5_hash:
_check_md5(download_hash, link)
if only_download:
_copy_file(temp_location, location, content_type, link)
else:
unpack_file(temp_location, location, content_type, link)
if target_file and target_file != temp_location:
cache_download(target_file, temp_location, content_type)
if target_file is None:
os.unlink(temp_location)
os.rmdir(temp_dir)
def _get_response_from_url(target_url, link):
try:
resp = urlopen(target_url)
except urllib2.HTTPError:
e = sys.exc_info()[1]
logger.fatal("HTTP error %s while getting %s" % (e.code, link))
raise
except IOError:
e = sys.exc_info()[1]
# Typically an FTP error
logger.fatal("Error %s while getting %s" % (e, link))
raise
return resp
class Urllib2HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/exceptions.py
-17
View File
@@ -1,17 +0,0 @@
"""Exceptions used throughout package"""
class InstallationError(Exception):
"""General exception during installation"""
class UninstallationError(Exception):
"""General exception during uninstallation"""
class DistributionNotFound(InstallationError):
"""Raised when a distribution cannot be found to satisfy a requirement"""
class BadCommand(Exception):
"""Raised when virtualenv or a command is not found"""
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/index.py
-689
View File
@@ -1,689 +0,0 @@
"""Routines related to PyPI, indexes"""
import sys
import os
import re
import mimetypes
import threading
import posixpath
import pkg_resources
import random
import socket
import string
from pip.log import logger
from pip.util import Inf
from pip.util import normalize_name, splitext
from pip.exceptions import DistributionNotFound
from pip.backwardcompat import (WindowsError,
Queue, httplib, urlparse,
URLError, HTTPError, u,
product, url2pathname)
from pip.backwardcompat import Empty as QueueEmpty
from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest
__all__ = ['PackageFinder']
DEFAULT_MIRROR_URL = "last.pypi.python.org"
class PackageFinder(object):
"""This finds packages.
This is meant to match easy_install's technique for looking for
packages, by reading pages and looking for appropriate links
"""
def __init__(self, find_links, index_urls,
use_mirrors=False, mirrors=None, main_mirror_url=None):
self.find_links = find_links
self.index_urls = index_urls
self.dependency_links = []
self.cache = PageCache()
# These are boring links that have already been logged somehow:
self.logged_links = set()
if use_mirrors:
self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url)
logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls))
else:
self.mirror_urls = []
def add_dependency_links(self, links):
## FIXME: this shouldn't be global list this, it should only
## apply to requirements of the package that specifies the
## dependency_links value
## FIXME: also, we should track comes_from (i.e., use Link)
self.dependency_links.extend(links)
@staticmethod
def _sort_locations(locations):
"""
Sort locations into "files" (archives) and "urls", and return
a pair of lists (files,urls)
"""
files = []
urls = []
# puts the url for the given file path into the appropriate
# list
def sort_path(path):
url = path_to_url2(path)
if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
urls.append(url)
else:
files.append(url)
for url in locations:
if url.startswith('file:'):
path = url_to_path(url)
if os.path.isdir(path):
path = os.path.realpath(path)
for item in os.listdir(path):
sort_path(os.path.join(path, item))
elif os.path.isfile(path):
sort_path(path)
else:
urls.append(url)
return files, urls
def find_requirement(self, req, upgrade):
url_name = req.url_name
# Only check main index if index URL is given:
main_index_url = None
if self.index_urls:
# Check that we have the url_name correctly spelled:
main_index_url = Link(posixpath.join(self.index_urls[0], url_name))
# This will also cache the page, so it's okay that we get it again later:
page = self._get_page(main_index_url, req)
if page is None:
url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name
# Combine index URLs with mirror URLs here to allow
# adding more index URLs from requirements files
all_index_urls = self.index_urls + self.mirror_urls
def mkurl_pypi_url(url):
loc = posixpath.join(url, url_name)
# For maximum compatibility with easy_install, ensure the path
# ends in a trailing slash. Although this isn't in the spec
# (and PyPI can handle it without the slash) some other index
# implementations might break if they relied on easy_install's behavior.
if not loc.endswith('/'):
loc = loc + '/'
return loc
if url_name is not None:
locations = [
mkurl_pypi_url(url)
for url in all_index_urls] + self.find_links
else:
locations = list(self.find_links)
locations.extend(self.dependency_links)
for version in req.absolute_versions:
if url_name is not None and main_index_url is not None:
locations = [
posixpath.join(main_index_url.url, version)] + locations
file_locations, url_locations = self._sort_locations(locations)
locations = [Link(url) for url in url_locations]
logger.debug('URLs to search for versions for %s:' % req)
for location in locations:
logger.debug('* %s' % location)
found_versions = []
found_versions.extend(
self._package_versions(
[Link(url, '-f') for url in self.find_links], req.name.lower()))
page_versions = []
for page in self._get_pages(locations, req):
logger.debug('Analyzing links from page %s' % page.url)
logger.indent += 2
try:
page_versions.extend(self._package_versions(page.links, req.name.lower()))
finally:
logger.indent -= 2
dependency_versions = list(self._package_versions(
[Link(url) for url in self.dependency_links], req.name.lower()))
if dependency_versions:
logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions]))
file_versions = list(self._package_versions(
[Link(url) for url in file_locations], req.name.lower()))
if not found_versions and not page_versions and not dependency_versions and not file_versions:
logger.fatal('Could not find any downloads that satisfy the requirement %s' % req)
raise DistributionNotFound('No distributions at all found for %s' % req)
if req.satisfied_by is not None:
found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version))
if file_versions:
file_versions.sort(reverse=True)
logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions]))
found_versions = file_versions + found_versions
all_versions = found_versions + page_versions + dependency_versions
applicable_versions = []
for (parsed_version, link, version) in all_versions:
if version not in req.req:
logger.info("Ignoring link %s, version %s doesn't match %s"
% (link, version, ','.join([''.join(s) for s in req.req.specs])))
continue
applicable_versions.append((link, version))
applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True)
existing_applicable = bool([link for link, version in applicable_versions if link is Inf])
if not upgrade and existing_applicable:
if applicable_versions[0][1] is Inf:
logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement'
% req.satisfied_by.version)
else:
logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)'
% (req.satisfied_by.version, applicable_versions[0][1]))
return None
if not applicable_versions:
logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)'
% (req, ', '.join([version for parsed_version, link, version in found_versions])))
raise DistributionNotFound('No distributions matching the version for %s' % req)
if applicable_versions[0][0] is Inf:
# We have an existing version, and its the best version
logger.info('Installed version (%s) is most up-to-date (past versions: %s)'
% (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none'))
return None
if len(applicable_versions) > 1:
logger.info('Using version %s (newest of versions: %s)' %
(applicable_versions[0][1], ', '.join([version for link, version in applicable_versions])))
return applicable_versions[0][0]
def _find_url_name(self, index_url, url_name, req):
"""Finds the true URL name of a package, when the given name isn't quite correct.
This is usually used to implement case-insensitivity."""
if not index_url.url.endswith('/'):
# Vaguely part of the PyPI API... weird but true.
## FIXME: bad to modify this?
index_url.url += '/'
page = self._get_page(index_url, req)
if page is None:
logger.fatal('Cannot fetch index base URL %s' % index_url)
return
norm_name = normalize_name(req.url_name)
for link in page.links:
base = posixpath.basename(link.path.rstrip('/'))
if norm_name == normalize_name(base):
logger.notify('Real name of requirement %s is %s' % (url_name, base))
return base
return None
def _get_pages(self, locations, req):
"""Yields (page, page_url) from the given locations, skipping
locations that have errors, and adding download/homepage links"""
pending_queue = Queue()
for location in locations:
pending_queue.put(location)
done = []
seen = set()
threads = []
for i in range(min(10, len(locations))):
t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen))
t.setDaemon(True)
threads.append(t)
t.start()
for t in threads:
t.join()
return done
_log_lock = threading.Lock()
def _get_queued_page(self, req, pending_queue, done, seen):
while 1:
try:
location = pending_queue.get(False)
except QueueEmpty:
return
if location in seen:
continue
seen.add(location)
page = self._get_page(location, req)
if page is None:
continue
done.append(page)
for link in page.rel_links():
pending_queue.put(link)
_egg_fragment_re = re.compile(r'#egg=([^&]*)')
_egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
_py_version_re = re.compile(r'-py([123]\.[0-9])$')
def _sort_links(self, links):
"Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates"
eggs, no_eggs = [], []
seen = set()
for link in links:
if link not in seen:
seen.add(link)
if link.egg_fragment:
eggs.append(link)
else:
no_eggs.append(link)
return no_eggs + eggs
def _package_versions(self, links, search_name):
for link in self._sort_links(links):
for v in self._link_package_versions(link, search_name):
yield v
def _link_package_versions(self, link, search_name):
"""
Return an iterable of triples (pkg_resources_version_key,
link, python_version) that can be extracted from the given
link.
Meant to be overridden by subclasses, not called by clients.
"""
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug('Skipping link %s; not a file' % link)
self.logged_links.add(link)
return []
if egg_info.endswith('.tar'):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = '.tar' + ext
if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
if link not in self.logged_links:
logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
self.logged_links.add(link)
return []
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
return []
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug('Skipping %s because Python version is incorrect' % link)
return []
logger.debug('Found link %s, version: %s' % (link, version))
return [(pkg_resources.parse_version(version),
link,
version)]
def _egg_info_matches(self, egg_info, search_name, link):
match = self._egg_info_re.search(egg_info)
if not match:
logger.debug('Could not parse version from link: %s' % link)
return None
name = match.group(0).lower()
# To match the "safe" name that pkg_resources creates:
name = name.replace('_', '-')
if name.startswith(search_name.lower()):
return match.group(0)[len(search_name):].lstrip('-')
else:
return None
def _get_page(self, link, req):
return HTMLPage.get_page(link, req, cache=self.cache)
def _get_mirror_urls(self, mirrors=None, main_mirror_url=None):
"""Retrieves a list of URLs from the main mirror DNS entry
unless a list of mirror URLs are passed.
"""
if not mirrors:
mirrors = get_mirrors(main_mirror_url)
# Should this be made "less random"? E.g. netselect like?
random.shuffle(mirrors)
mirror_urls = set()
for mirror_url in mirrors:
# Make sure we have a valid URL
if not ("http://" or "https://" or "file://") in mirror_url:
mirror_url = "http://%s" % mirror_url
if not mirror_url.endswith("/simple"):
mirror_url = "%s/simple/" % mirror_url
mirror_urls.add(mirror_url)
return list(mirror_urls)
class PageCache(object):
"""Cache of HTML pages"""
failure_limit = 3
def __init__(self):
self._failures = {}
self._pages = {}
self._archives = {}
def too_many_failures(self, url):
return self._failures.get(url, 0) >= self.failure_limit
def get_page(self, url):
return self._pages.get(url)
def is_archive(self, url):
return self._archives.get(url, False)
def set_is_archive(self, url, value=True):
self._archives[url] = value
def add_page_failure(self, url, level):
self._failures[url] = self._failures.get(url, 0)+level
def add_page(self, urls, page):
for url in urls:
self._pages[url] = page
class HTMLPage(object):
"""Represents one page, along with its URL"""
## FIXME: these regexes are horrible hacks:
_homepage_re = re.compile(r'<th>\s*home\s*page', re.I)
_download_re = re.compile(r'<th>\s*download\s+url', re.I)
## These aren't so aweful:
_rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I)
_href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S)
_base_re = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I)
def __init__(self, content, url, headers=None):
self.content = content
self.url = url
self.headers = headers
def __str__(self):
return self.url
@classmethod
def get_page(cls, link, req, cache=None, skip_archives=True):
url = link.url
url = url.split('#', 1)[0]
if cache.too_many_failures(url):
return None
# Check for VCS schemes that do not support lookup as web pages.
from pip.vcs import VcsSupport
for scheme in VcsSupport.schemes:
if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals())
return None
if cache is not None:
inst = cache.get_page(url)
if inst is not None:
return inst
try:
if skip_archives:
if cache is not None:
if cache.is_archive(url):
return None
filename = link.filename
for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
if filename.endswith(bad_ext):
content_type = cls._get_content_type(url)
if content_type.lower().startswith('text/html'):
break
else:
logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type))
if cache is not None:
cache.set_is_archive(url)
return None
logger.debug('Getting page %s' % url)
# Tack index.html onto file:// URLs that point to directories
(scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
if scheme == 'file' and os.path.isdir(url2pathname(path)):
# add trailing slash if not present so urljoin doesn't trim final segment
if not url.endswith('/'):
url += '/'
url = urlparse.urljoin(url, 'index.html')
logger.debug(' file: URL is directory, getting %s' % url)
resp = urlopen(url)
real_url = geturl(resp)
headers = resp.info()
inst = cls(u(resp.read()), real_url, headers)
except (HTTPError, URLError, socket.timeout, socket.error, OSError, WindowsError):
e = sys.exc_info()[1]
desc = str(e)
if isinstance(e, socket.timeout):
log_meth = logger.info
level =1
desc = 'timed out'
elif isinstance(e, URLError):
log_meth = logger.info
if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
desc = 'timed out'
level = 1
else:
level = 2
elif isinstance(e, HTTPError) and e.code == 404:
## FIXME: notify?
log_meth = logger.info
level = 2
else:
log_meth = logger.info
level = 1
log_meth('Could not fetch URL %s: %s' % (link, desc))
log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req))
if cache is not None:
cache.add_page_failure(url, level)
return None
if cache is not None:
cache.add_page([url, real_url], inst)
return inst
@staticmethod
def _get_content_type(url):
"""Get the Content-Type of the given url, using a HEAD request"""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
if not scheme in ('http', 'https', 'ftp', 'ftps'):
## FIXME: some warning or something?
## assertion error?
return ''
req = Urllib2HeadRequest(url, headers={'Host': netloc})
resp = urlopen(req)
try:
if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'):
## FIXME: doesn't handle redirects
return ''
return resp.info().get('content-type', '')
finally:
resp.close()
@property
def base_url(self):
if not hasattr(self, "_base_url"):
match = self._base_re.search(self.content)
if match:
self._base_url = match.group(1)
else:
self._base_url = self.url
return self._base_url
@property
def links(self):
"""Yields all links in the page"""
for match in self._href_re.finditer(self.content):
url = match.group(1) or match.group(2) or match.group(3)
url = self.clean_link(urlparse.urljoin(self.base_url, url))
yield Link(url, self)
def rel_links(self):
for url in self.explicit_rel_links():
yield url
for url in self.scraped_rel_links():
yield url
def explicit_rel_links(self, rels=('homepage', 'download')):
"""Yields all links with the given relations"""
for match in self._rel_re.finditer(self.content):
found_rels = match.group(1).lower().split()
for rel in rels:
if rel in found_rels:
break
else:
continue
match = self._href_re.search(match.group(0))
if not match:
continue
url = match.group(1) or match.group(2) or match.group(3)
url = self.clean_link(urlparse.urljoin(self.base_url, url))
yield Link(url, self)
def scraped_rel_links(self):
for regex in (self._homepage_re, self._download_re):
match = regex.search(self.content)
if not match:
continue
href_match = self._href_re.search(self.content, pos=match.end())
if not href_match:
continue
url = match.group(1) or match.group(2) or match.group(3)
if not url:
continue
url = self.clean_link(urlparse.urljoin(self.base_url, url))
yield Link(url, self)
_clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
def clean_link(self, url):
"""Makes sure a link is fully encoded. That is, if a ' ' shows up in
the link, it will be rewritten to %20 (while not over-quoting
% or other characters)."""
return self._clean_re.sub(
lambda match: '%%%2x' % ord(match.group(0)), url)
class Link(object):
def __init__(self, url, comes_from=None):
self.url = url
self.comes_from = comes_from
def __str__(self):
if self.comes_from:
return '%s (from %s)' % (self.url, self.comes_from)
else:
return self.url
def __repr__(self):
return '<Link %s>' % self
def __eq__(self, other):
return self.url == other.url
def __hash__(self):
return hash(self.url)
@property
def filename(self):
url = self.url_fragment
name = posixpath.basename(url)
assert name, ('URL %r produced no filename' % url)
return name
@property
def scheme(self):
return urlparse.urlsplit(self.url)[0]
@property
def path(self):
return urlparse.urlsplit(self.url)[2]
def splitext(self):
return splitext(posixpath.basename(self.path.rstrip('/')))
@property
def url_fragment(self):
url = self.url
url = url.split('#', 1)[0]
url = url.split('?', 1)[0]
url = url.rstrip('/')
return url
_egg_fragment_re = re.compile(r'#egg=([^&]*)')
@property
def egg_fragment(self):
match = self._egg_fragment_re.search(self.url)
if not match:
return None
return match.group(1)
_md5_re = re.compile(r'md5=([a-f0-9]+)')
@property
def md5_hash(self):
match = self._md5_re.search(self.url)
if match:
return match.group(1)
return None
@property
def show_url(self):
return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
def get_requirement_from_url(url):
"""Get a requirement from the URL, if possible. This looks for #egg
in the URL"""
link = Link(url)
egg_info = link.egg_fragment
if not egg_info:
egg_info = splitext(link.filename)[0]
return package_to_requirement(egg_info)
def package_to_requirement(package_name):
"""Translate a name like Foo-1.2 to Foo==1.3"""
match = re.search(r'^(.*?)-(dev|\d.*)', package_name)
if match:
name = match.group(1)
version = match.group(2)
else:
name = package_name
version = ''
if version:
return '%s==%s' % (name, version)
else:
return name
def get_mirrors(hostname=None):
"""Return the list of mirrors from the last record found on the DNS
entry::
>>> from pip.index import get_mirrors
>>> get_mirrors()
['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org',
'd.pypi.python.org']
Originally written for the distutils2 project by Alexis Metaireau.
"""
if hostname is None:
hostname = DEFAULT_MIRROR_URL
# return the last mirror registered on PyPI.
try:
hostname = socket.gethostbyname_ex(hostname)[0]
except socket.gaierror:
return []
end_letter = hostname.split(".", 1)
# determine the list from the last one.
return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])]
def string_range(last):
"""Compute the range of string between "a" and last.
This works for simple "a to z" lists, but also for "a to zz" lists.
"""
for k in range(len(last)):
for x in product(string.ascii_lowercase, repeat=k+1):
result = ''.join(x)
yield result
if result == last:
return
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/locations.py
-50
View File
@@ -1,50 +0,0 @@
"""Locations where we look for configs, install stuff, etc"""
import sys
import os
from pip.backwardcompat import get_python_lib
def running_under_virtualenv():
"""
Return True if we're running inside a virtualenv, False otherwise.
"""
return hasattr(sys, 'real_prefix')
if running_under_virtualenv():
## FIXME: is build/ a good name?
build_prefix = os.path.join(sys.prefix, 'build')
src_prefix = os.path.join(sys.prefix, 'src')
else:
## FIXME: this isn't a very good default
build_prefix = os.path.join(os.getcwd(), 'build')
src_prefix = os.path.join(os.getcwd(), 'src')
# under Mac OS X + virtualenv sys.prefix is not properly resolved
# it is something like /path/to/python/bin/..
build_prefix = os.path.abspath(build_prefix)
src_prefix = os.path.abspath(src_prefix)
# FIXME doesn't account for venv linked to global site-packages
site_packages = get_python_lib()
user_dir = os.path.expanduser('~')
if sys.platform == 'win32':
bin_py = os.path.join(sys.prefix, 'Scripts')
# buildout uses 'bin' on Windows too?
if not os.path.exists(bin_py):
bin_py = os.path.join(sys.prefix, 'bin')
user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming
default_storage_dir = os.path.join(user_dir, 'pip')
default_config_file = os.path.join(default_storage_dir, 'pip.ini')
default_log_file = os.path.join(default_storage_dir, 'pip.log')
else:
bin_py = os.path.join(sys.prefix, 'bin')
default_storage_dir = os.path.join(user_dir, '.pip')
default_config_file = os.path.join(default_storage_dir, 'pip.conf')
default_log_file = os.path.join(default_storage_dir, 'pip.log')
# Forcing to use /usr/local/bin for standard Mac OS X framework installs
if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
bin_py = '/usr/local/bin'
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/log.py
-185
View File
@@ -1,185 +0,0 @@
"""Logging
"""
import sys
import logging
class Logger(object):
"""
Logging object for use in command-line script. Allows ranges of
levels, to avoid some redundancy of displayed information.
"""
VERBOSE_DEBUG = logging.DEBUG-1
DEBUG = logging.DEBUG
INFO = logging.INFO
NOTIFY = (logging.INFO+logging.WARN)/2
WARN = WARNING = logging.WARN
ERROR = logging.ERROR
FATAL = logging.FATAL
LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
def __init__(self):
self.consumers = []
self.indent = 0
self.explicit_levels = False
self.in_progress = None
self.in_progress_hanging = False
def debug(self, msg, *args, **kw):
self.log(self.DEBUG, msg, *args, **kw)
def info(self, msg, *args, **kw):
self.log(self.INFO, msg, *args, **kw)
def notify(self, msg, *args, **kw):
self.log(self.NOTIFY, msg, *args, **kw)
def warn(self, msg, *args, **kw):
self.log(self.WARN, msg, *args, **kw)
def error(self, msg, *args, **kw):
self.log(self.WARN, msg, *args, **kw)
def fatal(self, msg, *args, **kw):
self.log(self.FATAL, msg, *args, **kw)
def log(self, level, msg, *args, **kw):
if args:
if kw:
raise TypeError(
"You may give positional or keyword arguments, not both")
args = args or kw
rendered = None
for consumer_level, consumer in self.consumers:
if self.level_matches(level, consumer_level):
if (self.in_progress_hanging
and consumer in (sys.stdout, sys.stderr)):
self.in_progress_hanging = False
sys.stdout.write('\n')
sys.stdout.flush()
if rendered is None:
if args:
rendered = msg % args
else:
rendered = msg
rendered = ' '*self.indent + rendered
if self.explicit_levels:
## FIXME: should this be a name, not a level number?
rendered = '%02i %s' % (level, rendered)
if hasattr(consumer, 'write'):
consumer.write(rendered+'\n')
else:
consumer(rendered)
def _show_progress(self):
"""Should we display download progress?"""
return (self.stdout_level_matches(self.NOTIFY) and sys.stdout.isatty())
def start_progress(self, msg):
assert not self.in_progress, (
"Tried to start_progress(%r) while in_progress %r"
% (msg, self.in_progress))
if self._show_progress():
sys.stdout.write(' '*self.indent + msg)
sys.stdout.flush()
self.in_progress_hanging = True
else:
self.in_progress_hanging = False
self.in_progress = msg
self.last_message = None
def end_progress(self, msg='done.'):
assert self.in_progress, (
"Tried to end_progress without start_progress")
if self._show_progress():
if not self.in_progress_hanging:
# Some message has been printed out since start_progress
sys.stdout.write('...' + self.in_progress + msg + '\n')
sys.stdout.flush()
else:
# These erase any messages shown with show_progress (besides .'s)
logger.show_progress('')
logger.show_progress('')
sys.stdout.write(msg + '\n')
sys.stdout.flush()
self.in_progress = None
self.in_progress_hanging = False
def show_progress(self, message=None):
"""If we are in a progress scope, and no log messages have been
shown, write out another '.'"""
if self.in_progress_hanging:
if message is None:
sys.stdout.write('.')
sys.stdout.flush()
else:
if self.last_message:
padding = ' ' * max(0, len(self.last_message)-len(message))
else:
padding = ''
sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding))
sys.stdout.flush()
self.last_message = message
def stdout_level_matches(self, level):
"""Returns true if a message at this level will go to stdout"""
return self.level_matches(level, self._stdout_level())
def _stdout_level(self):
"""Returns the level that stdout runs at"""
for level, consumer in self.consumers:
if consumer is sys.stdout:
return level
return self.FATAL
def level_matches(self, level, consumer_level):
"""
>>> l = Logger()
>>> l.level_matches(3, 4)
False
>>> l.level_matches(3, 2)
True
>>> l.level_matches(slice(None, 3), 3)
False
>>> l.level_matches(slice(None, 3), 2)
True
>>> l.level_matches(slice(1, 3), 1)
True
>>> l.level_matches(slice(2, 3), 1)
False
"""
if isinstance(level, slice):
start, stop = level.start, level.stop
if start is not None and start > consumer_level:
return False
if stop is not None or stop <= consumer_level:
return False
return True
else:
return level >= consumer_level
@classmethod
def level_for_integer(cls, level):
levels = cls.LEVELS
if level < 0:
return levels[0]
if level >= len(levels):
return levels[-1]
return levels[level]
def move_stdout_to_stderr(self):
to_remove = []
to_add = []
for consumer_level, consumer in self.consumers:
if consumer == sys.stdout:
to_remove.append((consumer_level, consumer))
to_add.append((consumer_level, sys.stderr))
for item in to_remove:
self.consumers.remove(item)
self.consumers.extend(to_add)
logger = Logger()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/req.py
-1493
View File
File diff suppressed because it is too large Load Diff
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/runner.py
-18
View File
@@ -1,18 +0,0 @@
import sys
import os
def run():
base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
## FIXME: this is kind of crude; if we could create a fake pip
## module, then exec into it and update pip.__path__ properly, we
## wouldn't have to update sys.path:
sys.path.insert(0, base)
import pip
return pip.main()
if __name__ == '__main__':
exit = run()
if exit:
sys.exit(exit)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/util.py
-489
View File
@@ -1,489 +0,0 @@
import sys
import shutil
import os
import stat
import re
import posixpath
import pkg_resources
import zipfile
import tarfile
from pip.exceptions import InstallationError, BadCommand
from pip.backwardcompat import WindowsError, string_types, raw_input
from pip.locations import site_packages, running_under_virtualenv
from pip.log import logger
__all__ = ['rmtree', 'display_path', 'backup_dir',
'find_command', 'ask', 'Inf',
'normalize_name', 'splitext',
'format_size', 'is_installable_dir',
'is_svn_page', 'file_contents',
'split_leading_dir', 'has_leading_dir',
'make_path_relative', 'normalize_path',
'renames', 'get_terminal_size',
'unzip_file', 'untar_file', 'create_download_cache_folder',
'cache_download', 'unpack_file']
def rmtree(dir, ignore_errors=False):
shutil.rmtree(dir, ignore_errors=ignore_errors,
onerror=rmtree_errorhandler)
def rmtree_errorhandler(func, path, exc_info):
"""On Windows, the files in .svn are read-only, so when rmtree() tries to
remove them, an exception is thrown. We catch that here, remove the
read-only attribute, and hopefully continue without problems."""
exctype, value = exc_info[:2]
# On Python 2.4, it will be OSError number 13
# On all more recent Pythons, it'll be WindowsError number 5
if not ((exctype is WindowsError and value.args[0] == 5) or
(exctype is OSError and value.args[0] == 13)):
raise
# file type should currently be read only
if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD):
raise
# convert to read/write
os.chmod(path, stat.S_IWRITE)
# use the original function to repeat the operation
func(path)
def display_path(path):
"""Gives the display value for a given path, making it relative to cwd
if possible."""
path = os.path.normcase(os.path.abspath(path))
if path.startswith(os.getcwd() + os.path.sep):
path = '.' + path[len(os.getcwd()):]
return path
def backup_dir(dir, ext='.bak'):
"""Figure out the name of a directory to back up the given dir to
(adding .bak, .bak2, etc)"""
n = 1
extension = ext
while os.path.exists(dir + extension):
n += 1
extension = ext + str(n)
return dir + extension
def find_command(cmd, paths=None, pathext=None):
"""Searches the PATH for the given command and returns its path"""
if paths is None:
paths = os.environ.get('PATH', '').split(os.pathsep)
if isinstance(paths, string_types):
paths = [paths]
# check if there are funny path extensions for executables, e.g. Windows
if pathext is None:
pathext = get_pathext()
pathext = [ext for ext in pathext.lower().split(os.pathsep)]
# don't use extensions if the command ends with one of them
if os.path.splitext(cmd)[1].lower() in pathext:
pathext = ['']
# check if we find the command on PATH
for path in paths:
# try without extension first
cmd_path = os.path.join(path, cmd)
for ext in pathext:
# then including the extension
cmd_path_ext = cmd_path + ext
if os.path.isfile(cmd_path_ext):
return cmd_path_ext
if os.path.isfile(cmd_path):
return cmd_path
raise BadCommand('Cannot find command %r' % cmd)
def get_pathext(default_pathext=None):
"""Returns the path extensions from environment or a default"""
if default_pathext is None:
default_pathext = os.pathsep.join([ '.COM', '.EXE', '.BAT', '.CMD' ])
pathext = os.environ.get('PATHEXT', default_pathext)
return pathext
def ask(message, options):
"""Ask the message interactively, with the given possible responses"""
while 1:
if os.environ.get('PIP_NO_INPUT'):
raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message)
response = raw_input(message)
response = response.strip().lower()
if response not in options:
print('Your response (%r) was not one of the expected responses: %s' % (
response, ', '.join(options)))
else:
return response
class _Inf(object):
"""I am bigger than everything!"""
def __cmp__(self, a):
if self is a:
return 0
return 1
def __repr__(self):
return 'Inf'
Inf = _Inf()
del _Inf
_normalize_re = re.compile(r'[^a-z]', re.I)
def normalize_name(name):
return _normalize_re.sub('-', name.lower())
def format_size(bytes):
if bytes > 1000*1000:
return '%.1fMb' % (bytes/1000.0/1000)
elif bytes > 10*1000:
return '%iKb' % (bytes/1000)
elif bytes > 1000:
return '%.1fKb' % (bytes/1000.0)
else:
return '%ibytes' % bytes
def is_installable_dir(path):
"""Return True if `path` is a directory containing a setup.py file."""
if not os.path.isdir(path):
return False
setup_py = os.path.join(path, 'setup.py')
if os.path.isfile(setup_py):
return True
return False
def is_svn_page(html):
"""Returns true if the page appears to be the index page of an svn repository"""
return (re.search(r'<title>[^<]*Revision \d+:', html)
and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I))
def file_contents(filename):
fp = open(filename, 'rb')
try:
return fp.read().decode('utf-8')
finally:
fp.close()
def split_leading_dir(path):
path = str(path)
path = path.lstrip('/').lstrip('\\')
if '/' in path and (('\\' in path and path.find('/') < path.find('\\'))
or '\\' not in path):
return path.split('/', 1)
elif '\\' in path:
return path.split('\\', 1)
else:
return path, ''
def has_leading_dir(paths):
"""Returns true if all the paths have the same leading path name
(i.e., everything is in one subdirectory in an archive)"""
common_prefix = None
for path in paths:
prefix, rest = split_leading_dir(path)
if not prefix:
return False
elif common_prefix is None:
common_prefix = prefix
elif prefix != common_prefix:
return False
return True
def make_path_relative(path, rel_to):
"""
Make a filename relative, where the filename path, and it is
relative to rel_to
>>> make_relative_path('/usr/share/something/a-file.pth',
... '/usr/share/another-place/src/Directory')
'../../../something/a-file.pth'
>>> make_relative_path('/usr/share/something/a-file.pth',
... '/home/user/src/Directory')
'../../../usr/share/something/a-file.pth'
>>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
'a-file.pth'
"""
path_filename = os.path.basename(path)
path = os.path.dirname(path)
path = os.path.normpath(os.path.abspath(path))
rel_to = os.path.normpath(os.path.abspath(rel_to))
path_parts = path.strip(os.path.sep).split(os.path.sep)
rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep)
while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]:
path_parts.pop(0)
rel_to_parts.pop(0)
full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename]
if full_parts == ['']:
return '.' + os.path.sep
return os.path.sep.join(full_parts)
def normalize_path(path):
"""
Convert a path to its canonical, case-normalized, absolute version.
"""
return os.path.normcase(os.path.realpath(path))
def splitext(path):
"""Like os.path.splitext, but take off .tar too"""
base, ext = posixpath.splitext(path)
if base.lower().endswith('.tar'):
ext = base[-4:] + ext
base = base[:-4]
return base, ext
def renames(old, new):
"""Like os.renames(), but handles renaming across devices."""
# Implementation borrowed from os.renames().
head, tail = os.path.split(new)
if head and tail and not os.path.exists(head):
os.makedirs(head)
shutil.move(old, new)
head, tail = os.path.split(old)
if head and tail:
try:
os.removedirs(head)
except OSError:
pass
def is_local(path):
"""
Return True if path is within sys.prefix, if we're running in a virtualenv.
If we're not in a virtualenv, all paths are considered "local."
"""
if not running_under_virtualenv():
return True
return normalize_path(path).startswith(normalize_path(sys.prefix))
def dist_is_local(dist):
"""
Return True if given Distribution object is installed locally
(i.e. within current virtualenv).
Always True if we're not in a virtualenv.
"""
return is_local(dist_location(dist))
def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')):
"""
Return a list of installed Distribution objects.
If ``local_only`` is True (default), only return installations
local to the current virtualenv, if in a virtualenv.
``skip`` argument is an iterable of lower-case project names to
ignore; defaults to ('setuptools', 'pip', 'python'). [FIXME also
skip virtualenv?]
"""
if local_only:
local_test = dist_is_local
else:
local_test = lambda d: True
return [d for d in pkg_resources.working_set if local_test(d) and d.key not in skip]
def egg_link_path(dist):
"""
Return the path where we'd expect to find a .egg-link file for
this distribution. (There doesn't seem to be any metadata in the
Distribution object for a develop egg that points back to its
.egg-link and easy-install.pth files).
This won't find a globally-installed develop egg if we're in a
virtualenv.
"""
return os.path.join(site_packages, dist.project_name) + '.egg-link'
def dist_location(dist):
"""
Get the site-packages location of this distribution. Generally
this is dist.location, except in the case of develop-installed
packages, where dist.location is the source code location, and we
want to know where the egg-link file is.
"""
egg_link = egg_link_path(dist)
if os.path.exists(egg_link):
return egg_link
return dist.location
def get_terminal_size():
"""Returns a tuple (x, y) representing the width(x) and the height(x)
in characters of the terminal window."""
def ioctl_GWINSZ(fd):
try:
import fcntl
import termios
import struct
cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
'1234'))
except:
return None
if cr == (0, 0):
return None
if cr == (0, 0):
return None
return cr
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr:
try:
fd = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_GWINSZ(fd)
os.close(fd)
except:
pass
if not cr:
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
return int(cr[1]), int(cr[0])
def unzip_file(filename, location, flatten=True):
"""Unzip the file (zip file located at filename) to the destination
location"""
if not os.path.exists(location):
os.makedirs(location)
zipfp = open(filename, 'rb')
try:
zip = zipfile.ZipFile(zipfp)
leading = has_leading_dir(zip.namelist()) and flatten
for name in zip.namelist():
data = zip.read(name)
fn = name
if leading:
fn = split_leading_dir(name)[1]
fn = os.path.join(location, fn)
dir = os.path.dirname(fn)
if not os.path.exists(dir):
os.makedirs(dir)
if fn.endswith('/') or fn.endswith('\\'):
# A directory
if not os.path.exists(fn):
os.makedirs(fn)
else:
fp = open(fn, 'wb')
try:
fp.write(data)
finally:
fp.close()
finally:
zipfp.close()
def untar_file(filename, location):
"""Untar the file (tar file located at filename) to the destination location"""
if not os.path.exists(location):
os.makedirs(location)
if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
mode = 'r:gz'
elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'):
mode = 'r:bz2'
elif filename.lower().endswith('.tar'):
mode = 'r'
else:
logger.warn('Cannot determine compression type for file %s' % filename)
mode = 'r:*'
tar = tarfile.open(filename, mode)
try:
# note: python<=2.5 doesnt seem to know about pax headers, filter them
leading = has_leading_dir([
member.name for member in tar.getmembers()
if member.name != 'pax_global_header'
])
for member in tar.getmembers():
fn = member.name
if fn == 'pax_global_header':
continue
if leading:
fn = split_leading_dir(fn)[1]
path = os.path.join(location, fn)
if member.isdir():
if not os.path.exists(path):
os.makedirs(path)
else:
try:
fp = tar.extractfile(member)
except (KeyError, AttributeError):
e = sys.exc_info()[1]
# Some corrupt tar files seem to produce this
# (specifically bad symlinks)
logger.warn(
'In the tar file %s the member %s is invalid: %s'
% (filename, member.name, e))
continue
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
destfp = open(path, 'wb')
try:
shutil.copyfileobj(fp, destfp)
finally:
destfp.close()
fp.close()
finally:
tar.close()
def create_download_cache_folder(folder):
logger.indent -= 2
logger.notify('Creating supposed download cache at %s' % folder)
logger.indent += 2
os.makedirs(folder)
def cache_download(target_file, temp_location, content_type):
logger.notify('Storing download in cache at %s' % display_path(target_file))
shutil.copyfile(temp_location, target_file)
fp = open(target_file+'.content-type', 'w')
fp.write(content_type)
fp.close()
os.unlink(temp_location)
def unpack_file(filename, location, content_type, link):
if (content_type == 'application/zip'
or filename.endswith('.zip')
or filename.endswith('.pybundle')
or zipfile.is_zipfile(filename)):
unzip_file(filename, location, flatten=not filename.endswith('.pybundle'))
elif (content_type == 'application/x-gzip'
or tarfile.is_tarfile(filename)
or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')):
untar_file(filename, location)
elif (content_type and content_type.startswith('text/html')
and is_svn_page(file_contents(filename))):
# We don't really care about this
from pip.vcs.subversion import Subversion
Subversion('svn+' + link.url).unpack(location)
else:
## FIXME: handle?
## FIXME: magic signatures?
logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format'
% (filename, location, content_type))
raise InstallationError('Cannot determine archive format of %s' % location)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/vcs/__init__.py
-236
View File
@@ -1,236 +0,0 @@
"""Handles all VCS (version control) support"""
import os
import shutil
from pip.backwardcompat import urlparse, urllib
from pip.log import logger
from pip.util import display_path, backup_dir, find_command, ask, rmtree
__all__ = ['vcs', 'get_src_requirement']
class VcsSupport(object):
_registry = {}
schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn']
def __init__(self):
# Register more schemes with urlparse for various version control systems
urlparse.uses_netloc.extend(self.schemes)
urlparse.uses_fragment.extend(self.schemes)
super(VcsSupport, self).__init__()
def __iter__(self):
return self._registry.__iter__()
@property
def backends(self):
return list(self._registry.values())
@property
def dirnames(self):
return [backend.dirname for backend in self.backends]
@property
def all_schemes(self):
schemes = []
for backend in self.backends:
schemes.extend(backend.schemes)
return schemes
def register(self, cls):
if not hasattr(cls, 'name'):
logger.warn('Cannot register VCS %s' % cls.__name__)
return
if cls.name not in self._registry:
self._registry[cls.name] = cls
def unregister(self, cls=None, name=None):
if name in self._registry:
del self._registry[name]
elif cls in self._registry.values():
del self._registry[cls.name]
else:
logger.warn('Cannot unregister because no class or name given')
def get_backend_name(self, location):
"""
Return the name of the version control backend if found at given
location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
"""
for vc_type in self._registry.values():
path = os.path.join(location, vc_type.dirname)
if os.path.exists(path):
return vc_type.name
return None
def get_backend(self, name):
name = name.lower()
if name in self._registry:
return self._registry[name]
def get_backend_from_location(self, location):
vc_type = self.get_backend_name(location)
if vc_type:
return self.get_backend(vc_type)
return None
vcs = VcsSupport()
class VersionControl(object):
name = ''
dirname = ''
def __init__(self, url=None, *args, **kwargs):
self.url = url
self._cmd = None
super(VersionControl, self).__init__(*args, **kwargs)
def _filter(self, line):
return (logger.INFO, line)
def _is_local_repository(self, repo):
"""
posix absolute paths start with os.path.sep,
win32 ones ones start with drive (like c:\\folder)
"""
drive, tail = os.path.splitdrive(repo)
return repo.startswith(os.path.sep) or drive
@property
def cmd(self):
if self._cmd is not None:
return self._cmd
command = find_command(self.name)
logger.info('Found command %r at %r' % (self.name, command))
self._cmd = command
return command
def get_url_rev(self):
"""
Returns the correct repository URL and revision by parsing the given
repository URL
"""
url = self.url.split('+', 1)[1]
scheme, netloc, path, query, frag = urlparse.urlsplit(url)
rev = None
if '@' in path:
path, rev = path.rsplit('@', 1)
url = urlparse.urlunsplit((scheme, netloc, path, query, ''))
return url, rev
def get_info(self, location):
"""
Returns (url, revision), where both are strings
"""
assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location
return self.get_url(location), self.get_revision(location)
def normalize_url(self, url):
"""
Normalize a URL for comparison by unquoting it and removing any trailing slash.
"""
return urllib.unquote(url).rstrip('/')
def compare_urls(self, url1, url2):
"""
Compare two repo URLs for identity, ignoring incidental differences.
"""
return (self.normalize_url(url1) == self.normalize_url(url2))
def parse_vcs_bundle_file(self, content):
"""
Takes the contents of the bundled text file that explains how to revert
the stripped off version control data of the given package and returns
the URL and revision of it.
"""
raise NotImplementedError
def obtain(self, dest):
"""
Called when installing or updating an editable package, takes the
source path of the checkout.
"""
raise NotImplementedError
def switch(self, dest, url, rev_options):
"""
Switch the repo at ``dest`` to point to ``URL``.
"""
raise NotImplemented
def update(self, dest, rev_options):
"""
Update an already-existing repo to the given ``rev_options``.
"""
raise NotImplementedError
def check_destination(self, dest, url, rev_options, rev_display):
"""
Prepare a location to receive a checkout/clone.
Return True if the location is ready for (and requires) a
checkout/clone, False otherwise.
"""
checkout = True
prompt = False
if os.path.exists(dest):
checkout = False
if os.path.exists(os.path.join(dest, self.dirname)):
existing_url = self.get_url(dest)
if self.compare_urls(existing_url, url):
logger.info('%s in %s exists, and has correct URL (%s)'
% (self.repo_name.title(), display_path(dest), url))
logger.notify('Updating %s %s%s'
% (display_path(dest), self.repo_name, rev_display))
self.update(dest, rev_options)
else:
logger.warn('%s %s in %s exists with URL %s'
% (self.name, self.repo_name, display_path(dest), existing_url))
prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
else:
logger.warn('Directory %s already exists, and is not a %s %s.'
% (dest, self.name, self.repo_name))
prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
if prompt:
logger.warn('The plan is to install the %s repository %s'
% (self.name, url))
response = ask('What to do? %s' % prompt[0], prompt[1])
if response == 's':
logger.notify('Switching %s %s to %s%s'
% (self.repo_name, display_path(dest), url, rev_display))
self.switch(dest, url, rev_options)
elif response == 'i':
# do nothing
pass
elif response == 'w':
logger.warn('Deleting %s' % display_path(dest))
rmtree(dest)
checkout = True
elif response == 'b':
dest_dir = backup_dir(dest)
logger.warn('Backing up %s to %s'
% (display_path(dest), dest_dir))
shutil.move(dest, dest_dir)
checkout = True
return checkout
def unpack(self, location):
if os.path.exists(location):
rmtree(location)
self.obtain(location)
def get_src_requirement(self, dist, location, find_tags=False):
raise NotImplementedError
def get_src_requirement(dist, location, find_tags):
version_control = vcs.get_backend_from_location(location)
if version_control:
return version_control().get_src_requirement(dist, location, find_tags)
logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location)
return dist.as_requirement()
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/vcs/bazaar.py
-123
View File
@@ -1,123 +0,0 @@
import os
import tempfile
import re
from pip import call_subprocess
from pip.log import logger
from pip.util import rmtree, display_path
from pip.vcs import vcs, VersionControl
from pip.download import path_to_url2
class Bazaar(VersionControl):
name = 'bzr'
dirname = '.bzr'
repo_name = 'branch'
bundle_file = 'bzr-branch.txt'
schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp')
guide = ('# This was a Bazaar branch; to make it a branch again run:\n'
'bzr branch -r %(rev)s %(url)s .\n')
def parse_vcs_bundle_file(self, content):
url = rev = None
for line in content.splitlines():
if not line.strip() or line.strip().startswith('#'):
continue
match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line)
if match:
rev = match.group(1).strip()
url = line[match.end():].strip().split(None, 1)[0]
if url and rev:
return url, rev
return None, None
def export(self, location):
"""Export the Bazaar repository at the url to the destination location"""
temp_dir = tempfile.mkdtemp('-export', 'pip-')
self.unpack(temp_dir)
if os.path.exists(location):
# Remove the location to make sure Bazaar can export it correctly
rmtree(location)
try:
call_subprocess([self.cmd, 'export', location], cwd=temp_dir,
filter_stdout=self._filter, show_stdout=False)
finally:
rmtree(temp_dir)
def switch(self, dest, url, rev_options):
call_subprocess([self.cmd, 'switch', url], cwd=dest)
def update(self, dest, rev_options):
call_subprocess(
[self.cmd, 'pull', '-q'] + rev_options, cwd=dest)
def obtain(self, dest):
url, rev = self.get_url_rev()
if rev:
rev_options = ['-r', rev]
rev_display = ' (to revision %s)' % rev
else:
rev_options = []
rev_display = ''
if self.check_destination(dest, url, rev_options, rev_display):
logger.notify('Checking out %s%s to %s'
% (url, rev_display, display_path(dest)))
call_subprocess(
[self.cmd, 'branch', '-q'] + rev_options + [url, dest])
def get_url_rev(self):
# hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it
url, rev = super(Bazaar, self).get_url_rev()
if url.startswith('ssh://'):
url = 'bzr+' + url
return url, rev
def get_url(self, location):
urls = call_subprocess(
[self.cmd, 'info'], show_stdout=False, cwd=location)
for line in urls.splitlines():
line = line.strip()
for x in ('checkout of branch: ',
'parent branch: '):
if line.startswith(x):
repo = line.split(x)[1]
if self._is_local_repository(repo):
return path_to_url2(repo)
return repo
return None
def get_revision(self, location):
revision = call_subprocess(
[self.cmd, 'revno'], show_stdout=False, cwd=location)
return revision.splitlines()[-1]
def get_tag_revs(self, location):
tags = call_subprocess(
[self.cmd, 'tags'], show_stdout=False, cwd=location)
tag_revs = []
for line in tags.splitlines():
tags_match = re.search(r'([.\w-]+)\s*(.*)$', line)
if tags_match:
tag = tags_match.group(1)
rev = tags_match.group(2)
tag_revs.append((rev.strip(), tag.strip()))
return dict(tag_revs)
def get_src_requirement(self, dist, location, find_tags):
repo = self.get_url(location)
if not repo.lower().startswith('bzr:'):
repo = 'bzr+' + repo
egg_project_name = dist.egg_name().split('-', 1)[0]
if not repo:
return None
current_rev = self.get_revision(location)
tag_revs = self.get_tag_revs(location)
if current_rev in tag_revs:
# It's a tag
full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
else:
full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev)
return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name)
vcs.register(Bazaar)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/vcs/git.py
-205
View File
@@ -1,205 +0,0 @@
import tempfile
import re
from pip import call_subprocess
from pip.util import display_path, rmtree
from pip.vcs import vcs, VersionControl
from pip.log import logger
from pip.backwardcompat import url2pathname, urlparse
urlsplit = urlparse.urlsplit
urlunsplit = urlparse.urlunsplit
class Git(VersionControl):
name = 'git'
dirname = '.git'
repo_name = 'clone'
schemes = ('git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file')
bundle_file = 'git-clone.txt'
guide = ('# This was a Git repo; to make it a repo again run:\n'
'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n')
def __init__(self, url=None, *args, **kwargs):
# Works around an apparent Git bug
# (see http://article.gmane.org/gmane.comp.version-control.git/146500)
if url:
scheme, netloc, path, query, fragment = urlsplit(url)
if scheme.endswith('file'):
initial_slashes = path[:-len(path.lstrip('/'))]
newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/')
url = urlunsplit((scheme, netloc, newpath, query, fragment))
after_plus = scheme.find('+')+1
url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment))
super(Git, self).__init__(url, *args, **kwargs)
def parse_vcs_bundle_file(self, content):
url = rev = None
for line in content.splitlines():
if not line.strip() or line.strip().startswith('#'):
continue
url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line)
if url_match:
url = url_match.group(1).strip()
rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line)
if rev_match:
rev = rev_match.group(1).strip()
if url and rev:
return url, rev
return None, None
def export(self, location):
"""Export the Git repository at the url to the destination location"""
temp_dir = tempfile.mkdtemp('-export', 'pip-')
self.unpack(temp_dir)
try:
if not location.endswith('/'):
location = location + '/'
call_subprocess(
[self.cmd, 'checkout-index', '-a', '-f', '--prefix', location],
filter_stdout=self._filter, show_stdout=False, cwd=temp_dir)
finally:
rmtree(temp_dir)
def check_rev_options(self, rev, dest, rev_options):
"""Check the revision options before checkout to compensate that tags
and branches may need origin/ as a prefix.
Returns the SHA1 of the branch or tag if found.
"""
revisions = self.get_tag_revs(dest)
revisions.update(self.get_branch_revs(dest))
origin_rev = 'origin/%s' % rev
if origin_rev in revisions:
# remote branch
return [revisions[origin_rev]]
elif rev in revisions:
# a local tag or branch name
return [revisions[rev]]
else:
logger.warn("Could not find a tag or branch '%s', assuming commit." % rev)
return rev_options
def switch(self, dest, url, rev_options):
call_subprocess(
[self.cmd, 'config', 'remote.origin.url', url], cwd=dest)
call_subprocess(
[self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
def update(self, dest, rev_options):
# First fetch changes from the default remote
call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest)
# Then reset to wanted revision (maby even origin/master)
if rev_options:
rev_options = self.check_rev_options(rev_options[0], dest, rev_options)
call_subprocess([self.cmd, 'reset', '--hard', '-q'] + rev_options, cwd=dest)
def obtain(self, dest):
url, rev = self.get_url_rev()
if rev:
rev_options = [rev]
rev_display = ' (to %s)' % rev
else:
rev_options = ['origin/master']
rev_display = ''
if self.check_destination(dest, url, rev_options, rev_display):
logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest)))
call_subprocess([self.cmd, 'clone', '-q', url, dest])
if rev:
rev_options = self.check_rev_options(rev, dest, rev_options)
# Only do a checkout if rev_options differs from HEAD
if not self.get_revision(dest).startswith(rev_options[0]):
call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
def get_url(self, location):
url = call_subprocess(
[self.cmd, 'config', 'remote.origin.url'],
show_stdout=False, cwd=location)
return url.strip()
def get_revision(self, location):
current_rev = call_subprocess(
[self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location)
return current_rev.strip()
def get_tag_revs(self, location):
tags = self._get_all_tag_names(location)
tag_revs = {}
for line in tags.splitlines():
tag = line.strip()
rev = self._get_revision_from_rev_parse(tag, location)
tag_revs[tag] = rev.strip()
return tag_revs
def get_branch_revs(self, location):
branches = self._get_all_branch_names(location)
branch_revs = {}
for line in branches.splitlines():
if '(no branch)' in line:
continue
line = line.split('->')[0].strip()
# actual branch case
branch = "".join(b for b in line.split() if b != '*')
rev = self._get_revision_from_rev_parse(branch, location)
branch_revs[branch] = rev.strip()
return branch_revs
def get_src_requirement(self, dist, location, find_tags):
repo = self.get_url(location)
if not repo.lower().startswith('git:'):
repo = 'git+' + repo
egg_project_name = dist.egg_name().split('-', 1)[0]
if not repo:
return None
current_rev = self.get_revision(location)
tag_revs = self.get_tag_revs(location)
branch_revs = self.get_branch_revs(location)
if current_rev in tag_revs:
# It's a tag
full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
elif (current_rev in branch_revs and
branch_revs[current_rev] != 'origin/master'):
# It's the head of a branch
full_egg_name = '%s-%s' % (egg_project_name,
branch_revs[current_rev].replace('origin/', ''))
else:
full_egg_name = '%s-dev' % egg_project_name
return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name)
def get_url_rev(self):
"""
Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
That's required because although they use SSH they sometimes doesn't
work with a ssh:// scheme (e.g. Github). But we need a scheme for
parsing. Hence we remove it again afterwards and return it as a stub.
"""
if not '://' in self.url:
assert not 'file:' in self.url
self.url = self.url.replace('git+', 'git+ssh://')
url, rev = super(Git, self).get_url_rev()
url = url.replace('ssh://', '')
else:
url, rev = super(Git, self).get_url_rev()
return url, rev
def _get_all_tag_names(self, location):
return call_subprocess([self.cmd, 'tag', '-l'],
show_stdout=False,
raise_on_returncode=False,
cwd=location)
def _get_all_branch_names(self, location):
remote_branches = call_subprocess([self.cmd, 'branch', '-r'],
show_stdout=False, cwd=location)
local_branches = call_subprocess([self.cmd, 'branch', '-l'],
show_stdout=False, cwd=location)
return remote_branches + local_branches
def _get_revision_from_rev_parse(self, name, location):
return call_subprocess([self.cmd, 'rev-parse', name],
show_stdout=False, cwd=location)
vcs.register(Git)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/vcs/mercurial.py
-151
View File
@@ -1,151 +0,0 @@
import os
import tempfile
import re
import sys
from pip import call_subprocess
from pip.util import display_path, rmtree
from pip.log import logger
from pip.vcs import vcs, VersionControl
from pip.download import path_to_url2
from pip.backwardcompat import ConfigParser
class Mercurial(VersionControl):
name = 'hg'
dirname = '.hg'
repo_name = 'clone'
schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http')
bundle_file = 'hg-clone.txt'
guide = ('# This was a Mercurial repo; to make it a repo again run:\n'
'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n')
def parse_vcs_bundle_file(self, content):
url = rev = None
for line in content.splitlines():
if not line.strip() or line.strip().startswith('#'):
continue
url_match = re.search(r'hg\s*pull\s*(.*)\s*', line)
if url_match:
url = url_match.group(1).strip()
rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line)
if rev_match:
rev = rev_match.group(1).strip()
if url and rev:
return url, rev
return None, None
def export(self, location):
"""Export the Hg repository at the url to the destination location"""
temp_dir = tempfile.mkdtemp('-export', 'pip-')
self.unpack(temp_dir)
try:
call_subprocess(
[self.cmd, 'archive', location],
filter_stdout=self._filter, show_stdout=False, cwd=temp_dir)
finally:
rmtree(temp_dir)
def switch(self, dest, url, rev_options):
repo_config = os.path.join(dest, self.dirname, 'hgrc')
config = ConfigParser.SafeConfigParser()
try:
config.read(repo_config)
config.set('paths', 'default', url)
config_file = open(repo_config, 'w')
config.write(config_file)
config_file.close()
except (OSError, ConfigParser.NoSectionError):
e = sys.exc_info()[1]
logger.warn(
'Could not switch Mercurial repository to %s: %s'
% (url, e))
else:
call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def update(self, dest, rev_options):
call_subprocess([self.cmd, 'pull', '-q'], cwd=dest)
call_subprocess(
[self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def obtain(self, dest):
url, rev = self.get_url_rev()
if rev:
rev_options = [rev]
rev_display = ' (to revision %s)' % rev
else:
rev_options = []
rev_display = ''
if self.check_destination(dest, url, rev_options, rev_display):
logger.notify('Cloning hg %s%s to %s'
% (url, rev_display, display_path(dest)))
call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest])
call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def get_url(self, location):
url = call_subprocess(
[self.cmd, 'showconfig', 'paths.default'],
show_stdout=False, cwd=location).strip()
if self._is_local_repository(url):
url = path_to_url2(url)
return url.strip()
def get_tag_revs(self, location):
tags = call_subprocess(
[self.cmd, 'tags'], show_stdout=False, cwd=location)
tag_revs = []
for line in tags.splitlines():
tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line)
if tags_match:
tag = tags_match.group(1)
rev = tags_match.group(2)
if "tip" != tag:
tag_revs.append((rev.strip(), tag.strip()))
return dict(tag_revs)
def get_branch_revs(self, location):
branches = call_subprocess(
[self.cmd, 'branches'], show_stdout=False, cwd=location)
branch_revs = []
for line in branches.splitlines():
branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line)
if branches_match:
branch = branches_match.group(1)
rev = branches_match.group(2)
if "default" != branch:
branch_revs.append((rev.strip(), branch.strip()))
return dict(branch_revs)
def get_revision(self, location):
current_revision = call_subprocess(
[self.cmd, 'parents', '--template={rev}'],
show_stdout=False, cwd=location).strip()
return current_revision
def get_revision_hash(self, location):
current_rev_hash = call_subprocess(
[self.cmd, 'parents', '--template={node}'],
show_stdout=False, cwd=location).strip()
return current_rev_hash
def get_src_requirement(self, dist, location, find_tags):
repo = self.get_url(location)
if not repo.lower().startswith('hg:'):
repo = 'hg+' + repo
egg_project_name = dist.egg_name().split('-', 1)[0]
if not repo:
return None
current_rev = self.get_revision(location)
current_rev_hash = self.get_revision_hash(location)
tag_revs = self.get_tag_revs(location)
branch_revs = self.get_branch_revs(location)
if current_rev in tag_revs:
# It's a tag
full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
elif current_rev in branch_revs:
# It's the tip of a branch
full_egg_name = '%s-%s' % (egg_project_name, branch_revs[current_rev])
else:
full_egg_name = '%s-dev' % egg_project_name
return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name)
vcs.register(Mercurial)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/vcs/subversion.py
-244
View File
@@ -1,244 +0,0 @@
import os
import re
from pip import call_subprocess
from pip.index import Link
from pip.util import rmtree, display_path
from pip.log import logger
from pip.vcs import vcs, VersionControl
_svn_xml_url_re = re.compile('url="([^"]+)"')
_svn_rev_re = re.compile('committed-rev="(\d+)"')
_svn_url_re = re.compile(r'URL: (.+)')
_svn_revision_re = re.compile(r'Revision: (.+)')
class Subversion(VersionControl):
name = 'svn'
dirname = '.svn'
repo_name = 'checkout'
schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn')
bundle_file = 'svn-checkout.txt'
guide = ('# This was an svn checkout; to make it a checkout again run:\n'
'svn checkout --force -r %(rev)s %(url)s .\n')
def get_info(self, location):
"""Returns (url, revision), where both are strings"""
assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location
output = call_subprocess(
[self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'})
match = _svn_url_re.search(output)
if not match:
logger.warn('Cannot determine URL of svn checkout %s' % display_path(location))
logger.info('Output that cannot be parsed: \n%s' % output)
return None, None
url = match.group(1).strip()
match = _svn_revision_re.search(output)
if not match:
logger.warn('Cannot determine revision of svn checkout %s' % display_path(location))
logger.info('Output that cannot be parsed: \n%s' % output)
return url, None
return url, match.group(1)
def parse_vcs_bundle_file(self, content):
for line in content.splitlines():
if not line.strip() or line.strip().startswith('#'):
continue
match = re.search(r'^-r\s*([^ ])?', line)
if not match:
return None, None
rev = match.group(1)
rest = line[match.end():].strip().split(None, 1)[0]
return rest, rev
return None, None
def export(self, location):
"""Export the svn repository at the url to the destination location"""
url, rev = self.get_url_rev()
logger.notify('Exporting svn repository %s to %s' % (url, location))
logger.indent += 2
try:
if os.path.exists(location):
# Subversion doesn't like to check out over an existing directory
# --force fixes this, but was only added in svn 1.5
rmtree(location)
call_subprocess(
[self.cmd, 'export', url, location],
filter_stdout=self._filter, show_stdout=False)
finally:
logger.indent -= 2
def switch(self, dest, url, rev_options):
call_subprocess(
[self.cmd, 'switch'] + rev_options + [url, dest])
def update(self, dest, rev_options):
call_subprocess(
[self.cmd, 'update'] + rev_options + [dest])
def obtain(self, dest):
url, rev = self.get_url_rev()
if rev:
rev_options = ['-r', rev]
rev_display = ' (to revision %s)' % rev
else:
rev_options = []
rev_display = ''
if self.check_destination(dest, url, rev_options, rev_display):
logger.notify('Checking out %s%s to %s'
% (url, rev_display, display_path(dest)))
call_subprocess(
[self.cmd, 'checkout', '-q'] + rev_options + [url, dest])
def get_location(self, dist, dependency_links):
for url in dependency_links:
egg_fragment = Link(url).egg_fragment
if not egg_fragment:
continue
if '-' in egg_fragment:
## FIXME: will this work when a package has - in the name?
key = '-'.join(egg_fragment.split('-')[:-1]).lower()
else:
key = egg_fragment
if key == dist.key:
return url.split('#', 1)[0]
return None
def get_revision(self, location):
"""
Return the maximum revision for all files under a given location
"""
# Note: taken from setuptools.command.egg_info
revision = 0
for base, dirs, files in os.walk(location):
if self.dirname not in dirs:
dirs[:] = []
continue # no sense walking uncontrolled subdirs
dirs.remove(self.dirname)
entries_fn = os.path.join(base, self.dirname, 'entries')
if not os.path.exists(entries_fn):
## FIXME: should we warn?
continue
f = open(entries_fn)
data = f.read()
f.close()
if data.startswith('8') or data.startswith('9') or data.startswith('10'):
data = list(map(str.splitlines, data.split('\n\x0c\n')))
del data[0][0] # get rid of the '8'
dirurl = data[0][3]
revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0]
if revs:
localrev = max(revs)
else:
localrev = 0
elif data.startswith('<?xml'):
dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL
revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0]
if revs:
localrev = max(revs)
else:
localrev = 0
else:
logger.warn("Unrecognized .svn/entries format; skipping %s", base)
dirs[:] = []
continue
if base == location:
base_url = dirurl+'/' # save the root url
elif not dirurl.startswith(base_url):
dirs[:] = []
continue # not part of the same svn tree, skip it
revision = max(revision, localrev)
return revision
def get_url_rev(self):
# hotfix the URL scheme after removing svn+ from svn+ssh:// readd it
url, rev = super(Subversion, self).get_url_rev()
if url.startswith('ssh://'):
url = 'svn+' + url
return url, rev
def get_url(self, location):
# In cases where the source is in a subdirectory, not alongside setup.py
# we have to look up in the location until we find a real setup.py
orig_location = location
while not os.path.exists(os.path.join(location, 'setup.py')):
last_location = location
location = os.path.dirname(location)
if location == last_location:
# We've traversed up to the root of the filesystem without finding setup.py
logger.warn("Could not find setup.py for directory %s (tried all parent directories)"
% orig_location)
return None
f = open(os.path.join(location, self.dirname, 'entries'))
data = f.read()
f.close()
if data.startswith('8') or data.startswith('9') or data.startswith('10'):
data = list(map(str.splitlines, data.split('\n\x0c\n')))
del data[0][0] # get rid of the '8'
return data[0][3]
elif data.startswith('<?xml'):
match = _svn_xml_url_re.search(data)
if not match:
raise ValueError('Badly formatted data: %r' % data)
return match.group(1) # get repository URL
else:
logger.warn("Unrecognized .svn/entries format in %s" % location)
# Or raise exception?
return None
def get_tag_revs(self, svn_tag_url):
stdout = call_subprocess(
[self.cmd, 'ls', '-v', svn_tag_url], show_stdout=False)
results = []
for line in stdout.splitlines():
parts = line.split()
rev = int(parts[0])
tag = parts[-1].strip('/')
results.append((tag, rev))
return results
def find_tag_match(self, rev, tag_revs):
best_match_rev = None
best_tag = None
for tag, tag_rev in tag_revs:
if (tag_rev > rev and
(best_match_rev is None or best_match_rev > tag_rev)):
# FIXME: Is best_match > tag_rev really possible?
# or is it a sign something is wacky?
best_match_rev = tag_rev
best_tag = tag
return best_tag
def get_src_requirement(self, dist, location, find_tags=False):
repo = self.get_url(location)
if repo is None:
return None
parts = repo.split('/')
## FIXME: why not project name?
egg_project_name = dist.egg_name().split('-', 1)[0]
rev = self.get_revision(location)
if parts[-2] in ('tags', 'tag'):
# It's a tag, perfect!
full_egg_name = '%s-%s' % (egg_project_name, parts[-1])
elif parts[-2] in ('branches', 'branch'):
# It's a branch :(
full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev)
elif parts[-1] == 'trunk':
# Trunk :-/
full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev)
if find_tags:
tag_url = '/'.join(parts[:-1]) + '/tags'
tag_revs = self.get_tag_revs(tag_url)
match = self.find_tag_match(rev, tag_revs)
if match:
logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match)
repo = '%s/%s' % (tag_url, match)
full_egg_name = '%s-%s' % (egg_project_name, match)
else:
# Don't know what it is
logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo)
full_egg_name = '%s-dev_r%s' % (egg_project_name, rev)
return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name)
vcs.register(Subversion)
.pyenv/lib/python2.6/site-packages/pip-1.0.2-py2.6.egg/pip/venv.py
-53
View File
@@ -1,53 +0,0 @@
"""Tools for working with virtualenv environments"""
import os
import sys
import subprocess
from pip.exceptions import BadCommand
from pip.log import logger
def restart_in_venv(venv, base, site_packages, args):
"""
Restart this script using the interpreter in the given virtual environment
"""
if base and not os.path.isabs(venv) and not venv.startswith('~'):
base = os.path.expanduser(base)
# ensure we have an abs basepath at this point:
# a relative one makes no sense (or does it?)
if os.path.isabs(base):
venv = os.path.join(base, venv)
if venv.startswith('~'):
venv = os.path.expanduser(venv)
if not os.path.exists(venv):
try:
import virtualenv
except ImportError:
print('The virtual environment does not exist: %s' % venv)
print('and virtualenv is not installed, so a new environment cannot be created')
sys.exit(3)
print('Creating new virtualenv environment in %s' % venv)
virtualenv.logger = logger
logger.indent += 2
virtualenv.create_environment(venv, site_packages=site_packages)
if sys.platform == 'win32':
python = os.path.join(venv, 'Scripts', 'python.exe')
# check for bin directory which is used in buildouts
if not os.path.exists(python):
python = os.path.join(venv, 'bin', 'python.exe')
else:
python = os.path.join(venv, 'bin', 'python')
if not os.path.exists(python):
python = venv
if not os.path.exists(python):
raise BadCommand('Cannot find virtual environment interpreter at %s' % python)
base = os.path.dirname(os.path.dirname(python))
file = os.path.join(os.path.dirname(__file__), 'runner.py')
if file.endswith('.pyc'):
file = file[:-1]
proc = subprocess.Popen(
[python, file] + args + [base, '___VENV_RESTART___'])
proc.wait()
sys.exit(proc.returncode)
.pyenv/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg
BIN
View File
Binary file not shown.
.pyenv/lib/python2.6/site-packages/setuptools.pth
-1
View File
@@ -1 +0,0 @@
./setuptools-0.6c11-py2.6.egg
.pyenv/lib/python2.6/site.py
-732
View File
@@ -1,732 +0,0 @@
"""Append module search paths for third-party packages to sys.path.
****************************************************************
* This module is automatically imported during initialization. *
****************************************************************
In earlier versions of Python (up to 1.5a3), scripts or modules that
needed to use site-specific modules would place ``import site''
somewhere near the top of their code. Because of the automatic
import, this is no longer necessary (but code that does it still
works).
This will append site-specific paths to the module search path. On
Unix, it starts with sys.prefix and sys.exec_prefix (if different) and
appends lib/python<version>/site-packages as well as lib/site-python.
It also supports the Debian convention of
lib/python<version>/dist-packages. On other platforms (mainly Mac and
Windows), it uses just sys.prefix (and sys.exec_prefix, if different,
but this is unlikely). The resulting directories, if they exist, are
appended to sys.path, and also inspected for path configuration files.
FOR DEBIAN, this sys.path is augmented with directories in /usr/local.
Local addons go into /usr/local/lib/python<version>/site-packages
(resp. /usr/local/lib/site-python), Debian addons install into
/usr/{lib,share}/python<version>/dist-packages.
A path configuration file is a file whose name has the form
<package>.pth; its contents are additional directories (one per line)
to be added to sys.path. Non-existing directories (or
non-directories) are never added to sys.path; no directory is added to
sys.path more than once. Blank lines and lines beginning with
'#' are skipped. Lines starting with 'import' are executed.
For example, suppose sys.prefix and sys.exec_prefix are set to
/usr/local and there is a directory /usr/local/lib/python2.X/site-packages
with three subdirectories, foo, bar and spam, and two path
configuration files, foo.pth and bar.pth. Assume foo.pth contains the
following:
# foo package configuration
foo
bar
bletch
and bar.pth contains:
# bar package configuration
bar
Then the following directories are added to sys.path, in this order:
/usr/local/lib/python2.X/site-packages/bar
/usr/local/lib/python2.X/site-packages/foo
Note that bletch is omitted because it doesn't exist; bar precedes foo
because bar.pth comes alphabetically before foo.pth; and spam is
omitted because it is not mentioned in either path configuration file.
After these path manipulations, an attempt is made to import a module
named sitecustomize, which can perform arbitrary additional
site-specific customizations. If this import fails with an
ImportError exception, it is silently ignored.
"""
import sys
import os
try:
import __builtin__ as builtins
except ImportError:
import builtins
try:
set
except NameError:
from sets import Set as set
# Prefixes for site-packages; add additional prefixes like /usr/local here
PREFIXES = [sys.prefix, sys.exec_prefix]
# Enable per user site-packages directory
# set it to False to disable the feature or True to force the feature
ENABLE_USER_SITE = None
# for distutils.commands.install
USER_SITE = None
USER_BASE = None
_is_pypy = hasattr(sys, 'pypy_version_info')
_is_jython = sys.platform[:4] == 'java'
if _is_jython:
ModuleType = type(os)
def makepath(*paths):
dir = os.path.join(*paths)
if _is_jython and (dir == '__classpath__' or
dir.startswith('__pyclasspath__')):
return dir, dir
dir = os.path.abspath(dir)
return dir, os.path.normcase(dir)
def abs__file__():
"""Set all module' __file__ attribute to an absolute path"""
for m in sys.modules.values():
if ((_is_jython and not isinstance(m, ModuleType)) or
hasattr(m, '__loader__')):
# only modules need the abspath in Jython. and don't mess
# with a PEP 302-supplied __file__
continue
f = getattr(m, '__file__', None)
if f is None:
continue
m.__file__ = os.path.abspath(f)
def removeduppaths():
""" Remove duplicate entries from sys.path along with making them
absolute"""
# This ensures that the initial path provided by the interpreter contains
# only absolute pathnames, even if we're running from the build directory.
L = []
known_paths = set()
for dir in sys.path:
# Filter out duplicate paths (on case-insensitive file systems also
# if they only differ in case); turn relative paths into absolute
# paths.
dir, dircase = makepath(dir)
if not dircase in known_paths:
L.append(dir)
known_paths.add(dircase)
sys.path[:] = L
return known_paths
# XXX This should not be part of site.py, since it is needed even when
# using the -S option for Python. See http://www.python.org/sf/586680
def addbuilddir():
"""Append ./build/lib.<platform> in case we're running in the build dir
(especially for Guido :-)"""
from distutils.util import get_platform
s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
if hasattr(sys, 'gettotalrefcount'):
s += '-pydebug'
s = os.path.join(os.path.dirname(sys.path[-1]), s)
sys.path.append(s)
def _init_pathinfo():
"""Return a set containing all existing directory entries from sys.path"""
d = set()
for dir in sys.path:
try:
if os.path.isdir(dir):
dir, dircase = makepath(dir)
d.add(dircase)
except TypeError:
continue
return d
def addpackage(sitedir, name, known_paths):
"""Add a new path to known_paths by combining sitedir and 'name' or execute
sitedir if it starts with 'import'"""
if known_paths is None:
_init_pathinfo()
reset = 1
else:
reset = 0
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
return
try:
for line in f:
if line.startswith("#"):
continue
if line.startswith("import"):
exec(line)
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
finally:
f.close()
if reset:
known_paths = None
return known_paths
def addsitedir(sitedir, known_paths=None):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
if known_paths is None:
known_paths = _init_pathinfo()
reset = 1
else:
reset = 0
sitedir, sitedircase = makepath(sitedir)
if not sitedircase in known_paths:
sys.path.append(sitedir) # Add path component
try:
names = os.listdir(sitedir)
except os.error:
return
names.sort()
for name in names:
if name.endswith(os.extsep + "pth"):
addpackage(sitedir, name, known_paths)
if reset:
known_paths = None
return known_paths
def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix):
"""Add site-packages (and possibly site-python) to sys.path"""
prefixes = [os.path.join(sys_prefix, "local"), sys_prefix]
if exec_prefix != sys_prefix:
prefixes.append(os.path.join(exec_prefix, "local"))
for prefix in prefixes:
if prefix:
if sys.platform in ('os2emx', 'riscos') or _is_jython:
sitedirs = [os.path.join(prefix, "Lib", "site-packages")]
elif _is_pypy:
sitedirs = [os.path.join(prefix, 'site-packages')]
elif sys.platform == 'darwin' and prefix == sys_prefix:
if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python
sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"),
os.path.join(prefix, "Extras", "lib", "python")]
else: # any other Python distros on OSX work this way
sitedirs = [os.path.join(prefix, "lib",
"python" + sys.version[:3], "site-packages")]
elif os.sep == '/':
sitedirs = [os.path.join(prefix,
"lib",
"python" + sys.version[:3],
"site-packages"),
os.path.join(prefix, "lib", "site-python"),
os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")]
lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages")
if (os.path.exists(lib64_dir) and
os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]):
sitedirs.append(lib64_dir)
try:
# sys.getobjects only available in --with-pydebug build
sys.getobjects
sitedirs.insert(0, os.path.join(sitedirs[0], 'debug'))
except AttributeError:
pass
# Debian-specific dist-packages directories:
sitedirs.append(os.path.join(prefix, "lib",
"python" + sys.version[:3],
"dist-packages"))
sitedirs.append(os.path.join(prefix, "local/lib",
"python" + sys.version[:3],
"dist-packages"))
sitedirs.append(os.path.join(prefix, "lib", "dist-python"))
else:
sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")]
if sys.platform == 'darwin':
# for framework builds *only* we add the standard Apple
# locations. Currently only per-user, but /Library and
# /Network/Library could be added too
if 'Python.framework' in prefix:
home = os.environ.get('HOME')
if home:
sitedirs.append(
os.path.join(home,
'Library',
'Python',
sys.version[:3],
'site-packages'))
for sitedir in sitedirs:
if os.path.isdir(sitedir):
addsitedir(sitedir, known_paths)
return None
def check_enableusersite():
"""Check if user site directory is safe for inclusion
The function tests for the command line flag (including environment var),
process uid/gid equal to effective uid/gid.
None: Disabled for security reasons
False: Disabled by user (command line option)
True: Safe and enabled
"""
if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False):
return False
if hasattr(os, "getuid") and hasattr(os, "geteuid"):
# check process uid == effective uid
if os.geteuid() != os.getuid():
return None
if hasattr(os, "getgid") and hasattr(os, "getegid"):
# check process gid == effective gid
if os.getegid() != os.getgid():
return None
return True
def addusersitepackages(known_paths):
"""Add a per user site-package to sys.path
Each user has its own python directory with site-packages in the
home directory.
USER_BASE is the root directory for all Python versions
USER_SITE is the user specific site-packages directory
USER_SITE/.. can be used for data.
"""
global USER_BASE, USER_SITE, ENABLE_USER_SITE
env_base = os.environ.get("PYTHONUSERBASE", None)
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
#if sys.platform in ('os2emx', 'riscos'):
# # Don't know what to put here
# USER_BASE = ''
# USER_SITE = ''
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
if env_base:
USER_BASE = env_base
else:
USER_BASE = joinuser(base, "Python")
USER_SITE = os.path.join(USER_BASE,
"Python" + sys.version[0] + sys.version[2],
"site-packages")
else:
if env_base:
USER_BASE = env_base
else:
USER_BASE = joinuser("~", ".local")
USER_SITE = os.path.join(USER_BASE, "lib",
"python" + sys.version[:3],
"site-packages")
if ENABLE_USER_SITE and os.path.isdir(USER_SITE):
addsitedir(USER_SITE, known_paths)
if ENABLE_USER_SITE:
for dist_libdir in ("lib", "local/lib"):
user_site = os.path.join(USER_BASE, dist_libdir,
"python" + sys.version[:3],
"dist-packages")
if os.path.isdir(user_site):
addsitedir(user_site, known_paths)
return known_paths
def setBEGINLIBPATH():
"""The OS/2 EMX port has optional extension modules that do double duty
as DLLs (and must use the .DLL file extension) for other extensions.
The library search path needs to be amended so these will be found
during module import. Use BEGINLIBPATH so that these are at the start
of the library search path.
"""
dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload")
libpath = os.environ['BEGINLIBPATH'].split(';')
if libpath[-1]:
libpath.append(dllpath)
else:
libpath[-1] = dllpath
os.environ['BEGINLIBPATH'] = ';'.join(libpath)
def setquit():
"""Define new built-ins 'quit' and 'exit'.
These are simply strings that display a hint on how to exit.
"""
if os.sep == ':':
eof = 'Cmd-Q'
elif os.sep == '\\':
eof = 'Ctrl-Z plus Return'
else:
eof = 'Ctrl-D (i.e. EOF)'
class Quitter(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return 'Use %s() or %s to exit' % (self.name, eof)
def __call__(self, code=None):
# Shells like IDLE catch the SystemExit, but listen when their
# stdin wrapper is closed.
try:
sys.stdin.close()
except:
pass
raise SystemExit(code)
builtins.quit = Quitter('quit')
builtins.exit = Quitter('exit')
class _Printer(object):
"""interactive prompt objects for printing the license text, a list of
contributors and the copyright notice."""
MAXLINES = 23
def __init__(self, name, data, files=(), dirs=()):
self.__name = name
self.__data = data
self.__files = files
self.__dirs = dirs
self.__lines = None
def __setup(self):
if self.__lines:
return
data = None
for dir in self.__dirs:
for filename in self.__files:
filename = os.path.join(dir, filename)
try:
fp = file(filename, "rU")
data = fp.read()
fp.close()
break
except IOError:
pass
if data:
break
if not data:
data = self.__data
self.__lines = data.split('\n')
self.__linecnt = len(self.__lines)
def __repr__(self):
self.__setup()
if len(self.__lines) <= self.MAXLINES:
return "\n".join(self.__lines)
else:
return "Type %s() to see the full %s text" % ((self.__name,)*2)
def __call__(self):
self.__setup()
prompt = 'Hit Return for more, or q (and Return) to quit: '
lineno = 0
while 1:
try:
for i in range(lineno, lineno + self.MAXLINES):
print(self.__lines[i])
except IndexError:
break
else:
lineno += self.MAXLINES
key = None
while key is None:
try:
key = raw_input(prompt)
except NameError:
key = input(prompt)
if key not in ('', 'q'):
key = None
if key == 'q':
break
def setcopyright():
"""Set 'copyright' and 'credits' in __builtin__"""
builtins.copyright = _Printer("copyright", sys.copyright)
if _is_jython:
builtins.credits = _Printer(
"credits",
"Jython is maintained by the Jython developers (www.jython.org).")
elif _is_pypy:
builtins.credits = _Printer(
"credits",
"PyPy is maintained by the PyPy developers: http://codespeak.net/pypy")
else:
builtins.credits = _Printer("credits", """\
Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
for supporting Python development. See www.python.org for more information.""")
here = os.path.dirname(os.__file__)
builtins.license = _Printer(
"license", "See http://www.python.org/%.3s/license.html" % sys.version,
["LICENSE.txt", "LICENSE"],
[os.path.join(here, os.pardir), here, os.curdir])
class _Helper(object):
"""Define the built-in 'help'.
This is a wrapper around pydoc.help (with a twist).
"""
def __repr__(self):
return "Type help() for interactive help, " \
"or help(object) for help about object."
def __call__(self, *args, **kwds):
import pydoc
return pydoc.help(*args, **kwds)
def sethelper():
builtins.help = _Helper()
def aliasmbcs():
"""On Windows, some default encodings are not provided by Python,
while they are always available as "mbcs" in each locale. Make
them usable by aliasing to "mbcs" in such a case."""
if sys.platform == 'win32':
import locale, codecs
enc = locale.getdefaultlocale()[1]
if enc.startswith('cp'): # "cp***" ?
try:
codecs.lookup(enc)
except LookupError:
import encodings
encodings._cache[enc] = encodings._unknown
encodings.aliases.aliases[enc] = 'mbcs'
def setencoding():
"""Set the string encoding used by the Unicode implementation. The
default is 'ascii', but if you're willing to experiment, you can
change this."""
encoding = "ascii" # Default value set by _PyUnicode_Init()
if 0:
# Enable to support locale aware default string encodings.
import locale
loc = locale.getdefaultlocale()
if loc[1]:
encoding = loc[1]
if 0:
# Enable to switch off string to Unicode coercion and implicit
# Unicode to string conversion.
encoding = "undefined"
if encoding != "ascii":
# On Non-Unicode builds this will raise an AttributeError...
sys.setdefaultencoding(encoding) # Needs Python Unicode build !
def execsitecustomize():
"""Run custom site specific code, if available."""
try:
import sitecustomize
except ImportError:
pass
def virtual_install_main_packages():
f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt'))
sys.real_prefix = f.read().strip()
f.close()
pos = 2
hardcoded_relative_dirs = []
if sys.path[0] == '':
pos += 1
if sys.platform == 'win32':
paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')]
elif _is_jython:
paths = [os.path.join(sys.real_prefix, 'Lib')]
elif _is_pypy:
if sys.pypy_version_info >= (1, 5):
cpyver = '%d.%d' % sys.version_info[:2]
else:
cpyver = '%d.%d.%d' % sys.version_info[:3]
paths = [os.path.join(sys.real_prefix, 'lib_pypy'),
os.path.join(sys.real_prefix, 'lib-python', 'modified-%s' % cpyver),
os.path.join(sys.real_prefix, 'lib-python', cpyver)]
hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
#
# This is hardcoded in the Python executable, but relative to sys.prefix:
for path in paths[:]:
plat_path = os.path.join(path, 'plat-%s' % sys.platform)
if os.path.exists(plat_path):
paths.append(plat_path)
else:
paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])]
hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3])
if os.path.exists(lib64_path):
paths.append(lib64_path)
# This is hardcoded in the Python executable, but relative to sys.prefix:
plat_path = os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3],
'plat-%s' % sys.platform)
if os.path.exists(plat_path):
paths.append(plat_path)
# This is hardcoded in the Python executable, but
# relative to sys.prefix, so we have to fix up:
for path in list(paths):
tk_dir = os.path.join(path, 'lib-tk')
if os.path.exists(tk_dir):
paths.append(tk_dir)
# These are hardcoded in the Apple's Python executable,
# but relative to sys.prefix, so we have to fix them up:
if sys.platform == 'darwin':
hardcoded_paths = [os.path.join(relative_dir, module)
for relative_dir in hardcoded_relative_dirs
for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')]
for path in hardcoded_paths:
if os.path.exists(path):
paths.append(path)
sys.path.extend(paths)
def force_global_eggs_after_local_site_packages():
"""
Force easy_installed eggs in the global environment to get placed
in sys.path after all packages inside the virtualenv. This
maintains the "least surprise" result that packages in the
virtualenv always mask global packages, never the other way
around.
"""
egginsert = getattr(sys, '__egginsert', 0)
for i, path in enumerate(sys.path):
if i > egginsert and path.startswith(sys.prefix):
egginsert = i
sys.__egginsert = egginsert + 1
def virtual_addsitepackages(known_paths):
force_global_eggs_after_local_site_packages()
return addsitepackages(known_paths, sys_prefix=sys.real_prefix)
def fixclasspath():
"""Adjust the special classpath sys.path entries for Jython. These
entries should follow the base virtualenv lib directories.
"""
paths = []
classpaths = []
for path in sys.path:
if path == '__classpath__' or path.startswith('__pyclasspath__'):
classpaths.append(path)
else:
paths.append(path)
sys.path = paths
sys.path.extend(classpaths)
def execusercustomize():
"""Run custom user specific code, if available."""
try:
import usercustomize
except ImportError:
pass
def main():
global ENABLE_USER_SITE
virtual_install_main_packages()
abs__file__()
paths_in_sys = removeduppaths()
if (os.name == "posix" and sys.path and
os.path.basename(sys.path[-1]) == "Modules"):
addbuilddir()
if _is_jython:
fixclasspath()
GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt'))
if not GLOBAL_SITE_PACKAGES:
ENABLE_USER_SITE = False
if ENABLE_USER_SITE is None:
ENABLE_USER_SITE = check_enableusersite()
paths_in_sys = addsitepackages(paths_in_sys)
paths_in_sys = addusersitepackages(paths_in_sys)
if GLOBAL_SITE_PACKAGES:
paths_in_sys = virtual_addsitepackages(paths_in_sys)
if sys.platform == 'os2emx':
setBEGINLIBPATH()
setquit()
setcopyright()
sethelper()
aliasmbcs()
setencoding()
execsitecustomize()
if ENABLE_USER_SITE:
execusercustomize()
# Remove sys.setdefaultencoding() so that users cannot change the
# encoding after initialization. The test for presence is needed when
# this module is run as a script, because this code is executed twice.
if hasattr(sys, "setdefaultencoding"):
del sys.setdefaultencoding
main()
def _script():
help = """\
%s [--user-base] [--user-site]
Without arguments print some useful information
With arguments print the value of USER_BASE and/or USER_SITE separated
by '%s'.
Exit codes with --user-base or --user-site:
0 - user site directory is enabled
1 - user site directory is disabled by user
2 - uses site directory is disabled by super user
or for security reasons
>2 - unknown error
"""
args = sys.argv[1:]
if not args:
print("sys.path = [")
for dir in sys.path:
print(" %r," % (dir,))
print("]")
def exists(path):
if os.path.isdir(path):
return "exists"
else:
return "doesn't exist"
print("USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)))
print("USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)))
print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE)
sys.exit(0)
buffer = []
if '--user-base' in args:
buffer.append(USER_BASE)
if '--user-site' in args:
buffer.append(USER_SITE)
if buffer:
print(os.pathsep.join(buffer))
if ENABLE_USER_SITE:
sys.exit(0)
elif ENABLE_USER_SITE is False:
sys.exit(1)
elif ENABLE_USER_SITE is None:
sys.exit(2)
else:
sys.exit(3)
else:
import textwrap
print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
sys.exit(10)
if __name__ == '__main__':
_script()
.pyenv/lib/python2.6/sre.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre.py
.pyenv/lib/python2.6/sre_compile.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_compile.py
.pyenv/lib/python2.6/sre_constants.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_constants.py
.pyenv/lib/python2.6/sre_parse.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_parse.py
.pyenv/lib/python2.6/stat.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/stat.py
.pyenv/lib/python2.6/types.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/types.py
.pyenv/lib/python2.6/warnings.py
-1
View File
@@ -1 +0,0 @@
/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/warnings.py
README.md
+7 -2
View File
@@ -1,6 +1,11 @@
Zipline
=======
Zipline is a distributed realtime stream processing system.
System Setup
==============
You need to have zeromq installed - http://www.zeromq.org/intro:get-the-software.
Running
@@ -55,7 +60,7 @@ QBT relies heavily on scientific python components (numpy, scikit, pandas, matpl
- mac: homebrew/macport/fink (I highly recommend homebrew: https://github.com/mxcl/homebrew)
- windows: probably best if you use a complete distribution, like: enthought, ActiveState, or Python(x,y)
- Python also provides good package management tools to help you manage the components you install for Python.
- pip
- pip
- easy_install/setuptools. I have always used setuptools, and I've been quite happy with it. Just remember that setuptools is coupled to your python version.
- virtualenv and virtualenvwrapper are your very best friends. They complement your python package manager by allowing you to create and quickly switch between named configurations.
- *Install all the versions of Python you like to use, but install setuptools, virtualenv, and virtualenvwrapper with the very latest python.* Use the latest python to install the latest setuptools, and the latest setuptools to install virtualenv and virtualenvwrapper. virtualenvwrapper allows you to specify the python version you wish to use (mkvirtualenv -p <python executable> <env name>), so you can create envs of any python denomination.
docs/conf.py
+1 -1
View File
@@ -40,7 +40,7 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'QSim'
project = u'Zipline'
copyright = u'2012, Quantopian: jean, fawce, sdiehl'
# The version info for the project you're documenting, acts as replacement for
docs/index.rst
+7 -7
View File
@@ -1,4 +1,4 @@
.. QSim documentation master file, created by
.. Zipline documentation master file, created by
sphinx-quickstart on Wed Feb 8 15:29:56 2012.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
@@ -12,15 +12,15 @@ Contents:
modules.rst
messaging.rst
Quantopian Simulator: QSim
================================
Zipline
=======
Qsim runs backtests using asynchronous components and zeromq messaging for communication and coordination.
Zipline runs backtests using asynchronous components and zeromq messaging for communication and coordination.
Simulator is the heart of QSim, and the primary access point for creating, launching, and tracking simulations. You can find it in :py:class:`~zipline.core.Simulator`
Simulator is the heart of Zipline, and the primary access point for creating, launching, and tracking simulations. You can find it in :py:class:`~zipline.core.Simulator`
Simulator Sub-Components
==========================
========================
Each simulation contains numerous subcomponents, each operating asynchronously from all others, and communicating
via zeromq.
@@ -61,4 +61,4 @@ Indices and tables
* :ref:`modindex`
* :ref:`search`
.. _coverage: cover/index.html
.. _coverage: cover/index.html
docs/qsim.rst → docs/zipline.rst
+40 -12
View File
@@ -1,30 +1,26 @@
============
zipline Package
============
QSim API
===========================
===============
:mod:`zipline` Package
-------------------
----------------------
.. automodule:: zipline.__init__
:members:
:undoc-members:
:show-inheritance:
:mod:`config` Module
--------------------
:mod:`cli` Module
-----------------
.. automodule:: zipline.config
.. automodule:: zipline.cli
:members:
:undoc-members:
:show-inheritance:
:mod:`core` Module
------------------
:mod:`component` Module
-----------------------
.. automodule:: zipline.core
.. automodule:: zipline.component
:members:
:undoc-members:
:show-inheritance:
@@ -37,6 +33,22 @@ QSim API
:undoc-members:
:show-inheritance:
:mod:`monitor` Module
---------------------
.. automodule:: zipline.monitor
:members:
:undoc-members:
:show-inheritance:
:mod:`protocol` Module
----------------------
.. automodule:: zipline.protocol
:members:
:undoc-members:
:show-inheritance:
:mod:`sources` Module
---------------------
@@ -45,6 +57,14 @@ QSim API
:undoc-members:
:show-inheritance:
:mod:`topology` Module
----------------------
.. automodule:: zipline.topology
:members:
:undoc-members:
:show-inheritance:
:mod:`util` Module
------------------
@@ -53,6 +73,14 @@ QSim API
:undoc-members:
:show-inheritance:
:mod:`webui` Module
-------------------
.. automodule:: zipline.webui
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
docs/qsim.test.rst → docs/zipline.test.rst
+8
View File
@@ -17,3 +17,11 @@ test Package
:undoc-members:
:show-inheritance:
:mod:`test_sanity` Module
-------------------------
.. automodule:: zipline.test.test_sanity
:members:
:undoc-members:
:show-inheritance:
docs/qsim.transforms.rst → docs/zipline.transforms.rst
+3 -11
View File
@@ -1,18 +1,10 @@
transforms Package
==================
:mod:`core` Module
------------------
:mod:`transforms` Package
-------------------------
.. automodule:: zipline.transforms.core
:members:
:undoc-members:
:show-inheritance:
:mod:`merge` Module
-------------------
.. automodule:: zipline.transforms.merge
.. automodule:: zipline.transforms
:members:
:undoc-members:
:show-inheritance:
pavement.py
+1
View File
@@ -185,4 +185,5 @@ def apidocs():
Recursively autogenerate the Sphinx autodoc for the module and
its submodules.
"""
call('rm docs/zipline*.rst', shell=True)
call('sphinx-apidoc -o docs/ zipline', shell=True)
zipline/component.py
+210
View File
@@ -0,0 +1,210 @@
"""
Commonly used messaging components.
"""
import json
import uuid
import datetime
import zipline.util as qutil
class Component(object):
def __init__(self):
"""
:addresses: a dict of name_string -> zmq port address strings. Must have the following entries::
- sync_address: socket address used for synchronizing the start of all workers, heartbeating, and exit notification
will be used in REP/REQ sockets. Bind is always on the REP side.
- control_address: socket address used for controlling and
monitoring the status of the simulation
- data_address: socket address used for data sources to stream their records.
will be used in PUSH/PULL sockets between data sources and a ParallelBuffer (aka the Feed). Bind
will always be on the PULL side (we always have N producers and 1 consumer)
- feed_address: socket address used to publish consolidated feed from serialization of data sources
will be used in PUB/SUB sockets between Feed and Transforms. Bind is always on the PUB side.
- merge_address: socket address used to publish transformed values.
will be used in PUSH/PULL from many transforms to one MergedParallelBuffer (aka the Merge). Bind
will always be on the PULL side (we always have N producers and 1 consumer)
- result_address: socket address used to publish merged data source feed and transforms to clients
will be used in PUB/SUB from one Merge to one or many clients. Bind is always on the PUB side.
Bind/Connect methods will return the correct socket type for each address. Any sockets on which recv is expected to be called
will also return a Poller.
"""
self.zmq = None
self.context = None
self.addresses = None
self.out_socket = None
self.gevent_needed = False
self.killed = False
# TODO: could probably mkae this into a property instead of a
# method
def get_id(self):
raise NotImplementedError
def open(self):
raise NotImplementedError
def destroy(self):
"""
Tear down after normal operation.
"""
raise NotImplementedError
def kill(self):
"""
Tear down ( fast ) as a mode of failure in the
simulation.
"""
raise NotImplementedError
def do_work(self):
raise NotImplementedError
def run(self):
fail = None
#try:
#TODO: can't initialize these values in the __init__?
self.done = False
self.sockets = []
if self.gevent_needed:
qutil.LOGGER.info("Loading gevent specific zmq for {id}".format(id=self.get_id()))
import gevent_zeromq
self.zmq = gevent_zeromq.zmq
else:
import zmq
self.zmq = zmq
self.context = self.zmq.Context()
self.open()
self.setup_sync()
self.setup_control()
self.loop()
#close all the sockets
for sock in self.sockets:
sock.close()
#except Exception as e:
#qutil.LOGGER.exception("Unexpected error in run for {id}.".format(id=self.get_id()))
#fail = e
#finally:
#if(self.context != None):
#self.context.destroy()
#if fail:
#raise fail
def loop(self):
while not self.done:
self.confirm()
self.do_work()
def signal_done(self):
#notify down stream components that we're done
if(self.out_socket != None):
self.out_socket.send("DONE")
#notify host we're done
self.sync_socket.send(self.get_id() + ":DONE")
self.receive_sync_ack()
#notify internal work look that we're done
self.done = True
# TODO: probably don't need a method here ... or move into
# higher level framing protocol
def is_done_message(self, message):
return message == "DONE"
def confirm(self):
# send a synchronization request to the host
self.sync_socket.send(self.get_id() + ":RUN")
self.receive_sync_ack()
def receive_sync_ack(self):
# wait for synchronization reply from the host
socks = dict(self.sync_poller.poll(2000)) #timeout after 2 seconds.
if self.sync_socket in socks and socks[self.sync_socket] == self.zmq.POLLIN:
message = self.sync_socket.recv()
else:
raise Exception("Sync ack timed out on response for {id}".format(id=self.get_id()))
def bind_data(self):
return self.bind_pull_socket(self.addresses['data_address'])
def connect_data(self):
return self.connect_push_socket(self.addresses['data_address'])
def bind_feed(self):
return self.bind_pub_socket(self.addresses['feed_address'])
def connect_feed(self):
return self.connect_sub_socket(self.addresses['feed_address'])
def bind_merge(self):
return self.bind_pull_socket(self.addresses['merge_address'])
def connect_merge(self):
return self.connect_push_socket(self.addresses['merge_address'])
def bind_result(self):
return self.bind_pub_socket(self.addresses['result_address'])
def connect_result(self):
return self.connect_sub_socket(self.addresses['result_address'])
def bind_pull_socket(self, addr):
pull_socket = self.context.socket(self.zmq.PULL)
pull_socket.bind(addr)
poller = self.zmq.Poller()
poller.register(pull_socket, self.zmq.POLLIN)
self.sockets.append(pull_socket)
return pull_socket, poller
def connect_push_socket(self, addr):
push_socket = self.context.socket(self.zmq.PUSH)
push_socket.connect(addr)
#push_socket.setsockopt(self.zmq.LINGER,0)
self.sockets.append(push_socket)
self.out_socket = push_socket
return push_socket
def bind_pub_socket(self, addr):
pub_socket = self.context.socket(self.zmq.PUB)
pub_socket.bind(addr)
#pub_socket.setsockopt(self.zmq.LINGER,0)
self.out_socket = pub_socket
return pub_socket
def connect_sub_socket(self, addr):
sub_socket = self.context.socket(self.zmq.SUB)
sub_socket.connect(addr)
sub_socket.setsockopt(self.zmq.SUBSCRIBE,'')
poller = self.zmq.Poller()
poller.register(sub_socket, self.zmq.POLLIN)
self.sockets.append(sub_socket)
return sub_socket, poller
def setup_control(self):
"""
Set up the control socket. Used to monitor the the
overall status of the simulation and to forcefully tear
down the simulation in case of a failure.
"""
pass
def setup_sync(self):
qutil.LOGGER.debug("Connecting sync client for {id}".format(id=self.get_id()))
self.sync_socket = self.context.socket(self.zmq.REQ)
self.sync_socket.connect(self.addresses['sync_address'])
#self.sync_socket.setsockopt(self.zmq.LINGER,0)
self.sync_poller = self.zmq.Poller()
self.sync_poller.register(self.sync_socket, self.zmq.POLLIN)
self.sockets.append(self.sync_socket)
zipline/messaging.py
+154 -296
View File
@@ -5,224 +5,82 @@ import json
import uuid
import datetime
import zipline.util as qutil
from zipline.component import Component
class Component(object):
def __init__(self):
"""
:addresses: a dict of name_string -> zmq port address strings. Must have the following entries::
- sync_address: socket address used for synchronizing the start of all workers, heartbeating, and exit notification
will be used in REP/REQ sockets. Bind is always on the REP side.
- data_address: socket address used for data sources to stream their records.
will be used in PUSH/PULL sockets between data sources and a ParallelBuffer (aka the Feed). Bind
will always be on the PULL side (we always have N producers and 1 consumer)
- feed_address: socket address used to publish consolidated feed from serialization of data sources
will be used in PUB/SUB sockets between Feed and Transforms. Bind is always on the PUB side.
- merge_address: socket address used to publish transformed values.
will be used in PUSH/PULL from many transforms to one MergedParallelBuffer (aka the Merge). Bind
will always be on the PULL side (we always have N producers and 1 consumer)
- result_address: socket address used to publish merged data source feed and transforms to clients
will be used in PUB/SUB from one Merge to one or many clients. Bind is always on the PUB side.
Bind/Connect methods will return the correct socket type for each address. Any sockets on which recv is expected to be called
will also return a Poller.
"""
self.zmq = None
self.context = None
self.addresses = None
self.out_socket = None
self.gevent_needed = False
def get_id(self):
NotImplemented
def open(self):
NotImplemented
def do_work(self):
NotImplemented
def run(self):
try:
#TODO: can't initialize these values in the __init__?
self.done = False
self.sockets = []
if self.gevent_needed:
qutil.LOGGER.info("Loading gevent specific zmq for {id}".format(id=self.get_id()))
import gevent_zeromq
self.zmq = gevent_zeromq.zmq
else:
import zmq
self.zmq = zmq
self.context = self.zmq.Context()
self.open()
self.setup_sync()
self.loop()
#close all the sockets
for sock in self.sockets:
sock.close()
except Exception as e:
qutil.LOGGER.exception("Unexpected error in run for {id}.".format(id=self.get_id()))
raise e
finally:
if(self.context != None):
self.context.destroy()
def loop(self):
while not self.done:
self.confirm()
self.do_work()
def signal_done(self):
#notify down stream components that we're done
if(self.out_socket != None):
self.out_socket.send("DONE")
#notify host we're done
self.sync_socket.send(self.get_id() + ":DONE")
self.receive_sync_ack()
#notify internal work look that we're done
self.done = True
def is_done_message(self, message):
return message == "DONE"
def confirm(self):
# send a synchronization request to the host
self.sync_socket.send(self.get_id() + ":RUN")
self.receive_sync_ack()
def receive_sync_ack(self):
# wait for synchronization reply from the host
socks = dict(self.sync_poller.poll(2000)) #timeout after 2 seconds.
if self.sync_socket in socks and socks[self.sync_socket] == self.zmq.POLLIN:
message = self.sync_socket.recv()
else:
raise Exception("Sync ack timed out on response for {id}".format(id=self.get_id()))
def bind_data(self):
return self.bind_pull_socket(self.addresses['data_address'])
def connect_data(self):
return self.connect_push_socket(self.addresses['data_address'])
def bind_order(self):
return self.bind_pull_socket(self.addresses['order_address'])
def connect_order(self):
return self.connect_push_socket(self.addresses['order_address'])
def bind_feed(self):
return self.bind_pub_socket(self.addresses['feed_address'])
def connect_feed(self):
return self.connect_sub_socket(self.addresses['feed_address'])
def bind_merge(self):
return self.bind_pull_socket(self.addresses['merge_address'])
def connect_merge(self):
return self.connect_push_socket(self.addresses['merge_address'])
def bind_result(self):
return self.bind_pub_socket(self.addresses['result_address'])
def connect_result(self):
return self.connect_sub_socket(self.addresses['result_address'])
def bind_pull_socket(self, addr):
pull_socket = self.context.socket(self.zmq.PULL)
pull_socket.bind(addr)
poller = self.zmq.Poller()
poller.register(pull_socket, self.zmq.POLLIN)
self.sockets.append(pull_socket)
return pull_socket, poller
def connect_push_socket(self, addr):
push_socket = self.context.socket(self.zmq.PUSH)
push_socket.connect(addr)
#push_socket.setsockopt(self.zmq.LINGER,0)
self.sockets.append(push_socket)
self.out_socket = push_socket
return push_socket
def bind_pub_socket(self, addr):
pub_socket = self.context.socket(self.zmq.PUB)
pub_socket.bind(addr)
#pub_socket.setsockopt(self.zmq.LINGER,0)
self.out_socket = pub_socket
return pub_socket
def connect_sub_socket(self, addr):
sub_socket = self.context.socket(self.zmq.SUB)
sub_socket.connect(addr)
sub_socket.setsockopt(self.zmq.SUBSCRIBE,'')
poller = self.zmq.Poller()
poller.register(sub_socket, self.zmq.POLLIN)
self.sockets.append(sub_socket)
return sub_socket, poller
def setup_sync(self):
qutil.LOGGER.debug("Connecting sync client for {id}".format(id=self.get_id()))
self.sync_socket = self.context.socket(self.zmq.REQ)
self.sync_socket.connect(self.addresses['sync_address'])
#self.sync_socket.setsockopt(self.zmq.LINGER,0)
self.sync_poller = self.zmq.Poller()
self.sync_poller.register(self.sync_socket, self.zmq.POLLIN)
self.sockets.append(self.sync_socket)
class ComponentHost(Component):
"""Component that can launch multiple sub-components, synchronize their start, and then wait for all
components to be finished."""
"""
Components that can launch multiple sub-components, synchronize their start, and then wait for all
components to be finished.
"""
def __init__(self, addresses, gevent_needed=False):
Component.__init__(self)
self.addresses = addresses
#workaround for defect in threaded use of strptime: http://bugs.python.org/issue11108
qutil.parse_date("2012/02/13-10:04:28.114")
self.components = {}
self.sync_register = {}
self.timeout = datetime.timedelta(seconds=5)
self.gevent_needed = gevent_needed
self.feed = ParallelBuffer()
self.merge = MergedParallelBuffer()
self.passthrough = PassthroughTransform()
self.gevent_needed = gevent_needed
self.controller = None
#register the feed and the merge
self.register_components([self.feed, self.merge, self.passthrough])
def register_components(self, component_list):
for component in component_list:
def register_controller(self, controller):
self.controller = controller
for component in self.components.itervalues():
component.controller = controller
def register_components(self, components):
for component in components:
component.gevent_needed = self.gevent_needed
component.addresses = self.addresses
if self.controller:
component.controller = self.controller
self.components[component.get_id()] = component
self.sync_register[component.get_id()] = datetime.datetime.utcnow()
if(isinstance(component, DataSource)):
self.feed.add_source(component.get_id())
if(isinstance(component, BaseTransform)):
self.merge.add_source(component.get_id())
def unregister_component(self, component_id):
del(self.components[component_id])
del(self.sync_register[component_id])
del self.components[component_id]
del self.sync_register[component_id]
def setup_sync(self):
"""Start the sync server."""
"""
Start the sync server.
"""
qutil.LOGGER.debug("Connecting sync server.")
self.sync_socket = self.context.socket(self.zmq.REP)
self.sync_socket.bind(self.addresses['sync_address'])
self.poller = self.zmq.Poller()
self.poller.register(self.sync_socket, self.zmq.POLLIN)
self.sockets.append(self.sync_socket)
def open(self):
for component in self.components.values():
self.launch_component(component)
self.launch_controller()
def is_timed_out(self):
cur_time = datetime.datetime.utcnow()
if(len(self.components) == 0):
if len(self.components) == 0:
qutil.LOGGER.info("Component register is empty.")
return True
for source, last_dt in self.sync_register.iteritems():
@@ -230,7 +88,7 @@ class ComponentHost(Component):
qutil.LOGGER.info("Time out for {source}. Current component registery: {reg}".format(source=source, reg=self.components))
return True
return False
def loop(self):
while not self.is_timed_out():
# wait for synchronization request
@@ -246,20 +104,26 @@ class ComponentHost(Component):
if(self.is_done_message(status)):
qutil.LOGGER.info("{id} is DONE".format(id=sync_id))
self.unregister_component(sync_id)
else:
else:
self.sync_register[sync_id] = datetime.datetime.utcnow()
#qutil.LOGGER.info("confirmed {id}".format(id=msg))
# send synchronization reply
self.sync_socket.send('ack', self.zmq.NOBLOCK)
def launch_controller(self, controller):
raise NotImplementedError
def launch_component(self, component):
NotImplemented
raise NotImplementedError
class ParallelBuffer(Component):
"""Connects to N PULL sockets, publishing all messages received to a PUB socket.
Published messages are guaranteed to be in chronological order based on message property dt.
Expects to be instantiated in one execution context (thread, process, etc) and run in another."""
"""
Connects to N PULL sockets, publishing all messages received to a PUB socket.
Published messages are guaranteed to be in chronological order based on message property dt.
Expects to be instantiated in one execution context (thread, process, etc) and run in another.
"""
def __init__(self):
Component.__init__(self)
self.sent_count = 0
@@ -268,27 +132,27 @@ class ParallelBuffer(Component):
#data source component ID -> List of messages
self.data_buffer = {}
self.ds_finished_counter = 0
def get_id(self):
return "FEED"
def add_source(self, source_id):
self.data_buffer[source_id] = []
def open(self):
self.pull_socket, self.poller = self.bind_data()
self.feed_socket = self.bind_feed()
def do_work(self):
def open(self):
self.pull_socket, self.poller = self.bind_data()
self.feed_socket = self.bind_feed()
def do_work(self):
# wait for synchronization reply from the host
socks = dict(self.poller.poll(2000)) #timeout after 2 seconds.
if self.pull_socket in socks and socks[self.pull_socket] == self.zmq.POLLIN:
message = self.pull_socket.recv()
if(self.is_done_message(message)):
if self.is_done_message(message):
self.ds_finished_counter += 1
if(len(self.data_buffer) == self.ds_finished_counter):
if len(self.data_buffer) == self.ds_finished_counter:
#drain any remaining messages in the buffer
self.drain()
self.signal_done()
@@ -298,92 +162,110 @@ class ParallelBuffer(Component):
self.send_next()
def __len__(self):
"""buffer's length is same as internal map holding separate sorted arrays of events keyed by source id"""
return len(self.data_buffer)
"""
Buffer's length is same as internal map holding separate
sorted arrays of events keyed by source id.
"""
return len(self.data_buffer)
def append(self, source_id, value):
"""add an event to the buffer for the source specified by source_id"""
"""
Add an event to the buffer for the source specified by
source_id.
"""
self.data_buffer[source_id].append(value)
self.received_count += 1
def next(self):
"""Get the next message in chronological order"""
if(not(self.is_full() or self.draining)):
"""
Get the next message in chronological order.
"""
if not(self.is_full() or self.draining):
return
cur = None
earliest = None
for events in self.data_buffer.values():
if len(events) == 0:
continue
cur = events
if(earliest == None) or (cur[0]['dt'] <= earliest[0]['dt']):
if (earliest == None) or (cur[0]['dt'] <= earliest[0]['dt']):
earliest = cur
if(earliest != None):
if earliest != None:
return earliest.pop(0)
def is_full(self):
"""indicates whether the buffer has messages in buffer for all un-DONE sources"""
"""
Indicates whether the buffer has messages in buffer for
all un-DONE sources.
"""
for events in self.data_buffer.values():
if (len(events) == 0):
if len(events) == 0:
return False
return True
def pending_messages(self):
"""returns the count of all events from all sources in the buffer"""
"""
Returns the count of all events from all sources in the
buffer.
"""
total = 0
for events in self.data_buffer.values():
total += len(events)
return total
def drain(self):
"""send all messages in the buffer"""
"""
Send all messages in the buffer
"""
self.draining = True
while(self.pending_messages() > 0):
self.send_next()
def send_next(self):
"""send the (chronologically) next message in the buffer."""
"""
Send the (chronologically) next message in the buffer.
"""
if(not(self.is_full() or self.draining)):
return
event = self.next()
if(event != None):
self.feed_socket.send(json.dumps(event), self.zmq.NOBLOCK)
self.sent_count += 1
self.sent_count += 1
class MergedParallelBuffer(ParallelBuffer):
"""
Merges multiple streams of events into single messages.
"""
def __init__(self):
ParallelBuffer.__init__(self)
def open(self):
self.pull_socket, self.poller = self.bind_merge()
self.feed_socket = self.bind_result()
self.pull_socket, self.poller = self.bind_merge()
self.feed_socket = self.bind_result()
def next(self):
"""Get the next merged message from the feed buffer."""
if(not(self.is_full() or self.draining)):
return
#get the raw event from the passthrough transform.
result = self.data_buffer["PASSTHROUGH"].pop(0)['value']
for source, events in self.data_buffer.iteritems():
if(source == "PASSTHROUGH"):
if source == "PASSTHROUGH":
continue
if(len(events) > 0):
if len(events) > 0:
cur = events.pop(0)
result[source] = cur['value']
return result
def get_id(self):
return "MERGE"
class BaseTransform(Component):
"""Top level execution entry point for the transform::
@@ -393,40 +275,41 @@ class BaseTransform(Component):
- processes all messages received from feed, until DONE message received
- pushes all transforms
- sends DONE to result socket, closes all sockets and context
Parent class for feed transforms. Subclass and override transform
Parent class for feed transforms. Subclass and override transform
method to create a new derived value from the combined feed."""
def __init__(self, name):
Component.__init__(self)
self.state = {}
self.state['name'] = name
self.state = {}
self.state['name'] = name
def get_id(self):
return self.state['name']
def open(self):
def open(self):
"""
Establishes zmq connections.
"""
#create the feed.
"""
#create the feed.
self.feed_socket, self.poller = self.connect_feed()
#create the result PUSH
self.result_socket = self.connect_merge()
def do_work(self):
"""
Loops until feed's DONE message is received:
- receive an event from the data feed
- receive an event from the data feed
- call transform (subclass' method) on event
- send the transformed event
"""
socks = dict(self.poller.poll(2000)) #timeout after 2 seconds.
if self.feed_socket in socks and socks[self.feed_socket] == self.zmq.POLLIN:
message = self.feed_socket.recv()
if(self.is_done_message(message)):
if self.is_done_message(message):
self.signal_done()
return
event = json.loads(message)
cur_state = self.transform(event)
#TODO: do we want to relay the datetime again? maybe drop this?
@@ -435,82 +318,57 @@ class BaseTransform(Component):
self.result_socket.send(json.dumps(cur_state), self.zmq.NOBLOCK)
def transform(self, event):
""" Must return the transformed value as a map with {name:"name of new transform", value: "value of new field"}
Transforms run in parallel and results are merged into a single map, so transform names must be unique.
Best practice is to use the self.state object initialized from the transform configuration, and only set the
transformed value:
self.state['value'] = transformed_value
"""
NotImplemented
Must return the transformed value as a map with::
{name:"name of new transform", value: "value of new field"}
Transforms run in parallel and results are merged into a single map, so transform names must be unique.
Best practice is to use the self.state object initialized from the transform configuration, and only set the
transformed value::
self.state['value'] = transformed_value
"""
raise NotImplementedError
class PassthroughTransform(BaseTransform):
def __init__(self):
BaseTransform.__init__(self, "PASSTHROUGH")
def transform(self, event):
def transform(self, event):
return {'value':event}
class DataSource(Component):
"""
Baseclass for data sources. Subclass and implement send_all - usually this
Baseclass for data sources. Subclass and implement send_all - usually this
means looping through all records in a store, converting to a dict, and
calling send(map).
"""
def __init__(self, source_id):
Component.__init__(self)
self.id = source_id
self.cur_event = None
self.id = source_id
self.cur_event = None
def get_id(self):
return self.id
def open(self):
#create the data sink. Based on http://zguide.zeromq.org/py:tasksink2
def open(self):
#create the data sink. Based on http://zguide.zeromq.org/py:tasksink2
self.data_socket = self.connect_data()
def send(self, event):
"""
event is expected to be a dict
sets id and type properties in the dict
sends to the data_socket.
event is expected to be a dict
sets id and type properties in the dict
sends to the data_socket.
"""
event['id'] = self.id
event['id'] = self.id
event['type'] = self.get_type()
self.data_socket.send(json.dumps(event))
def get_type(self):
raise NotImplemented
class Client(Component):
def __init__(self):
Component.__init__(self)
self.received_count = 0
self.prev_dt = None
def open(self):
self.data_feed, self.poller = self.connect_result()
self.order_feed, self.poller = self.connect_order()
def do_work(self):
socks = dict(self.poller.poll(2000)) #timeout after 2 seconds.
if self.data_feed in socks and socks[self.data_feed] == self.zmq.POLLIN:
msg = self.data_feed.recv()
if(self.is_done_message(msg)):
qutil.LOGGER.info("Client is DONE!")
self.signal_done()
return
self.received_count += 1
event = json.loads(msg)
self.handle_event(event)
def handle_event(self, event):
NotImplemented
def order(self, sid, volume):
order = {'sid':sid, 'volume':volume}
zipline/monitor.py
+129
View File
@@ -0,0 +1,129 @@
import zmq
class Controller(object):
"""
A broker of sorts.
"""
polling = False
debug = False
def __init__(self, pull_socket, pub_socket, context=None, logging = None):
if not context:
self._ctx = zmq.Context()
else:
self._ctx = context
self.pull_socket = pull_socket
self.pub_socket = pub_socket
self.pull = self._ctx.socket(zmq.PULL)
self.pub = self._ctx.socket(zmq.PUB)
self.associated = [self.pull, self.pub]
if logging:
self.logging = logging
self.dologging = True
else:
self.logging = False
self.dologging = False
self.success = 0
self.failed = 0
try:
self.pull.bind(pull_socket)
except zmq.ZMQError:
raise Exception('Cannot not bind on %s' % pull_socket)
try:
self.pub.bind(pub_socket)
except zmq.ZMQError:
raise Exception('Cannot not bind on %s' % pub_socket)
def run(self, debug_step=False, stats=True):
self.polling = True
if self.debug or debug_step:
return self._poll_verbose(True, stats)
else:
return self._poll(False, stats)
def _poll(self, debug_step, stats):
while self.polling:
try:
self.logging.info('msg')
self.pub.send(self.pull.recv())
#self.pub.send(self.pull.recv(copy=False))
except KeyboardInterrupt:
self.polling = False
break
except Exception as e:
# Its common to wrap these in wildcard exceptions so
# that we don't loose messages, ever
self.logging.error(str(e))
self.failed += 1
continue
def _poll_verbose(self, debug_step, stats):
while self.polling:
try:
if debug_step:
msg = self.pull.recv(copy=False)
if self.dologging:
self.logging.info(msg)
self.pub.send(msg)
self.success += 1
except KeyboardInterrupt:
self.polling = False
break
except Exception as e:
# Its common to wrap these in wildcard exceptions so
# that we don't loose messages, ever
self.logging.error(str(e))
self.failed += 1
continue
def qos(self):
return float(self.success) / (self.success + self.failed)
def destroy(self):
"""
Manual cleanup.
"""
self.polling = False
for asoc in self.associated:
asoc.close()
#if self._ctx:
#self._ctx.destroy()
def __del__(self):
self.destroy()
def message_sender(self):
"""
Spin off a socket used for sending messages to this
controller.
"""
s = self._ctx.socket(zmq.PUSH)
s.connect(self.pull_socket)
s.setsockopt(zmq.LINGER, -1)
self.associated.append(s)
return s
def message_listener(self):
"""
Spin off a socket used for receiving messages from this
controller.
"""
s = self._ctx.socket(zmq.SUB)
s.connect(self.pub_socket)
s.setsockopt(zmq.SUBSCRIBE, '')
self.associated.append(s)
return s
zipline/protocol.py
+3
View File
@@ -0,0 +1,3 @@
#import msgpack
#import ujson
#import ultrajson_numpy
zipline/simulator.py
-49
View File
@@ -1,49 +0,0 @@
"""
Provides simulated data feed services...
"""
import multiprocessing
import json
import copy
import threading
import zipline.util as qutil
import zipline.messaging as qmsg
class SimulatorBase(qmsg.ComponentHost):
"""
Simulator coordinates the launch and communication of source, feed, transform, and merge components.
"""
def __init__(self, addresses, gevent_needed=False):
"""
"""
qmsg.ComponentHost.__init__(self, addresses, gevent_needed)
def simulate(self):
self.run()
def get_id(self):
return "Simulator"
class ThreadSimulator(SimulatorBase):
def __init__(self, addresses):
SimulatorBase.__init__(self, addresses)
def launch_component(self, component):
qutil.LOGGER.info("starting {name}".format(name=component.get_id()))
thread = threading.Thread(target=component.run)
thread.start()
return thread
class ProcessSimulator(SimulatorBase):
def __init__(self, addresses):
SimulatorBase.__init__(self, addresses)
def launch_component(self, component):
qutil.LOGGER.info("starting {name}".format(name=component.get_id()))
proc = multiprocessing.Process(target=component.run)
proc.start()
return proc
zipline/sources.py
+13 -14
View File
@@ -2,15 +2,14 @@
Provides data handlers that can push messages to a zipline.core.DataFeed
"""
import datetime
import json
import random
import zipline.util as qutil
import zipline.messaging as qmsg
import zipline.messaging as qmsg
class RandomEquityTrades(qmsg.DataSource):
"""Generates a random stream of trades for testing."""
def __init__(self, sid, source_id, count):
qmsg.DataSource.__init__(self, source_id)
self.count = count
@@ -19,21 +18,21 @@ class RandomEquityTrades(qmsg.DataSource):
self.trade_start = datetime.datetime.now()
self.minute = datetime.timedelta(minutes=1)
self.price = random.uniform(5.0, 50.0)
def get_type(self):
return 'equity_trade'
return 'equity_trade'
def do_work(self):
if(self.incr == self.count):
self.signal_done()
return
self.price = self.price + random.uniform(-0.05, 0.05)
event = {'sid':self.sid,
'dt':qutil.format_date(self.trade_start + (self.minute * self.incr)),
'price':self.price,
'volume':random.randrange(100,10000,100)}
event = {
'sid' : self.sid,
'dt' : qutil.format_date(self.trade_start + (self.minute * self.incr)),
'price' : self.price,
'volume' : random.randrange(100,10000,100)
}
self.send(event)
self.incr += 1
zipline/test/client.py
+6 -10
View File
@@ -3,20 +3,20 @@ import zipline.util as qutil
import zipline.messaging as qmsg
class TestClient(qmsg.Component):
def __init__(self, utest, expected_msg_count=0):
qmsg.Component.__init__(self)
self.received_count = 0
self.expected_msg_count = expected_msg_count
self.utest = utest
self.prev_dt = None
def get_id(self):
return "TEST_CLIENT"
return "TEST_CLIENT"
def open(self):
self.data_feed, self.poller = self.connect_result()
def do_work(self):
socks = dict(self.poller.poll(2000)) #timeout after 2 seconds.
if self.data_feed in socks and socks[self.data_feed] == self.zmq.POLLIN:
@@ -28,7 +28,7 @@ class TestClient(qmsg.Component):
"The client should have received ({n}) the same number of messages as the feed sent ({m})."
.format(n=self.received_count, m=self.expected_msg_count))
return
self.received_count += 1
event = json.loads(msg)
if(self.prev_dt != None):
@@ -38,7 +38,3 @@ class TestClient(qmsg.Component):
self.prev_dt = event['dt']
if(self.received_count % 100 == 0):
qutil.LOGGER.info("received {n} messages".format(n=self.received_count))
zipline/test/test_devsimulator.py
+87
View File
@@ -0,0 +1,87 @@
"""
Dummy simulator backported from Qexec for development on Zipline.
"""
import threading
import mock
from unittest2 import TestCase
from zipline.test.test_messaging import SimulatorTestCase
from zipline.monitor import Controller
from zipline.messaging import ComponentHost
import zipline.util as qutil
class DummyAllocator(object):
def __init__(self, ns):
self.idx = 0
self.sockets = [
'tcp://127.0.0.1:%s' % (10000 + n)
for n in xrange(ns)
]
def lease(self, n):
sockets = self.sockets[self.idx:self.idx+n]
self.idx += n
return sockets
def reaquire(self, *conn):
pass
class SimulatorBase(ComponentHost):
"""
Simulator coordinates the launch and communication of source, feed, transform, and merge components.
"""
def __init__(self, addresses, gevent_needed=False):
"""
"""
ComponentHost.__init__(self, addresses, gevent_needed)
def simulate(self):
self.run()
def get_id(self):
return "Simulator"
class ThreadSimulator(SimulatorBase):
def __init__(self, addresses):
SimulatorBase.__init__(self, addresses)
def launch_controller(self):
thread = threading.Thread(target=self.controller.run)
thread.start()
self.cuc = thread
return thread
def launch_component(self, component):
thread = threading.Thread(target=component.run)
thread.start()
return thread
class ThreadPoolExecutor(SimulatorTestCase, TestCase):
allocator = DummyAllocator(100)
def setup_logging(self):
qutil.configure_logging()
# lazy import by design
self.logger = mock.Mock()
def setup_allocator(self):
pass
def get_simulator(self, addresses):
return ThreadSimulator(addresses)
def get_controller(self):
# Allocate two more sockets
controller_sockets = self.allocate_sockets(2)
return Controller(
controller_sockets[0],
controller_sockets[1],
logging = self.logger,
)
zipline/test/test_messaging.py
+193 -44
View File
@@ -1,70 +1,222 @@
"""
Test suite for the messaging infrastructure of QSim.
Test suite for the messaging infrastructure of Zipline.
"""
#don't worry about excessive public methods pylint: disable=R0904
#don't worry about excessive public methods pylint: disable=R0904
import unittest2 as unittest
import multiprocessing
import time
from collections import defaultdict
import zipline.messaging as qmsg
from zipline.simulator import ThreadSimulator, ProcessSimulator
from zipline.transforms.technical import MovingAverage
from zipline.sources import RandomEquityTrades
import zipline.util as qutil
import zipline.messaging as qmsg
from zipline.test.client import TestClient
qutil.configure_logging()
class SimulatorTestCase(unittest.TestCase):
"""Tests the message passing: datasources -> feed -> transforms -> merge -> client"""
# Should not inherit form TestCase since test runners will pick
# it up as a test. Its a Mixin of sorts at this point.
class SimulatorTestCase(object):
leased_sockets = defaultdict(list)
def setUp(self):
"""generate some config objects for the datafeed, sources, and transforms."""
self.addresses = {'sync_address' : "tcp://127.0.0.1:10100",
'data_address' : "tcp://127.0.0.1:10101",
'feed_address' : "tcp://127.0.0.1:10102",
'merge_address' : "tcp://127.0.0.1:10103",
'result_address' : "tcp://127.0.0.1:10104",
'order_address' : "tcp://127.0.0.1:10105"
}
self.setup_logging()
# TODO: how to make Nose use this cross-process????
self.setup_allocator()
def tearDown(self):
pass
#self.unallocate_sockets()
# Assert the sockets were properly cleaned up
#self.assertEmpty(self.leased_sockets[self.id()].values())
# Assert they were returned to the heap
#self.allocator.socketheap.assert
def get_simulator(self):
return ThreadSimulator(self.addresses)
"""
Return a new simulator instance to be tested.
"""
raise NotImplementedError
def get_controller(self):
"""
Return a new controler for simulator instance to be tested.
"""
raise NotImplementedError
def setup_allocator(self):
"""
Setup the socket allocator for this test case.
"""
raise NotImplementedError
def allocate_sockets(self, n):
"""
Allocate sockets local to this test case, track them so
we can gc after test run.
"""
assert isinstance(n, int)
assert n > 0
leased = self.allocator.lease(n)
self.leased_sockets[self.id()].extend(leased)
return leased
def unallocate_sockets(self):
self.allocator.reaquire(*self.leased_sockets[self.id()])
# -------
# Cases
# -------
def test_simple(self):
# Simple test just to make sure that the archiecture is
# responding.
# Base Simuation
# --------------
# Allocate sockets for the simulator components
sockets = self.allocate_sockets(5)
addresses = {
'sync_address' : sockets[0],
'data_address' : sockets[1],
'feed_address' : sockets[2],
'merge_address' : sockets[3],
'result_address' : sockets[4]
}
sim = self.get_simulator(addresses)
con = self.get_controller()
# Simulation Components
# ---------------------
ret1 = RandomEquityTrades(133, "ret1", 1)
ret2 = RandomEquityTrades(134, "ret2", 1)
client = TestClient(self, expected_msg_count=ret1.count + ret2.count)
sim.register_controller( con )
sim.register_components([ret1, ret2, client])
# Simulation
# ----------
sim.simulate()
# Stop Running
# ------------
# TODO: less abrupt later, just shove a StopIteration
# down the pipe to make it stop spinning
sim.cuc._Thread__stop()
self.assertEqual(sim.feed.pending_messages(), 0,
"The feed should be drained of all messages, found {n} remaining."
.format(n=sim.feed.pending_messages())
)
def test_sources_only(self):
"""streams events from two data sources, no transforms."""
sim = self.get_simulator()
# Base Simuation
# --------------
# Allocate sockets for the simulator components
sockets = self.allocate_sockets(5)
addresses = {
'sync_address' : sockets[0],
'data_address' : sockets[1],
'feed_address' : sockets[2],
'merge_address' : sockets[3],
'result_address' : sockets[4]
}
sim = self.get_simulator(addresses)
con = self.get_controller()
# Simulation Components
# ---------------------
ret1 = RandomEquityTrades(133, "ret1", 400)
ret2 = RandomEquityTrades(134, "ret2", 400)
client = TestClient(self, expected_msg_count=800)
client = TestClient(self, expected_msg_count=ret1.count + ret2.count)
sim.register_controller( con )
sim.register_components([ret1, ret2, client])
# Simulation
# ----------
sim.simulate()
self.assertEqual(sim.feed.pending_messages(), 0,
"The feed should be drained of all messages, found {n} remaining."
.format(n=sim.feed.pending_messages()))
# Stop Running
# ------------
# TODO: less abrupt later, just shove a StopIteration
# down the pipe to make it stop spinning
sim.cuc._Thread__stop()
self.assertEqual(sim.feed.pending_messages(), 0,
"The feed should be drained of all messages, found {n} remaining."
.format(n=sim.feed.pending_messages())
)
def test_transforms(self):
"""
2 datasources -> feed -> 2 moving average transforms -> transform merge -> testclient
verify message count at client.
"""
sim = self.get_simulator()
# Base Simuation
# --------------
# Allocate sockets for the simulator components
sockets = self.allocate_sockets(5)
addresses = {
'sync_address' : sockets[0],
'data_address' : sockets[1],
'feed_address' : sockets[2],
'merge_address' : sockets[3],
'result_address' : sockets[4]
}
sim = self.get_simulator(addresses)
con = self.get_controller()
# Simulation Components
# ---------------------
ret1 = RandomEquityTrades(133, "ret1", 5000)
ret2 = RandomEquityTrades(134, "ret2", 5000)
mavg1 = MovingAverage("mavg1", 30)
mavg2 = MovingAverage("mavg2", 60)
client = TestClient(self, expected_msg_count=10000)
sim.register_components([ret1, ret2, mavg1, mavg2, client])
sim.register_controller( con )
# Simulation
# ----------
sim.simulate()
self.assertEqual(sim.feed.pending_messages(), 0, "The feed should be drained of all messages.")
# Stop Running
# ------------
# TODO: less abrupt later, just shove a StopIteration
# down the pipe to make it stop spinning
sim.cuc._Thread__stop()
self.assertEqual(sim.feed.pending_messages(), 0,
"The feed should be drained of all messages, found {n} remaining."
.format(n=sim.feed.pending_messages())
)
# TODO used?
def dtest_error_in_feed(self):
ret1 = RandomEquityTrades(133, "ret1", 400)
ret2 = RandomEquityTrades(134, "ret2", 400)
sources = {"ret1":ret1, "ret2":ret2}
@@ -73,12 +225,9 @@ class SimulatorTestCase(unittest.TestCase):
transforms = {"mavg1":mavg1, "mavg2":mavg2}
client = TestClient(self, expected_msg_count=0)
sim = self.get_simulator(sources, transforms, client)
# TODO: way too long
sim.feed = DataFeedErr(sources.keys(), sim.data_address, sim.feed_address, sim.performance_address, qmsg.Sync(sim, "DataFeedErrorGenerator"))
sim.simulate()
class ProcessSimulatorTestCase(SimulatorTestCase):
def get_simulator(self):
return ProcessSimulator(self.addresses)
zipline/test/test_sanity.py
+7
View File
@@ -0,0 +1,7 @@
from unittest2 import TestCase
class TestEnviroment(TestCase):
def test_universe(self):
# first order logic is working today. Yay!
self.assertTrue(True != False)
zipline/transforms/technical.py
+13 -13
View File
@@ -2,31 +2,31 @@
Transformations for common technical indicators.
TODO: add MACD transform
TODO: add trailing stop
"""
import datetime
from zipline.messaging import BaseTransform
import zipline.util as qutil
class MovingAverage(BaseTransform):
"""
Calculate a unweighted moving average for props['sid'] security
TODO: add sid -> mvavg dict.
Calculate a unweighted moving average for props['sid'] security
TODO: add sid -> mvavg dict.
"""
def __init__(self, name, days):
def __init__(self, name, days):
BaseTransform.__init__(self, name)
self.events = []
self.current_total = 0
self.window = datetime.timedelta(days = days)
self.window = datetime.timedelta(days = days)
def transform(self, event):
"""Update the moving average with the latest data point."""
self.events.append(event)
self.current_total += event['price']
event_date = qutil.parse_date(event['dt'])
index = 0
for cur_event in self.events:
cur_date = qutil.parse_date(cur_event['dt'])
@@ -36,12 +36,12 @@ class MovingAverage(BaseTransform):
index += 1
else:
break
if(len(self.events) == 0):
return 0.0
self.average = self.current_total/len(self.events)
self.state['value'] = self.average
return self.state