From 86938d445713d208a90318dbcc4246eeb5634a94 Mon Sep 17 00:00:00 2001 From: Eric LaFevers Date: Thu, 2 Jun 2016 19:22:35 -0700 Subject: [PATCH 1/7] DEV: VagrantFile for Ubuntu 14.04 that includes the development environment. --- Vagrantfile | 83 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index cea3af95..d0c82382 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,11 +1,82 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.configure("2") do |config| - config.vm.box = "precise64" - config.vm.box_url = "http://files.vagrantup.com/precise64.box" - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", 2048, "--cpus", 2] +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "ubuntu/trusty64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # Customize the amount of memory on the VM: + vb.memory = "2048" end - config.vm.provision "shell", path: "vagrant_init.sh" + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell", inline: <<-SHELL + sudo apt-get install -y python3-pip + sudo apt-get install -y libopenblas-dev liblapack-dev gfortran libfreetype6-dev pkg-config + wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + tar -xvzf ta-lib-0.4.0-src.tar.gz + cd ta-lib/ + ./configure --prefix=/usr + make + sudo make install + cd + sudo apt-get install -y git + git clone https://github.com/quantopian/zipline + cd ./zipline/ + sudo pip3 install -e .[all] + SHELL end From a728ef2bf1e3942dac75bb54409477c546ba3ef8 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 28 Jun 2016 08:34:35 -0400 Subject: [PATCH 2/7] DEV: Merged updated Vagrantfile with vagrant_init.sh --- Vagrantfile | 80 +++---------------------------------------------- vagrant_init.sh | 29 +++++++++--------- 2 files changed, 19 insertions(+), 90 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d0c82382..b1f9671f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,82 +1,10 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - # Every Vagrant development environment requires a box. You can search for - # boxes at https://atlas.hashicorp.com/search. +Vagrant.configure("2") do |config| config.vm.box = "ubuntu/trusty64" - - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # config.vm.network "forwarded_port", guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # Customize the amount of memory on the VM: - vb.memory = "2048" + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", 2048, "--cpus", 2] end - # - # View the documentation for the provider you are using for more - # information on available options. - - # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies - # such as FTP and Heroku are also available. See the documentation at - # https://docs.vagrantup.com/v2/push/atlas.html for more information. - # config.push.define "atlas" do |push| - # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" - # end - - # Enable provisioning with a shell script. Additional provisioners such as - # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the - # documentation for more information about their specific syntax and use. - config.vm.provision "shell", inline: <<-SHELL - sudo apt-get install -y python3-pip - sudo apt-get install -y libopenblas-dev liblapack-dev gfortran libfreetype6-dev pkg-config - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - tar -xvzf ta-lib-0.4.0-src.tar.gz - cd ta-lib/ - ./configure --prefix=/usr - make - sudo make install - cd - sudo apt-get install -y git - git clone https://github.com/quantopian/zipline - cd ./zipline/ - sudo pip3 install -e .[all] - SHELL + config.vm.provision "shell", path: "vagrant_init.sh" end diff --git a/vagrant_init.sh b/vagrant_init.sh index e92983b2..4f5cf920 100644 --- a/vagrant_init.sh +++ b/vagrant_init.sh @@ -21,17 +21,16 @@ VAGRANT_LOG="/home/vagrant/vagrant.log" echo "Obstructing updates to grub-pc..." apt-mark hold grub-pc 2>&1 >> "$VAGRANT_LOG" -# Run a full apt-get update first. +echo "Adding python apt repo..." +apt-add-repository -y ppa:fkrull/deadsnakes-python2.7 2>&1 >> "$VAGRANT_LOG" echo "Updating apt-get caches..." apt-get -y update 2>&1 >> "$VAGRANT_LOG" -# Install required packages -echo "Installing required packages..." -apt-get -y install python-pip python-dev g++ make libfreetype6-dev libpng-dev libopenblas-dev liblapack-dev gfortran 2>&1 >> "$VAGRANT_LOG" +echo "Installing required system packages..." +apt-get -y install python2.7 python-dev g++ make libfreetype6-dev libpng-dev libopenblas-dev liblapack-dev gfortran pkg-config git 2>&1 >> "$VAGRANT_LOG" -# Add ta-lib -echo "Installing ta-lib integration..." -wget http://switch.dl.sourceforge.net/project/ta-lib/ta-lib/0.4.0/ta-lib-0.4.0-src.tar.gz 2>&1 "$VAGRANT_LOG" +echo "Installing ta-lib..." +wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz --no-verbose -a "$VAGRANT_LOG" tar -xvzf ta-lib-0.4.0-src.tar.gz 2>&1 >> "$VAGRANT_LOG" cd ta-lib/ ./configure --prefix=/usr 2>&1 >> "$VAGRANT_LOG" @@ -39,12 +38,14 @@ make 2>&1 >> "$VAGRANT_LOG" sudo make install 2>&1 >> "$VAGRANT_LOG" cd ../ -# Add Zipline python dependencies -echo "Installing python package dependencies..." +echo "Installing pip and setuptools..." +wget https://bootstrap.pypa.io/get-pip.py 2>&1 >> "$VAGRANT_LOG" +python get-pip.py 2>&1 >> "$VAGRANT_LOG" +echo "Installing zipline python dependencies..." /vagrant/etc/ordered_pip.sh /vagrant/etc/requirements.txt 2>&1 >> "$VAGRANT_LOG" -# Add scipy next (if it's not done now, breaks installing of statsmodels for some reason ??) -echo "Installing scipy..." -pip install scipy==0.12.0 2>&1 >> "$VAGRANT_LOG" -echo "Installing zipline dev python dependencies..." -pip install --exists-action w -r /vagrant/etc/requirements_dev.txt 2>&1 >> "$VAGRANT_LOG" +echo "Installing zipline extra python dependencies..." +pip install -r /vagrant/etc/requirements_dev.txt -r /vagrant/etc/requirements_blaze.txt 2>&1 >> "$VAGRANT_LOG" +echo "Installing zipline package itself..." +find /vagrant/ -type f -name '*.c' -exec rm {} + +pip install -e /vagrant[all] 2>&1 >> "$VAGRANT_LOG" echo "Finished!" From ec4336ff0ce6ed089655a0bf26e9d1bf98de760c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 28 Jun 2016 08:35:15 -0400 Subject: [PATCH 3/7] DEV: Exit on error --- vagrant_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/vagrant_init.sh b/vagrant_init.sh index 4f5cf920..70e107ca 100644 --- a/vagrant_init.sh +++ b/vagrant_init.sh @@ -10,6 +10,7 @@ # from the root of the zipline github checkout. # This will put the zipline code in the # /vagrant folder in the system. +set -e VAGRANT_LOG="/home/vagrant/vagrant.log" From bd45bb0230ae34e05035dc048b96990a31997a3f Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 28 Jun 2016 08:35:31 -0400 Subject: [PATCH 4/7] MAINT: Tee output to console and log file --- vagrant_init.sh | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/vagrant_init.sh b/vagrant_init.sh index 70e107ca..da133328 100644 --- a/vagrant_init.sh +++ b/vagrant_init.sh @@ -19,34 +19,34 @@ VAGRANT_LOG="/home/vagrant/vagrant.log" # (grub-pc will complain that your boot device changed, probably # due to something that vagrant did, and break your console) -echo "Obstructing updates to grub-pc..." -apt-mark hold grub-pc 2>&1 >> "$VAGRANT_LOG" +echo "Obstructing updates to grub-pc..." | tee -a "$VAGRANT_LOG" +apt-mark hold grub-pc 2>&1 | tee -a "$VAGRANT_LOG" -echo "Adding python apt repo..." -apt-add-repository -y ppa:fkrull/deadsnakes-python2.7 2>&1 >> "$VAGRANT_LOG" -echo "Updating apt-get caches..." -apt-get -y update 2>&1 >> "$VAGRANT_LOG" +echo "Adding python apt repo..." | tee -a "$VAGRANT_LOG" +apt-add-repository -y ppa:fkrull/deadsnakes-python2.7 2>&1 | tee -a "$VAGRANT_LOG" +echo "Updating apt-get caches..." | tee -a "$VAGRANT_LOG" +apt-get -y update 2>&1 | tee -a "$VAGRANT_LOG" -echo "Installing required system packages..." -apt-get -y install python2.7 python-dev g++ make libfreetype6-dev libpng-dev libopenblas-dev liblapack-dev gfortran pkg-config git 2>&1 >> "$VAGRANT_LOG" +echo "Installing required system packages..." | tee -a "$VAGRANT_LOG" +apt-get -y install python2.7 python-dev g++ make libfreetype6-dev libpng-dev libopenblas-dev liblapack-dev gfortran pkg-config git 2>&1 | tee -a "$VAGRANT_LOG" -echo "Installing ta-lib..." +echo "Installing ta-lib..." | tee -a "$VAGRANT_LOG" wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz --no-verbose -a "$VAGRANT_LOG" -tar -xvzf ta-lib-0.4.0-src.tar.gz 2>&1 >> "$VAGRANT_LOG" +tar -xvzf ta-lib-0.4.0-src.tar.gz 2>&1 | tee -a "$VAGRANT_LOG" cd ta-lib/ -./configure --prefix=/usr 2>&1 >> "$VAGRANT_LOG" -make 2>&1 >> "$VAGRANT_LOG" -sudo make install 2>&1 >> "$VAGRANT_LOG" +./configure --prefix=/usr 2>&1 | tee -a "$VAGRANT_LOG" +make 2>&1 | tee -a "$VAGRANT_LOG" +sudo make install 2>&1 | tee -a "$VAGRANT_LOG" cd ../ -echo "Installing pip and setuptools..." -wget https://bootstrap.pypa.io/get-pip.py 2>&1 >> "$VAGRANT_LOG" -python get-pip.py 2>&1 >> "$VAGRANT_LOG" -echo "Installing zipline python dependencies..." -/vagrant/etc/ordered_pip.sh /vagrant/etc/requirements.txt 2>&1 >> "$VAGRANT_LOG" -echo "Installing zipline extra python dependencies..." -pip install -r /vagrant/etc/requirements_dev.txt -r /vagrant/etc/requirements_blaze.txt 2>&1 >> "$VAGRANT_LOG" -echo "Installing zipline package itself..." +echo "Installing pip and setuptools..." | tee -a "$VAGRANT_LOG" +wget https://bootstrap.pypa.io/get-pip.py 2>&1 | tee -a "$VAGRANT_LOG" +python get-pip.py 2>&1 >> "$VAGRANT_LOG" | tee -a "$VAGRANT_LOG" +echo "Installing zipline python dependencies..." | tee -a "$VAGRANT_LOG" +/vagrant/etc/ordered_pip.sh /vagrant/etc/requirements.txt 2>&1 | tee -a "$VAGRANT_LOG" +echo "Installing zipline extra python dependencies..." | tee -a "$VAGRANT_LOG" +pip install -r /vagrant/etc/requirements_dev.txt -r /vagrant/etc/requirements_blaze.txt 2>&1 | tee -a "$VAGRANT_LOG" +echo "Installing zipline package itself..." | tee -a "$VAGRANT_LOG" find /vagrant/ -type f -name '*.c' -exec rm {} + -pip install -e /vagrant[all] 2>&1 >> "$VAGRANT_LOG" -echo "Finished!" +pip install -e /vagrant[all] 2>&1 | tee -a "$VAGRANT_LOG" +echo "Finished!" | tee -a "$VAGRANT_LOG" From 95d59c1ddb6bb9ab4d39d7b53a7d7d5fad4d272a Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 28 Jun 2016 19:04:36 -0400 Subject: [PATCH 5/7] DOC: Include info about location of zipline repo inside vagrant --- vagrant_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vagrant_init.sh b/vagrant_init.sh index da133328..815406ff 100644 --- a/vagrant_init.sh +++ b/vagrant_init.sh @@ -49,4 +49,4 @@ pip install -r /vagrant/etc/requirements_dev.txt -r /vagrant/etc/requirements_bl echo "Installing zipline package itself..." | tee -a "$VAGRANT_LOG" find /vagrant/ -type f -name '*.c' -exec rm {} + pip install -e /vagrant[all] 2>&1 | tee -a "$VAGRANT_LOG" -echo "Finished!" | tee -a "$VAGRANT_LOG" +echo "Finished! zipline repo is in '/vagrant'." | tee -a "$VAGRANT_LOG" From 8f9bc98eb1e846eba42741899de30e60f6d377c5 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 29 Jun 2016 07:34:16 -0400 Subject: [PATCH 6/7] DEV: Added cleaning of cython assets to dev Dockerfile as well Added same comment to vagrant init --- Dockerfile-dev | 6 +++++- vagrant_init.sh | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index a95cc130..47dfd0c6 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -23,4 +23,8 @@ # FROM quantopian/zipline -RUN cd /zipline && pip install -r etc/requirements_dev.txt -r etc/requirements_blaze.txt +WORKDIR /zipline + +RUN pip install -r etc/requirements_dev.txt -r etc/requirements_blaze.txt +# Clean out any cython assets. The pip install re-builds them. +RUN find . -type f -name '*.c' -exec rm {} + && pip install -e .[all] diff --git a/vagrant_init.sh b/vagrant_init.sh index 815406ff..e2ca39ce 100644 --- a/vagrant_init.sh +++ b/vagrant_init.sh @@ -47,6 +47,7 @@ echo "Installing zipline python dependencies..." | tee -a "$VAGRANT_LOG" echo "Installing zipline extra python dependencies..." | tee -a "$VAGRANT_LOG" pip install -r /vagrant/etc/requirements_dev.txt -r /vagrant/etc/requirements_blaze.txt 2>&1 | tee -a "$VAGRANT_LOG" echo "Installing zipline package itself..." | tee -a "$VAGRANT_LOG" +# Clean out any cython assets. The pip install re-builds them. find /vagrant/ -type f -name '*.c' -exec rm {} + pip install -e /vagrant[all] 2>&1 | tee -a "$VAGRANT_LOG" echo "Finished! zipline repo is in '/vagrant'." | tee -a "$VAGRANT_LOG" From 424d7747dab086a022304fc18158bb0bb341cd86 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 7 Jul 2016 16:01:48 -0400 Subject: [PATCH 7/7] DOC: Updated whatsnew with VagrantFile update --- docs/source/whatsnew/1.0.2.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/whatsnew/1.0.2.txt b/docs/source/whatsnew/1.0.2.txt index f88bf8c8..46e609c6 100644 --- a/docs/source/whatsnew/1.0.2.txt +++ b/docs/source/whatsnew/1.0.2.txt @@ -12,6 +12,9 @@ Enhancements must search for when querying data. The checkpoints should have novel deltas applied (:issue:`1276`). +- Updated VagrantFile to include all dev requirements and use a newer image + (:issue:`1310`). + Bug Fixes ~~~~~~~~~