mirror of
https://github.com/wassname/cookiecutter-data-science.git
synced 2026-08-09 12:00:08 +08:00
WIP - New version with cleaner options
This commit is contained in:
Peter Bull
+6
-1
@@ -6,4 +6,9 @@ docs/site/
|
||||
# test cache
|
||||
.cache/*
|
||||
tests/__pycache__/*
|
||||
*.pytest_cache/
|
||||
*.pytest_cache/
|
||||
*.pyc
|
||||
|
||||
# other local dev info
|
||||
.vscode/
|
||||
cookiecutter_data_science.egg-info/
|
||||
@@ -63,8 +63,8 @@ The directory structure of your new project looks like this:
|
||||
├── requirements.txt <- The requirements file for reproducing the analysis environment, e.g.
|
||||
│ generated with `pip freeze > requirements.txt`
|
||||
│
|
||||
├── src <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes src a Python module
|
||||
├── {{ cookiecutter.module_name }} <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes {{ cookiecutter.module_name }} a Python module
|
||||
│ │
|
||||
│ ├── data <- Scripts to download or generate data
|
||||
│ │ └── make_dataset.py
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
# 2/3 compat
|
||||
try:
|
||||
input = raw_input
|
||||
except NameError:
|
||||
pass
|
||||
|
||||
import click
|
||||
|
||||
# Monkey-patch jinja to allow variables to not exist, which happens with sub-options
|
||||
import jinja2
|
||||
jinja2.StrictUndefined = jinja2.Undefined
|
||||
|
||||
|
||||
# Monkey-patch cookiecutter to allow sub-items
|
||||
import cookiecutter
|
||||
from cookiecutter import prompt
|
||||
from ccds.monkey_patch import prompt_for_config
|
||||
|
||||
prompt.prompt_for_config = prompt_for_config
|
||||
|
||||
from cookiecutter import cli
|
||||
main = cli.main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,156 @@
|
||||
from collections import OrderedDict
|
||||
import json
|
||||
|
||||
import click
|
||||
from past.builtins import basestring
|
||||
|
||||
from future.utils import iteritems
|
||||
|
||||
from jinja2.exceptions import UndefinedError
|
||||
|
||||
from cookiecutter.exceptions import UndefinedVariableInTemplate
|
||||
from cookiecutter.environment import StrictEnvironment
|
||||
|
||||
|
||||
from cookiecutter.prompt import (prompt_choice_for_config, render_variable, read_user_variable, read_user_choice)
|
||||
|
||||
def _prompt_choice_and_subitems(cookiecutter_dict, env, key, options, no_input):
|
||||
result = {}
|
||||
|
||||
# first, get the selection
|
||||
rendered_options = [
|
||||
render_variable(env, list(raw.keys())[0], cookiecutter_dict) for raw in options
|
||||
]
|
||||
|
||||
if no_input:
|
||||
selected = rendered_options[0]
|
||||
|
||||
selected = read_user_choice(key, rendered_options)
|
||||
|
||||
selected_item = [list(c.values())[0] for c in options if list(c.keys())[0] == selected][0]
|
||||
|
||||
result[selected] = {}
|
||||
|
||||
# then, fill in the sub values for that item
|
||||
for subkey, raw in selected_item.items():
|
||||
# We are dealing with a regular variable
|
||||
val = render_variable(env, raw, cookiecutter_dict)
|
||||
|
||||
if not no_input:
|
||||
val = read_user_variable(subkey, val)
|
||||
|
||||
result[selected][subkey] = val
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def prompt_for_config(context, no_input=False):
|
||||
"""
|
||||
Prompts the user to enter new config, using context as a source for the
|
||||
field names and sample values.
|
||||
:param no_input: Prompt the user at command line for manual configuration?
|
||||
"""
|
||||
cookiecutter_dict = OrderedDict([])
|
||||
env = StrictEnvironment(context=context)
|
||||
|
||||
# First pass: Handle simple and raw variables, plus choices.
|
||||
# These must be done first because the dictionaries keys and
|
||||
# values might refer to them.
|
||||
for key, raw in iteritems(context[u'cookiecutter']):
|
||||
if key.startswith(u'_'):
|
||||
cookiecutter_dict[key] = raw
|
||||
continue
|
||||
|
||||
try:
|
||||
if isinstance(raw, list):
|
||||
if isinstance(raw[0], dict):
|
||||
val = _prompt_choice_and_subitems(
|
||||
cookiecutter_dict, env, key, raw, no_input
|
||||
)
|
||||
cookiecutter_dict[key] = val
|
||||
else:
|
||||
# We are dealing with a choice variable
|
||||
val = prompt_choice_for_config(
|
||||
cookiecutter_dict, env, key, raw, no_input
|
||||
)
|
||||
cookiecutter_dict[key] = val
|
||||
elif not isinstance(raw, dict):
|
||||
# We are dealing with a regular variable
|
||||
val = render_variable(env, raw, cookiecutter_dict)
|
||||
|
||||
if not no_input:
|
||||
val = read_user_variable(key, val)
|
||||
|
||||
cookiecutter_dict[key] = val
|
||||
except UndefinedError as err:
|
||||
msg = "Unable to render variable '{}'".format(key)
|
||||
raise UndefinedVariableInTemplate(msg, err, context)
|
||||
|
||||
# Second pass; handle the dictionaries.
|
||||
for key, raw in iteritems(context[u'cookiecutter']):
|
||||
|
||||
try:
|
||||
if isinstance(raw, dict):
|
||||
# We are dealing with a dict variable
|
||||
val = render_variable(env, raw, cookiecutter_dict)
|
||||
|
||||
if not no_input:
|
||||
val = read_user_dict(key, val)
|
||||
|
||||
cookiecutter_dict[key] = val
|
||||
except UndefinedError as err:
|
||||
msg = "Unable to render variable '{}'".format(key)
|
||||
raise UndefinedVariableInTemplate(msg, err, context)
|
||||
|
||||
return cookiecutter_dict
|
||||
|
||||
# from cookiecutter.main import cookiecutter
|
||||
# from cookiecutter import prompt
|
||||
# from cookiecutter.cli import main as cc_main
|
||||
|
||||
# class NestedQuestion:
|
||||
# ''' [{'a': {'val1': 'default1', 'val2': 'default2'}}]
|
||||
|
||||
# Interprets lists as questions with multiple options, where the
|
||||
# and dictionaries as single questions with defaults values.
|
||||
# '''
|
||||
# @classmethod
|
||||
# def update_context(cls, context, question_structure):
|
||||
# qd = question_structure
|
||||
# if isinstance(qd, list):
|
||||
# selection = cls.get_user_option(qd)
|
||||
|
||||
# name, vals = list(selection.items())[0]
|
||||
|
||||
# context[name] = {}
|
||||
# cls.update_context(context[name], vals)
|
||||
|
||||
# elif isinstance(qd, dict):
|
||||
# for k, v in qd.items():
|
||||
# context[k]= {}
|
||||
|
||||
# if isinstance(v, (dict, list)):
|
||||
# context[k] = cls.update_context(context[k], v)
|
||||
# else:
|
||||
# context[k] = cls.get_user_input(k, v)
|
||||
|
||||
# return context
|
||||
|
||||
# @staticmethod
|
||||
# def get_user_input(key, default):
|
||||
# return prompt.read_user_variable(key, default)
|
||||
# # return input(f"{key} [{default}]: ") or default
|
||||
|
||||
# @staticmethod
|
||||
# def get_user_option(options):
|
||||
# prompt.read_user_choice()
|
||||
|
||||
# # input_msg = '\n'.join(
|
||||
# # f" [{ix + 1}] - {list(value.keys())[0]}" for ix, value in enumerate(options)
|
||||
# # )
|
||||
|
||||
# # prepend = 'Select an item:\n'
|
||||
# # postpend = "\n - Enter number [1]: "
|
||||
|
||||
# # ix = int(input(prepend + input_msg + postpend) or 1) - 1
|
||||
# # return options[ix]
|
||||
+22
-5
@@ -1,10 +1,27 @@
|
||||
{
|
||||
"project_name": "project_name",
|
||||
"repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_') }}",
|
||||
"module_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}",
|
||||
"author_name": "Your name (or your organization/company/team)",
|
||||
"description": "A short description of the project.",
|
||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"],
|
||||
"s3_bucket": "[OPTIONAL] your-bucket-for-syncing-data (do not include 's3://')",
|
||||
"aws_profile": "default",
|
||||
"python_interpreter": ["python3", "python"]
|
||||
}
|
||||
"python_version_number": "3.7",
|
||||
"dataset_storage": [
|
||||
{"none": {}},
|
||||
{"azure": {"container": "container-name"}},
|
||||
{"s3": {"bucket": "bucket-name", "aws_profile": "default"}},
|
||||
{"gcs": {"bucket": "bucket-name"}}
|
||||
],
|
||||
"environment_manager" : [
|
||||
"none",
|
||||
"conda",
|
||||
"virtualenv",
|
||||
"pipenv"
|
||||
],
|
||||
"dependency_file": [
|
||||
"none",
|
||||
"requirements.txt",
|
||||
"environment.yml",
|
||||
"Pipfile"
|
||||
],
|
||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"]
|
||||
}
|
||||
+10
-10
@@ -54,7 +54,7 @@ Disagree with a couple of the default folder names? Working on a project that's
|
||||
|
||||
## Getting started
|
||||
|
||||
With this in mind, we've created a data science cookiecutter template for projects in Python. Your analysis doesn't have to be in Python, but the template does provide some Python boilerplate that you'd want to remove (in the `src` folder for example, and the Sphinx documentation skeleton in `docs`).
|
||||
With this in mind, we've created a data science cookiecutter template for projects in Python. Your analysis doesn't have to be in Python, but the template does provide some Python boilerplate that you'd want to remove (in the `{{ cookiecutter.module_name }}` folder for example, and the Sphinx documentation skeleton in `docs`).
|
||||
|
||||
### Requirements
|
||||
|
||||
@@ -72,7 +72,7 @@ cookiecutter https://github.com/drivendata/cookiecutter-data-science
|
||||
|
||||
### Example
|
||||
|
||||
<script type="text/javascript" src="https://asciinema.org/a/9bgl5qh17wlop4xyxu9n9wr02.js" id="asciicast-9bgl5qh17wlop4xyxu9n9wr02" async></script>
|
||||
<script type="text/javascript" {{ cookiecutter.module_name }}="https://asciinema.org/a/9bgl5qh17wlop4xyxu9n9wr02.js" id="asciicast-9bgl5qh17wlop4xyxu9n9wr02" async></script>
|
||||
|
||||
## Directory structure
|
||||
|
||||
@@ -103,8 +103,8 @@ cookiecutter https://github.com/drivendata/cookiecutter-data-science
|
||||
│ generated with `pip freeze > requirements.txt`
|
||||
│
|
||||
├── setup.py <- Make this project pip installable with `pip install -e`
|
||||
├── src <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes src a Python module
|
||||
├── {{ cookiecutter.module_name }} <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes {{ cookiecutter.module_name }} a Python module
|
||||
│ │
|
||||
│ ├── data <- Scripts to download or generate data
|
||||
│ │ └── make_dataset.py
|
||||
@@ -129,7 +129,7 @@ There are some opinions implicit in the project structure that have grown out of
|
||||
|
||||
### Data is immutable
|
||||
|
||||
Don't ever edit your raw data, especially not manually, and especially not in Excel. Don't overwrite your raw data. Don't save multiple versions of the raw data. Treat the data (and its format) as immutable. The code you write should move the raw data through a pipeline to your final analysis. You shouldn't have to run all of the steps every time you want to make a new figure (see [Analysis is a DAG](#analysis-is-a-dag)), but anyone should be able to reproduce the final products with only the code in `src` and the data in `data/raw`.
|
||||
Don't ever edit your raw data, especially not manually, and especially not in Excel. Don't overwrite your raw data. Don't save multiple versions of the raw data. Treat the data (and its format) as immutable. The code you write should move the raw data through a pipeline to your final analysis. You shouldn't have to run all of the steps every time you want to make a new figure (see [Analysis is a DAG](#analysis-is-a-dag)), but anyone should be able to reproduce the final products with only the code in `{{ cookiecutter.module_name }}` and the data in `data/raw`.
|
||||
|
||||
Also, if data is immutable, it doesn't need source control in the same way that code does. Therefore, ***by default, the data folder is included in the `.gitignore` file.*** If you have a small amount of data that rarely changes, you may want to include the data in the repository. Github currently warns if files are over 50MB and rejects files over 100MB. Some other options for storing/syncing large data include [AWS S3](https://aws.amazon.com/s3/) with a syncing tool (e.g., [`s3cmd`](http://s3tools.org/s3cmd)), [Git Large File Storage](https://git-lfs.github.com/), [Git Annex](https://git-annex.branchable.com/), and [dat](http://dat-data.com/). Currently by default, we ask for an S3 bucket and use [AWS CLI](http://docs.aws.amazon.com/cli/latest/reference/s3/index.html) to sync data in the `data` folder with the server.
|
||||
|
||||
@@ -141,7 +141,7 @@ Since notebooks are challenging objects for source control (e.g., diffs of the `
|
||||
|
||||
1. Follow a naming convention that shows the owner and the order the analysis was done in. We use the format `<step>-<ghuser>-<description>.ipynb` (e.g., `0.3-bull-visualize-distributions.ipynb`).
|
||||
|
||||
2. Refactor the good parts. Don't write code to do the same task in multiple notebooks. If it's a data preprocessing task, put it in the pipeline at `src/data/make_dataset.py` and load data from `data/interim`. If it's useful utility code, refactor it to `src`.
|
||||
2. Refactor the good parts. Don't write code to do the same task in multiple notebooks. If it's a data preprocessing task, put it in the pipeline at `{{ cookiecutter.module_name }}/data/make_dataset.py` and load data from `data/interim`. If it's useful utility code, refactor it to `{{ cookiecutter.module_name }}`.
|
||||
|
||||
Now by default we turn the project into a Python package (see the `setup.py` file). You can import your code and use it in notebooks with a cell like the following:
|
||||
|
||||
@@ -149,10 +149,10 @@ Since notebooks are challenging objects for source control (e.g., diffs of the `
|
||||
# OPTIONAL: Load the "autoreload" extension so that code can change
|
||||
%load_ext autoreload
|
||||
|
||||
# OPTIONAL: always reload modules so that as you change code in src, it gets loaded
|
||||
# OPTIONAL: always reload modules so that as you change code in {{ cookiecutter.module_name }}, it gets loaded
|
||||
%autoreload 2
|
||||
|
||||
from src.data import make_dataset
|
||||
from {{ cookiecutter.module_name }}.data import make_dataset
|
||||
```
|
||||
|
||||
### Analysis is a DAG
|
||||
@@ -192,10 +192,10 @@ OTHER_VARIABLE=something
|
||||
|
||||
#### Use a package to load these variables automatically.
|
||||
|
||||
If you look at the stub script in `src/data/make_dataset.py`, it uses a package called [python-dotenv](https://github.com/theskumar/python-dotenv) to load up all the entries in this file as environment variables so they are accessible with `os.environ.get`. Here's an example snippet adapted from the `python-dotenv` documentation:
|
||||
If you look at the stub script in `{{ cookiecutter.module_name }}/data/make_dataset.py`, it uses a package called [python-dotenv](https://github.com/theskumar/python-dotenv) to load up all the entries in this file as environment variables so they are accessible with `os.environ.get`. Here's an example snippet adapted from the `python-dotenv` documentation:
|
||||
|
||||
```python
|
||||
# src/data/dotenv_example.py
|
||||
# {{ cookiecutter.module_name }}/data/dotenv_example.py
|
||||
import os
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
|
||||
packages = [
|
||||
'flake8',
|
||||
'pathlib2',
|
||||
'pip',
|
||||
'setuptools',
|
||||
'wheel',
|
||||
]
|
||||
|
||||
pip_only_packages = [
|
||||
'awscli',
|
||||
'python-dotenv',
|
||||
]
|
||||
|
||||
{% if cookiecutter.dataset_storage.s3 %}
|
||||
packages += ['awscli']
|
||||
{% endif %}
|
||||
|
||||
dependencies = '{{ cookiecutter.dependency_file }}'
|
||||
|
||||
def write_dependencies():
|
||||
if dependencies == 'requirements.txt':
|
||||
with open(dependencies, 'w') as f:
|
||||
lines = sorted(packages + pip_only_packages)
|
||||
|
||||
lines += [
|
||||
""
|
||||
"-e ."
|
||||
]
|
||||
|
||||
f.write("\n".join(lines))
|
||||
|
||||
elif dependencies == 'environment.yml':
|
||||
with open(dependencies, 'w') as f:
|
||||
lines = ["name: {{ cookiecutter.repo_name }}",
|
||||
"dependencies:"]
|
||||
|
||||
lines += [f" - {p}" for p in packages]
|
||||
|
||||
lines += [" - pip:"] + [f" - {p}" for p in pip_only_packages]
|
||||
|
||||
lines += [' - -e .']
|
||||
|
||||
lines += [" - python={{ cookiecutter.python_version_number }}"]
|
||||
|
||||
f.write("\n".join(lines))
|
||||
|
||||
|
||||
elif dependencies == 'Pipfile':
|
||||
with open(dependencies, 'w') as f:
|
||||
lines = ["[packages]"]
|
||||
lines += [f'{p} = "*"' for p in sorted(packages + pip_only_packages)]
|
||||
|
||||
lines += ['"{{ cookiecutter.module_name }}" = {editable = true, path = "."}']
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"[requires]",
|
||||
'python_version = "{{ cookiecutter.python_version_number }}"'
|
||||
]
|
||||
|
||||
f.write("\n".join(lines))
|
||||
|
||||
|
||||
write_dependencies()
|
||||
@@ -0,0 +1,53 @@
|
||||
from pathlib import Path
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
project_path = Path(__file__).parent
|
||||
long_description = open(project_path / 'README.md').read()
|
||||
|
||||
setup(
|
||||
name='cookiecutter-data-science',
|
||||
version='0.2.0',
|
||||
description='A logical, reasonably standardized, but flexible project structure for doing and sharing data science work.',
|
||||
long_description=long_description,
|
||||
long_description_content_type='text/markdown',
|
||||
author='DrivenData',
|
||||
author_email='info@drivendata.org',
|
||||
url='https://drivendata.github.io/cookiecutter-data-science/',
|
||||
project_urls={
|
||||
'Homepage': 'https://drivendata.github.io/cookiecutter-data-science/',
|
||||
'Source Code': 'https://github.com/drivendata/cookiecutter-data-science/',
|
||||
'DrivenData': 'http://drivendata.co'
|
||||
},
|
||||
classifiers=[
|
||||
# How mature is this project? Common values are
|
||||
# 3 - Alpha
|
||||
# 4 - Beta
|
||||
# 5 - Production/Stable
|
||||
'Development Status :: 3 - Alpha',
|
||||
|
||||
# Indicate who your project is intended for
|
||||
'Intended Audience :: Developers',
|
||||
'Intended Audience :: Science/Research',
|
||||
|
||||
# Pick your license as you wish (should match "license" above)
|
||||
'License :: OSI Approved :: MIT License',
|
||||
|
||||
# Specify the Python versions you support here. In particular, ensure
|
||||
# that you indicate whether you support Python 2, Python 3 or both.
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
|
||||
'Topic :: Scientific/Engineering',
|
||||
'Topic :: Scientific/Engineering :: Artificial Intelligence',
|
||||
],
|
||||
python_requires='>=3.5',
|
||||
install_requires=['cookiecutter', 'click'],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'ccds=ccds.__main__:main',
|
||||
],
|
||||
},
|
||||
packages=find_packages(exclude=['dist', 'docs', 'tests']),
|
||||
)
|
||||
@@ -96,11 +96,11 @@ class TestCookieSetup(object):
|
||||
'references',
|
||||
'reports',
|
||||
'reports/figures',
|
||||
'src',
|
||||
'src/data',
|
||||
'src/features',
|
||||
'src/models',
|
||||
'src/visualization',
|
||||
'{{ cookiecutter.module_name }}',
|
||||
'{{ cookiecutter.module_name }}/data',
|
||||
'{{ cookiecutter.module_name }}/features',
|
||||
'{{ cookiecutter.module_name }}/models',
|
||||
'{{ cookiecutter.module_name }}/visualization',
|
||||
]
|
||||
|
||||
ignored_dirs = [
|
||||
|
||||
@@ -1,33 +1,26 @@
|
||||
.PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3
|
||||
.PHONY: clean data lint requirements sync_data_down sync_data_up
|
||||
|
||||
#################################################################################
|
||||
# GLOBALS #
|
||||
#################################################################################
|
||||
|
||||
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||
BUCKET = {{ cookiecutter.s3_bucket }}
|
||||
PROFILE = {{ cookiecutter.aws_profile }}
|
||||
PROJECT_NAME = {{ cookiecutter.repo_name }}
|
||||
PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }}
|
||||
PYTHON_INTERPRETER = python{{ cookiecutter.python_version_number }}
|
||||
|
||||
ifeq (,$(shell which conda))
|
||||
HAS_CONDA=False
|
||||
else
|
||||
HAS_CONDA=True
|
||||
endif
|
||||
|
||||
#################################################################################
|
||||
# COMMANDS #
|
||||
#################################################################################
|
||||
|
||||
## Install Python Dependencies
|
||||
requirements: test_environment
|
||||
requirements:
|
||||
$(PYTHON_INTERPRETER) -m pip install -U pip setuptools wheel
|
||||
$(PYTHON_INTERPRETER) -m pip install -r requirements.txt
|
||||
|
||||
## Make Dataset
|
||||
data: requirements
|
||||
$(PYTHON_INTERPRETER) src/data/make_dataset.py
|
||||
$(PYTHON_INTERPRETER) {{ cookiecutter.module_name }}/data/make_dataset.py
|
||||
|
||||
## Delete all compiled Python files
|
||||
clean:
|
||||
@@ -36,45 +29,49 @@ clean:
|
||||
|
||||
## Lint using flake8
|
||||
lint:
|
||||
flake8 src
|
||||
flake8 {{ cookiecutter.module_name }}
|
||||
|
||||
## Upload Data to S3
|
||||
sync_data_to_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/
|
||||
else
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
||||
endif
|
||||
{% if cookiecutter.dataset_storage.s3 %}
|
||||
## Download Data from storage system
|
||||
sync_data_down:
|
||||
{% if cookiecutter.dataset_storage.s3 %}
|
||||
aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/\
|
||||
data/ {% if cookiecutter.dataset_storage.s3.aws_profile != 'default' %} --profile {{ cookiecutter.dataset_storage.s3.aws_profile }}{% endif %}
|
||||
{% elif cookiecutter.dataset_storage.azure %}
|
||||
az storage blob download-batch -s {{ cookiecutter.dataset_storage.azure.container }}/data/ \
|
||||
-d data/
|
||||
{% elif cookiecutter.dataset_storage.gcs %}
|
||||
gsutil cp {{ cookiecutter.dataset_storage.gcs.bucket }}/data/ data/
|
||||
{% endif %}
|
||||
|
||||
## Download Data from S3
|
||||
sync_data_from_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/
|
||||
else
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
||||
endif
|
||||
## Upload Data to storage system
|
||||
sync_data_up:
|
||||
{% if cookiecutter.dataset_storage.s3 %}
|
||||
aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/ data/\
|
||||
{% if cookiecutter.dataset_storage.s3.aws_profile %} --profile $(PROFILE){% endif %}
|
||||
{% elif cookiecutter.dataset_storage.azure %}
|
||||
az storage blob upload-batch -d {{ cookiecutter.dataset_storage.azure.container }}/data/ \
|
||||
-s data/
|
||||
{% elif cookiecutter.dataset_storage.gcs %}
|
||||
gsutil cp data/ {{ cookiecutter.dataset_storage.gcs.bucket }}/data/
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% if cookiecutter.environment_manager != 'none' %}
|
||||
## Set up python interpreter environment
|
||||
create_environment:
|
||||
ifeq (True,$(HAS_CONDA))
|
||||
@echo ">>> Detected conda, creating conda environment."
|
||||
ifeq (3,$(findstring 3,$(PYTHON_INTERPRETER)))
|
||||
conda create --name $(PROJECT_NAME) python=3
|
||||
else
|
||||
conda create --name $(PROJECT_NAME) python=2.7
|
||||
endif
|
||||
@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
|
||||
else
|
||||
$(PYTHON_INTERPRETER) -m pip install -q virtualenv virtualenvwrapper
|
||||
@echo ">>> Installing virtualenvwrapper if not already intalled.\nMake sure the following lines are in shell startup file\n\
|
||||
export WORKON_HOME=$$HOME/.virtualenvs\nexport PROJECT_HOME=$$HOME/Devel\nsource /usr/local/bin/virtualenvwrapper.sh\n"
|
||||
{% if cookiecutter.environment_manager == 'conda' %}
|
||||
conda create --name $(PROJECT_NAME) python={{ cookiecutter.python_version_number }}{% if cookiecutter.dependency_file == 'environment.yml' %} -f environment.yml{% endif %}
|
||||
@echo ">>> conda env created. Activate with:\nconda activate $(PROJECT_NAME)"
|
||||
{% elif cookiecutter.environment_manager == 'virtualenv' %}
|
||||
@bash -c "source `which virtualenvwrapper.sh`;mkvirtualenv $(PROJECT_NAME) --python=$(PYTHON_INTERPRETER)"
|
||||
@echo ">>> New virtualenv created. Activate with:\nworkon $(PROJECT_NAME)"
|
||||
endif
|
||||
{% elif cookiecutter.environment_manager == 'pipenv' %}
|
||||
pipenv install
|
||||
@echo ">>> New pipenv created. Activate with:\npipenv shell"
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
## Test python environment is setup correctly
|
||||
test_environment:
|
||||
$(PYTHON_INTERPRETER) test_environment.py
|
||||
|
||||
#################################################################################
|
||||
# PROJECT RULES #
|
||||
|
||||
@@ -31,9 +31,9 @@ Project Organization
|
||||
├── requirements.txt <- The requirements file for reproducing the analysis environment, e.g.
|
||||
│ generated with `pip freeze > requirements.txt`
|
||||
│
|
||||
├── setup.py <- makes project pip installable (pip install -e .) so src can be imported
|
||||
├── src <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes src a Python module
|
||||
├── setup.py <- makes project pip installable (pip install -e .) so {{ cookiecutter.module_name }} can be imported
|
||||
├── {{ cookiecutter.module_name }} <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes {{ cookiecutter.module_name }} a Python module
|
||||
│ │
|
||||
│ ├── data <- Scripts to download or generate data
|
||||
│ │ └── make_dataset.py
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# local package
|
||||
-e .
|
||||
|
||||
# external requirements
|
||||
click
|
||||
Sphinx
|
||||
coverage
|
||||
awscli
|
||||
flake8
|
||||
python-dotenv>=0.5.1
|
||||
{% if cookiecutter.python_interpreter != 'python3' %}
|
||||
|
||||
# backwards compatibility
|
||||
pathlib2
|
||||
{% endif %}
|
||||
@@ -1,7 +1,7 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
setup(
|
||||
name='src',
|
||||
name='{{ cookiecutter.module_name }}',
|
||||
packages=find_packages(),
|
||||
version='0.1.0',
|
||||
description='{{ cookiecutter.description }}',
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import sys
|
||||
|
||||
REQUIRED_PYTHON = "{{ cookiecutter.python_interpreter }}"
|
||||
|
||||
|
||||
def main():
|
||||
system_major = sys.version_info.major
|
||||
if REQUIRED_PYTHON == "python":
|
||||
required_major = 2
|
||||
elif REQUIRED_PYTHON == "python3":
|
||||
required_major = 3
|
||||
else:
|
||||
raise ValueError("Unrecognized python interpreter: {}".format(
|
||||
REQUIRED_PYTHON))
|
||||
|
||||
if system_major != required_major:
|
||||
raise TypeError(
|
||||
"This project requires Python {}. Found: Python {}".format(
|
||||
required_major, sys.version))
|
||||
else:
|
||||
print(">>> Development environment passes all tests!")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user