mirror of
https://github.com/wassname/cookiecutter-data-science.git
synced 2026-06-27 16:30:02 +08:00
poetry, just
This commit is contained in:
wassname
@@ -1,133 +0,0 @@
|
||||
# .PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3 doc_reqs test
|
||||
|
||||
# #################################################################################
|
||||
# # GLOBALS #
|
||||
# #################################################################################
|
||||
|
||||
# SHELL := /bin/bash
|
||||
# PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||
# BUCKET = {{ cookiecutter.s3_bucket }}
|
||||
# PROFILE = {{ cookiecutter.aws_profile }}
|
||||
# PROJECT_NAME = {{ cookiecutter.repo_name }}
|
||||
# PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }}
|
||||
|
||||
# #################################################################################
|
||||
# # COMMANDS #
|
||||
# #################################################################################
|
||||
|
||||
# ## Install Python Dependencies
|
||||
# requirements: test_environment
|
||||
# conda env create --name {{ cookiecutter.repo_name }} python=3.7 -f ./requirements/environment.yaml
|
||||
|
||||
# ## Make Dataset
|
||||
# data: requirements
|
||||
# $(PYTHON_INTERPRETER) src/data/make_dataset.py data/raw data/processed
|
||||
|
||||
# ## Delete all compiled Python files
|
||||
# clean:
|
||||
# find . -type f -name "*.py[co]" -delete
|
||||
# find . -type d -name "__pycache__" -delete
|
||||
|
||||
# ## Lint using flake8
|
||||
# lint:
|
||||
# flake8 src
|
||||
|
||||
# ## Upload Data to S3
|
||||
# sync_data_to_s3:
|
||||
# ifeq (default,$(PROFILE))
|
||||
# aws s3 sync data/ s3://$(BUCKET)/data/
|
||||
# else
|
||||
# aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
||||
# endif
|
||||
|
||||
# ## Download Data from S3
|
||||
# sync_data_from_s3:
|
||||
# ifeq (default,$(PROFILE))
|
||||
# aws s3 sync s3://$(BUCKET)/data/ data/
|
||||
# else
|
||||
# aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
||||
# endif
|
||||
|
||||
# ## Set up python interpreter environment
|
||||
# create_environment:
|
||||
# @echo ">>> Detected conda, creating conda environment."
|
||||
# conda env create --name $(PROJECT_NAME) python=3 -f ./requirements/environment.yaml
|
||||
# @echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
|
||||
|
||||
# ## Run pytest
|
||||
# test:
|
||||
# $(PYTHON_INTERPRETER) -m pytest ./test -v -s
|
||||
|
||||
# ## Export project requirements in multiple formats
|
||||
# doc_reqs:
|
||||
# conda env export --no-builds --from-history --name $(PROJECT_NAME) > requirements/environment.min.yaml
|
||||
# conda env export --name $(PROJECT_NAME) > requirements/environment.max.yaml
|
||||
# $(PYTHON_INTERPRETER) -m pip freeze > requirements/pip.conda.txt
|
||||
# cd requirements && conda-lock -f environment.max.yaml -p linux-64
|
||||
|
||||
# #################################################################################
|
||||
# # PROJECT RULES #
|
||||
# #################################################################################
|
||||
|
||||
|
||||
|
||||
# #################################################################################
|
||||
# # Self Documenting Commands #
|
||||
# #################################################################################
|
||||
|
||||
# .DEFAULT_GOAL := help
|
||||
|
||||
# # Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
|
||||
# # sed script explained:
|
||||
# # /^##/:
|
||||
# # * save line in hold space
|
||||
# # * purge line
|
||||
# # * Loop:
|
||||
# # * append newline + line to hold space
|
||||
# # * go to next line
|
||||
# # * if line starts with doc comment, strip comment character off and loop
|
||||
# # * remove target prerequisites
|
||||
# # * append hold space (+ newline) to line
|
||||
# # * replace newline plus comments by `---`
|
||||
# # * print line
|
||||
# # Separate expressions are necessary because labels cannot be delimited by
|
||||
# # semicolon; see <http://stackoverflow.com/a/11799865/1968>
|
||||
# .PHONY: help
|
||||
# help:
|
||||
# @echo "$$(tput bold)Available rules:$$(tput sgr0)"
|
||||
# @echo
|
||||
# @sed -n -e "/^## / { \
|
||||
# h; \
|
||||
# s/.*//; \
|
||||
# :doc" \
|
||||
# -e "H; \
|
||||
# n; \
|
||||
# s/^## //; \
|
||||
# t doc" \
|
||||
# -e "s/:.*//; \
|
||||
# G; \
|
||||
# s/\\n## /---/; \
|
||||
# s/\\n/ /g; \
|
||||
# p; \
|
||||
# }" ${MAKEFILE_LIST} \
|
||||
# | LC_ALL='C' sort --ignore-case \
|
||||
# | awk -F '---' \
|
||||
# -v ncol=$$(tput cols) \
|
||||
# -v indent=19 \
|
||||
# -v col_on="$$(tput setaf 6)" \
|
||||
# -v col_off="$$(tput sgr0)" \
|
||||
# '{ \
|
||||
# printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
|
||||
# n = split($$2, words, " "); \
|
||||
# line_length = ncol - indent; \
|
||||
# for (i = 1; i <= n; i++) { \
|
||||
# line_length -= length(words[i]) + 1; \
|
||||
# if (line_length <= 0) { \
|
||||
# line_length = ncol - indent - length(words[i]) - 1; \
|
||||
# printf "\n%*s ", -indent, " "; \
|
||||
# } \
|
||||
# printf "%s ", words[i]; \
|
||||
# } \
|
||||
# printf "\n"; \
|
||||
# }' \
|
||||
# | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
|
||||
@@ -1,11 +1,10 @@
|
||||
# {{cookiecutter.project_name}}
|
||||
|
||||
|
||||
{{cookiecutter.description}}
|
||||
|
||||
## Project Organization
|
||||
|
||||
├── Makefile <- Makefile with commands like `make data` or `make train`
|
||||
├── Justfile <- Makefile with commands like `make data` or `make train`
|
||||
├── README.md <- The top-level README for developers using this project.
|
||||
├── data
|
||||
│ ├── interim <- Intermediate data that has been transformed.
|
||||
@@ -13,15 +12,10 @@
|
||||
│ └── raw <- The original, immutable data dump.
|
||||
│
|
||||
│
|
||||
├── notebooks <- Jupyter notebooks. Naming convention is a number (for ordering),
|
||||
│ the creator's initials, and a short `-` delimited description, e.g.
|
||||
│ `1.0-jqp-initial-data-exploration`.
|
||||
├── notebooks <- Jupyter notebooks. Namiwith creator's initials, a number (for ordering), and short `-` delimited description, e.g.
|
||||
│ `jqp-1.0-initial-data-exploration`.
|
||||
│
|
||||
│
|
||||
├── requirements <- The requirements files for reproducing the analysis environment, e.g.
|
||||
│ generated with `make doc_reqs`
|
||||
│
|
||||
├── setup.py <- makes project pip installable (pip install -e .) so src can be imported
|
||||
├── pyproject.toml <- defines project dependencies and build configuration
|
||||
├── src <- Source code for use in this project.
|
||||
│ ├── __init__.py <- Makes src a Python module
|
||||
│ │
|
||||
@@ -42,12 +36,7 @@
|
||||
|
||||
## Install requirements
|
||||
```
|
||||
mamba env create --name {{ cookiecutter.repo_name }} python=3.9 -f ./requirements/environment.yaml
|
||||
conda activate {{ cookiecutter.repo_name }}
|
||||
# Install this package in editable mode
|
||||
python -m pip install -e .
|
||||
# Install kernel
|
||||
python -m ipykernel install --user --name {{ cookiecutter.repo_name }} --display-name {{ cookiecutter.repo_name }}
|
||||
poetry install
|
||||
```
|
||||
|
||||
## How to get data
|
||||
@@ -58,30 +47,6 @@ TODO document how to get the data
|
||||
|
||||
TODO document how to run the code
|
||||
|
||||
## AWS Policy for data sync
|
||||
|
||||
See [this link](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html)
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "ListObjectsInBucket",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::{{ cookiecutter.s3_bucket }}"]
|
||||
},
|
||||
{
|
||||
"Sid": "AllObjectActions",
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:*Object",
|
||||
"Resource": ["arn:aws:s3:::{{ cookiecutter.s3_bucket }}/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
--------
|
||||
|
||||
<p><small>Project based on the <a target="_blank" href="https://drivendata.github.io/cookiecutter-data-science/">cookiecutter data science project template</a>. #cookiecutterdatascience</small></p>
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
# Tasks
|
||||
|
||||
Tasks running for [mask](https://github.com/jacobdeichert/mask).
|
||||
|
||||
|
||||
## freeze
|
||||
> record pip and conda requirements
|
||||
|
||||
~~~sh
|
||||
export PROJECT_NAME={{ cookiecutter.repo_name }}
|
||||
mkdir -p requirements
|
||||
conda env export --no-builds --from-history --name $PROJECT_NAME > requirements/environment.min.yaml
|
||||
conda env export --name $PROJECT_NAME > requirements/environment.max.yaml
|
||||
python -m pip freeze > requirements/pip.conda.txt
|
||||
# cd requirements && conda-lock -f environment.max.yaml -p linux-64
|
||||
~~~
|
||||
|
||||
## create_environment
|
||||
|
||||
~~~sh
|
||||
export PROJECT_NAME={{ cookiecutter.repo_name }}
|
||||
@echo ">>> Detected conda, creating conda environment."
|
||||
conda env create --name $(PROJECT_NAME) python=3 -f ./requirements/environment.yaml
|
||||
@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
|
||||
~~~
|
||||
+3
-3
@@ -25,7 +25,7 @@
|
||||
"# autoreload your package\n",
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2\n",
|
||||
"import {{ cookiecutter.project_name.lower().replace(' ', '_') }}"
|
||||
"import {{ cookiecutter.project_name.lower().replace(' ', '_') }}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -72,7 +72,7 @@
|
||||
"# logging\n",
|
||||
"from loguru import logger\n",
|
||||
"logger.remove()\n",
|
||||
"logger.add(os.sys.stdout, level=\"ERROR\", colorize=True, format=\"<level>{time} | {message}</level>\")"
|
||||
"logger.add(os.sys.stdout, level=\"ERROR\", colorize=True, format=\"<level>{time} | {message}</level>\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -129,7 +129,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.4"
|
||||
"version": "3.10.12"
|
||||
},
|
||||
"toc": {
|
||||
"base_numbering": 1,
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e -x
|
||||
# This script will document the requirements for multiple conda environments, see ./readme.md for more
|
||||
|
||||
# inputs
|
||||
PROJECT_NAMES='{{ cookiecutter.python_interpreter }}'
|
||||
|
||||
for PROJECT_NAME in $PROJECT_NAMES
|
||||
do
|
||||
echo $PROJECT_NAME
|
||||
PYTHON_INTERPRETER=~/miniforge3/envs/$PROJECT_NAME/bin/python
|
||||
# minimal requirement, simpler, but no versions or pip
|
||||
conda env export --no-builds --from-history > requirements/environment.min.yaml
|
||||
# extensive requirements including pip and information overload
|
||||
conda env export > requirements/environment.max.yaml
|
||||
# requirements in a modified pip spec, usefull for dependabot and so on
|
||||
$PYTHON_INTERPRETER -m pip freeze > requirements/conda.requirements.txt
|
||||
done
|
||||
|
||||
# inputs
|
||||
for PROJECT_NAME in $PROJECT_NAMES
|
||||
do
|
||||
echo $PROJECT_NAME
|
||||
# conda lock is good for not overspecifying version, but it misses pip
|
||||
cd requirements && conda-lock -f environment.max.yaml -p linux-64
|
||||
done
|
||||
@@ -1,17 +0,0 @@
|
||||
name: $(PROJECT_NAME)
|
||||
channels:
|
||||
- pytorch
|
||||
- conda-forge
|
||||
- main
|
||||
- defaults
|
||||
dependencies:
|
||||
- python=3.7
|
||||
- pip
|
||||
- ipykernel
|
||||
# external requirements
|
||||
- awscli
|
||||
- python-dotenv
|
||||
- pip:
|
||||
# local package
|
||||
- -e ..
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
This project has multiple ways of documenting requirements. Since it doesn't cost anything we include them all.
|
||||
|
||||
- environment.yaml - This is the manual requirements, use it to install a new test or dev environment
|
||||
- environment.min.yaml - This is the minimum requirements, use it to install a new test or dev environment
|
||||
- environment.max.yaml - This pins all conda packages, use for production or finding vulnerabilities
|
||||
- conda.requirements.txt - For people or bots not using conda
|
||||
|
||||
To update the files run
|
||||
`bash requiresments/doc_reqs.sh`
|
||||
@@ -1,15 +0,0 @@
|
||||
# local package
|
||||
-e .
|
||||
|
||||
# external requirements
|
||||
click
|
||||
Sphinx
|
||||
coverage
|
||||
awscli
|
||||
flake8
|
||||
python-dotenv>=0.5.1
|
||||
{% if cookiecutter.python_interpreter != 'python3' %}
|
||||
|
||||
# backwards compatibility
|
||||
pathlib2
|
||||
{% endif %}
|
||||
@@ -1,10 +0,0 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
setup(
|
||||
name='{{ cookiecutter.project_name.lower().replace(' ', '_') }}',
|
||||
packages=find_packages(),
|
||||
version='0.1.0',
|
||||
description='{{ cookiecutter.description }}',
|
||||
author='{{ cookiecutter.author_name }}',
|
||||
license='proprietary',
|
||||
)
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package := "{{cookiecutter.package_name}}"
|
||||
|
||||
# put your run commands here
|
||||
app:
|
||||
echo "hello world"
|
||||
|
||||
# black and isort
|
||||
lint:
|
||||
ruff .
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
[tool.poetry]
|
||||
name = "{{ cookiecutter.project_name.lower().replace(' ', '_') }}"
|
||||
version = "0.1.0"
|
||||
description = "{{ cookiecutter.description }}"
|
||||
authors = ["{{ cookiecutter.author_name }}"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.11,<3.13"
|
||||
torch = {version = "^2.1.0+cu118", source = "pytorch"}
|
||||
simple-parsing = "^0.1.4"
|
||||
tqdm = "^4.66.1"
|
||||
numpy = "^1.26.1"
|
||||
pandas = "^2.1.1"
|
||||
lightning = "^2.1.0"
|
||||
matplotlib = "^3.8.0"
|
||||
loguru = "^0.7.2"
|
||||
einops = "^0.7.0"
|
||||
scikit-learn = "^1.3.1"
|
||||
pytorch-optimizer = "^2.12.0"
|
||||
torchinfo = "^1.8.0"
|
||||
|
||||
[[tool.poetry.source]]
|
||||
name = "pytorch"
|
||||
url = "https://download.pytorch.org/whl/cu118"
|
||||
priority = "explicit"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
ipykernel = "^6.25.2"
|
||||
black = "^23.10.0"
|
||||
pylama = "^8.4.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
Reference in New Issue
Block a user