diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile deleted file mode 100644 index 490679e..0000000 --- a/{{ cookiecutter.repo_name }}/Makefile +++ /dev/null @@ -1,133 +0,0 @@ -# .PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3 doc_reqs test - -# ################################################################################# -# # GLOBALS # -# ################################################################################# - -# SHELL := /bin/bash -# PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -# BUCKET = {{ cookiecutter.s3_bucket }} -# PROFILE = {{ cookiecutter.aws_profile }} -# PROJECT_NAME = {{ cookiecutter.repo_name }} -# PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }} - -# ################################################################################# -# # COMMANDS # -# ################################################################################# - -# ## Install Python Dependencies -# requirements: test_environment -# conda env create --name {{ cookiecutter.repo_name }} python=3.7 -f ./requirements/environment.yaml - -# ## Make Dataset -# data: requirements -# $(PYTHON_INTERPRETER) src/data/make_dataset.py data/raw data/processed - -# ## Delete all compiled Python files -# clean: -# find . -type f -name "*.py[co]" -delete -# find . -type d -name "__pycache__" -delete - -# ## Lint using flake8 -# lint: -# flake8 src - -# ## Upload Data to S3 -# sync_data_to_s3: -# ifeq (default,$(PROFILE)) -# aws s3 sync data/ s3://$(BUCKET)/data/ -# else -# aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE) -# endif - -# ## Download Data from S3 -# sync_data_from_s3: -# ifeq (default,$(PROFILE)) -# aws s3 sync s3://$(BUCKET)/data/ data/ -# else -# aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE) -# endif - -# ## Set up python interpreter environment -# create_environment: -# @echo ">>> Detected conda, creating conda environment." -# conda env create --name $(PROJECT_NAME) python=3 -f ./requirements/environment.yaml -# @echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)" - -# ## Run pytest -# test: -# $(PYTHON_INTERPRETER) -m pytest ./test -v -s - -# ## Export project requirements in multiple formats -# doc_reqs: -# conda env export --no-builds --from-history --name $(PROJECT_NAME) > requirements/environment.min.yaml -# conda env export --name $(PROJECT_NAME) > requirements/environment.max.yaml -# $(PYTHON_INTERPRETER) -m pip freeze > requirements/pip.conda.txt -# cd requirements && conda-lock -f environment.max.yaml -p linux-64 - -# ################################################################################# -# # PROJECT RULES # -# ################################################################################# - - - -# ################################################################################# -# # Self Documenting Commands # -# ################################################################################# - -# .DEFAULT_GOAL := help - -# # Inspired by -# # sed script explained: -# # /^##/: -# # * save line in hold space -# # * purge line -# # * Loop: -# # * append newline + line to hold space -# # * go to next line -# # * if line starts with doc comment, strip comment character off and loop -# # * remove target prerequisites -# # * append hold space (+ newline) to line -# # * replace newline plus comments by `---` -# # * print line -# # Separate expressions are necessary because labels cannot be delimited by -# # semicolon; see -# .PHONY: help -# help: -# @echo "$$(tput bold)Available rules:$$(tput sgr0)" -# @echo -# @sed -n -e "/^## / { \ -# h; \ -# s/.*//; \ -# :doc" \ -# -e "H; \ -# n; \ -# s/^## //; \ -# t doc" \ -# -e "s/:.*//; \ -# G; \ -# s/\\n## /---/; \ -# s/\\n/ /g; \ -# p; \ -# }" ${MAKEFILE_LIST} \ -# | LC_ALL='C' sort --ignore-case \ -# | awk -F '---' \ -# -v ncol=$$(tput cols) \ -# -v indent=19 \ -# -v col_on="$$(tput setaf 6)" \ -# -v col_off="$$(tput sgr0)" \ -# '{ \ -# printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ -# n = split($$2, words, " "); \ -# line_length = ncol - indent; \ -# for (i = 1; i <= n; i++) { \ -# line_length -= length(words[i]) + 1; \ -# if (line_length <= 0) { \ -# line_length = ncol - indent - length(words[i]) - 1; \ -# printf "\n%*s ", -indent, " "; \ -# } \ -# printf "%s ", words[i]; \ -# } \ -# printf "\n"; \ -# }' \ -# | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars') diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index b433fa8..052f1a0 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -1,11 +1,10 @@ # {{cookiecutter.project_name}} - {{cookiecutter.description}} ## Project Organization - ├── Makefile <- Makefile with commands like `make data` or `make train` + ├── Justfile <- Makefile with commands like `make data` or `make train` ├── README.md <- The top-level README for developers using this project. ├── data │   ├── interim <- Intermediate data that has been transformed. @@ -13,15 +12,10 @@ │   └── raw <- The original, immutable data dump. │ │ - ├── notebooks <- Jupyter notebooks. Naming convention is a number (for ordering), - │ the creator's initials, and a short `-` delimited description, e.g. - │ `1.0-jqp-initial-data-exploration`. + ├── notebooks <- Jupyter notebooks. Namiwith creator's initials, a number (for ordering), and short `-` delimited description, e.g. + │ `jqp-1.0-initial-data-exploration`. │ - │ - ├── requirements <- The requirements files for reproducing the analysis environment, e.g. - │ generated with `make doc_reqs` - │ - ├── setup.py <- makes project pip installable (pip install -e .) so src can be imported + ├── pyproject.toml <- defines project dependencies and build configuration ├── src <- Source code for use in this project. │   ├── __init__.py <- Makes src a Python module │ │ @@ -42,12 +36,7 @@ ## Install requirements ``` -mamba env create --name {{ cookiecutter.repo_name }} python=3.9 -f ./requirements/environment.yaml -conda activate {{ cookiecutter.repo_name }} -# Install this package in editable mode -python -m pip install -e . -# Install kernel -python -m ipykernel install --user --name {{ cookiecutter.repo_name }} --display-name {{ cookiecutter.repo_name }} +poetry install ``` ## How to get data @@ -58,30 +47,6 @@ TODO document how to get the data TODO document how to run the code -## AWS Policy for data sync - -See [this link](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ListObjectsInBucket", - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": ["arn:aws:s3:::{{ cookiecutter.s3_bucket }}"] - }, - { - "Sid": "AllObjectActions", - "Effect": "Allow", - "Action": "s3:*Object", - "Resource": ["arn:aws:s3:::{{ cookiecutter.s3_bucket }}/*"] - } - ] -} -``` - --------

Project based on the cookiecutter data science project template. #cookiecutterdatascience

diff --git a/{{ cookiecutter.repo_name }}/maskfile.md b/{{ cookiecutter.repo_name }}/maskfile.md deleted file mode 100644 index e18dc7b..0000000 --- a/{{ cookiecutter.repo_name }}/maskfile.md +++ /dev/null @@ -1,25 +0,0 @@ -# Tasks - -Tasks running for [mask](https://github.com/jacobdeichert/mask). - - -## freeze -> record pip and conda requirements - -~~~sh -export PROJECT_NAME={{ cookiecutter.repo_name }} -mkdir -p requirements -conda env export --no-builds --from-history --name $PROJECT_NAME > requirements/environment.min.yaml -conda env export --name $PROJECT_NAME > requirements/environment.max.yaml -python -m pip freeze > requirements/pip.conda.txt -# cd requirements && conda-lock -f environment.max.yaml -p linux-64 -~~~ - -## create_environment - -~~~sh -export PROJECT_NAME={{ cookiecutter.repo_name }} -@echo ">>> Detected conda, creating conda environment." -conda env create --name $(PROJECT_NAME) python=3 -f ./requirements/environment.yaml -@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)" -~~~ diff --git a/{{ cookiecutter.repo_name }}/notebooks/001_001_mjc_EDA.ipynb b/{{ cookiecutter.repo_name }}/notebooks/mjc_001_EDA.ipynb similarity index 97% rename from {{ cookiecutter.repo_name }}/notebooks/001_001_mjc_EDA.ipynb rename to {{ cookiecutter.repo_name }}/notebooks/mjc_001_EDA.ipynb index cb4bccf..e0b258d 100644 --- a/{{ cookiecutter.repo_name }}/notebooks/001_001_mjc_EDA.ipynb +++ b/{{ cookiecutter.repo_name }}/notebooks/mjc_001_EDA.ipynb @@ -25,7 +25,7 @@ "# autoreload your package\n", "%load_ext autoreload\n", "%autoreload 2\n", - "import {{ cookiecutter.project_name.lower().replace(' ', '_') }}" + "import {{ cookiecutter.project_name.lower().replace(' ', '_') }}\n" ] }, { @@ -72,7 +72,7 @@ "# logging\n", "from loguru import logger\n", "logger.remove()\n", - "logger.add(os.sys.stdout, level=\"ERROR\", colorize=True, format=\"{time} | {message}\")" + "logger.add(os.sys.stdout, level=\"ERROR\", colorize=True, format=\"{time} | {message}\")\n" ] }, { @@ -129,7 +129,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.10.12" }, "toc": { "base_numbering": 1, diff --git a/{{ cookiecutter.repo_name }}/requirements/doc_reqs.sh b/{{ cookiecutter.repo_name }}/requirements/doc_reqs.sh deleted file mode 100644 index b87161d..0000000 --- a/{{ cookiecutter.repo_name }}/requirements/doc_reqs.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e -x -# This script will document the requirements for multiple conda environments, see ./readme.md for more - -# inputs -PROJECT_NAMES='{{ cookiecutter.python_interpreter }}' - -for PROJECT_NAME in $PROJECT_NAMES -do - echo $PROJECT_NAME - PYTHON_INTERPRETER=~/miniforge3/envs/$PROJECT_NAME/bin/python - # minimal requirement, simpler, but no versions or pip - conda env export --no-builds --from-history > requirements/environment.min.yaml - # extensive requirements including pip and information overload - conda env export > requirements/environment.max.yaml - # requirements in a modified pip spec, usefull for dependabot and so on - $PYTHON_INTERPRETER -m pip freeze > requirements/conda.requirements.txt -done - -# inputs -for PROJECT_NAME in $PROJECT_NAMES -do - echo $PROJECT_NAME - # conda lock is good for not overspecifying version, but it misses pip - cd requirements && conda-lock -f environment.max.yaml -p linux-64 -done diff --git a/{{ cookiecutter.repo_name }}/requirements/environment.yaml b/{{ cookiecutter.repo_name }}/requirements/environment.yaml deleted file mode 100644 index bb75f9b..0000000 --- a/{{ cookiecutter.repo_name }}/requirements/environment.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: $(PROJECT_NAME) -channels: - - pytorch - - conda-forge - - main - - defaults -dependencies: - - python=3.7 - - pip - - ipykernel - # external requirements - - awscli - - python-dotenv - - pip: - # local package - - -e .. - diff --git a/{{ cookiecutter.repo_name }}/requirements/readme.md b/{{ cookiecutter.repo_name }}/requirements/readme.md deleted file mode 100644 index 6604301..0000000 --- a/{{ cookiecutter.repo_name }}/requirements/readme.md +++ /dev/null @@ -1,9 +0,0 @@ -This project has multiple ways of documenting requirements. Since it doesn't cost anything we include them all. - -- environment.yaml - This is the manual requirements, use it to install a new test or dev environment -- environment.min.yaml - This is the minimum requirements, use it to install a new test or dev environment -- environment.max.yaml - This pins all conda packages, use for production or finding vulnerabilities -- conda.requirements.txt - For people or bots not using conda - -To update the files run -`bash requiresments/doc_reqs.sh` diff --git a/{{ cookiecutter.repo_name }}/requirements/requirements.txt b/{{ cookiecutter.repo_name }}/requirements/requirements.txt deleted file mode 100644 index 29e674c..0000000 --- a/{{ cookiecutter.repo_name }}/requirements/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -# local package --e . - -# external requirements -click -Sphinx -coverage -awscli -flake8 -python-dotenv>=0.5.1 -{% if cookiecutter.python_interpreter != 'python3' %} - -# backwards compatibility -pathlib2 -{% endif %} diff --git a/{{ cookiecutter.repo_name }}/setup.py b/{{ cookiecutter.repo_name }}/setup.py deleted file mode 100644 index 73bce45..0000000 --- a/{{ cookiecutter.repo_name }}/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name='{{ cookiecutter.project_name.lower().replace(' ', '_') }}', - packages=find_packages(), - version='0.1.0', - description='{{ cookiecutter.description }}', - author='{{ cookiecutter.author_name }}', - license='proprietary', -) diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/justfile b/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/justfile new file mode 100644 index 0000000..b758d30 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/justfile @@ -0,0 +1,10 @@ +package := "{{cookiecutter.package_name}}" + +# put your run commands here +app: + echo "hello world" + +# black and isort +lint: + ruff . + diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/pyproject.toml b/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/pyproject.toml new file mode 100644 index 0000000..6655cd1 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/{{ cookiecutter.project_name.lower().replace(' ', '_') }}/pyproject.toml @@ -0,0 +1,37 @@ +[tool.poetry] +name = "{{ cookiecutter.project_name.lower().replace(' ', '_') }}" +version = "0.1.0" +description = "{{ cookiecutter.description }}" +authors = ["{{ cookiecutter.author_name }}"] +license = "MIT" +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.11,<3.13" +torch = {version = "^2.1.0+cu118", source = "pytorch"} +simple-parsing = "^0.1.4" +tqdm = "^4.66.1" +numpy = "^1.26.1" +pandas = "^2.1.1" +lightning = "^2.1.0" +matplotlib = "^3.8.0" +loguru = "^0.7.2" +einops = "^0.7.0" +scikit-learn = "^1.3.1" +pytorch-optimizer = "^2.12.0" +torchinfo = "^1.8.0" + +[[tool.poetry.source]] +name = "pytorch" +url = "https://download.pytorch.org/whl/cu118" +priority = "explicit" + + +[tool.poetry.group.dev.dependencies] +ipykernel = "^6.25.2" +black = "^23.10.0" +pylama = "^8.4.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api"