diff --git a/cookiecutter.json b/cookiecutter.json index 00b7d55..62d9e30 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -5,7 +5,7 @@ "author_name": "Your name (or your organization/company/team)", "description": "A short description of the project.", "python_version_number": "3.7", - "dataset_storage": [ + "data_storage": [ {"none": {}}, {"azure": {"container": "container-name"}}, {"s3": {"bucket": "bucket-name", "aws_profile": "default"}}, diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index a392875..6b92434 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -13,7 +13,7 @@ pip_only_packages = [ 'python-dotenv', ] -{% if cookiecutter.dataset_storage.s3 %} +{% if cookiecutter.data_storage.s3 %} packages += ['awscli'] {% endif %} diff --git a/tests/conftest.py b/tests/conftest.py index 8acbfb2..cf0191a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,26 @@ -import sys -import pytest -import shutil +import json from pathlib import Path -from cookiecutter import main +import pytest +import sys +import shutil + +from click.testing import CliRunner + +from ccds.__main__ import main + + +import yaml CCDS_ROOT = Path(__file__).parents[1].resolve() -args = { +args = {'default_context': { 'project_name': 'DrivenData', 'author_name': 'DrivenData', 'open_source_license': 'BSD-3-Clause', - 'python_interpreter': 'python' + 'description' : 'Test project', + 'data_storage': {'s3': {'bucket': 'test-bucket', 'aws_profile': 'default'}} } +} def system_check(basename): @@ -24,16 +33,43 @@ def system_check(basename): @pytest.fixture(scope='class', params=[{}, args]) def default_baked_project(tmpdir_factory, request): temp = tmpdir_factory.mktemp('data-project') + config_dir = tmpdir_factory.mktemp('config') + out_dir = Path(temp).resolve() pytest.param = request.param - main.cookiecutter( - str(CCDS_ROOT), - no_input=True, - extra_context=pytest.param, - output_dir=out_dir + + config_path = Path(config_dir) / 'config.yml' + with open(config_path, 'w') as f: + yaml.dump(pytest.param, f) + + runner = CliRunner() + result = runner.invoke( + main, + ['--no-input', + '-o', str(out_dir), + str(CCDS_ROOT)], + catch_exceptions=False, ) + # import pdb; pdb.set_trace() + + # assert result.output == "" + assert result.exit_code == 0 + + # main( + # str(CCDS_ROOT), + # pytest.param, + # True, + # None, + # False, + # False, + # out_dir, + # None, + # True, + # None + # ) + pn = pytest.param.get('project_name') or 'project_name' # project name gets converted to lower case on Linux but not Mac @@ -44,4 +80,6 @@ def default_baked_project(tmpdir_factory, request): yield # cleanup after - shutil.rmtree(out_dir) \ No newline at end of file + print("=======> ", out_dir) + # shutil.rmtree(out_dir) + # shutil.rmtree(config_dir) diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index f1ff948..fe0369b 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -31,29 +31,29 @@ clean: lint: flake8 {{ cookiecutter.module_name }} -{% if cookiecutter.dataset_storage.s3 %} +{% if cookiecutter.data_storage.s3 %} ## Download Data from storage system sync_data_down: - {% if cookiecutter.dataset_storage.s3 %} - aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/\ - data/ {% if cookiecutter.dataset_storage.s3.aws_profile != 'default' %} --profile {{ cookiecutter.dataset_storage.s3.aws_profile }}{% endif %} - {% elif cookiecutter.dataset_storage.azure %} - az storage blob download-batch -s {{ cookiecutter.dataset_storage.azure.container }}/data/ \ + {% if cookiecutter.data_storage.s3 %} + aws s3 sync s3://{{ cookiecutter.data_storage.s3.bucket }}/data/\ + data/ {% if cookiecutter.data_storage.s3.aws_profile != 'default' %} --profile {{ cookiecutter.data_storage.s3.aws_profile }}{% endif %} + {% elif cookiecutter.data_storage.azure %} + az storage blob download-batch -s {{ cookiecutter.data_storage.azure.container }}/data/ \ -d data/ - {% elif cookiecutter.dataset_storage.gcs %} - gsutil cp {{ cookiecutter.dataset_storage.gcs.bucket }}/data/ data/ + {% elif cookiecutter.data_storage.gcs %} + gsutil cp {{ cookiecutter.data_storage.gcs.bucket }}/data/ data/ {% endif %} ## Upload Data to storage system sync_data_up: - {% if cookiecutter.dataset_storage.s3 %} - aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/ data/\ - {% if cookiecutter.dataset_storage.s3.aws_profile %} --profile $(PROFILE){% endif %} - {% elif cookiecutter.dataset_storage.azure %} - az storage blob upload-batch -d {{ cookiecutter.dataset_storage.azure.container }}/data/ \ + {% if cookiecutter.data_storage.s3 %} + aws s3 sync s3://{{ cookiecutter.data_storage.s3.bucket }}/data/ data/\ + {% if cookiecutter.data_storage.s3.aws_profile %} --profile $(PROFILE){% endif %} + {% elif cookiecutter.data_storage.azure %} + az storage blob upload-batch -d {{ cookiecutter.data_storage.azure.container }}/data/ \ -s data/ - {% elif cookiecutter.dataset_storage.gcs %} - gsutil cp data/ {{ cookiecutter.dataset_storage.gcs.bucket }}/data/ + {% elif cookiecutter.data_storage.gcs %} + gsutil cp data/ {{ cookiecutter.data_storage.gcs.bucket }}/data/ {% endif %} {% endif %}