diff --git a/Makefile b/Makefile index 765b1f6..d58ec56 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ stop: docker-compose -f ./docker/docker-compose.yml down init: - pipenv --three && pipenv install --dev + pipenv --three && pipenv install --dev env: pipenv shell diff --git a/data_scraper/notifications.py b/data_scraper/notifications.py index 3e9d3c2..16a380e 100644 --- a/data_scraper/notifications.py +++ b/data_scraper/notifications.py @@ -62,7 +62,7 @@ def send_report(done, failed, scraper, op="scrape"): _symbol_str(len(failed)), ", ".join(failed)) else: - msg_fail = "No symbols fail to scrape! 🤩" + msg_fail = "No symbols failed to scrape! 🤩" msg = msg_success + '\n' + msg_fail slack_notification(msg, scraper, status=Status.Warning) diff --git a/data_scraper/validation.py b/data_scraper/validation.py index 6bfc92f..06b7b37 100644 --- a/data_scraper/validation.py +++ b/data_scraper/validation.py @@ -9,11 +9,13 @@ from .notifications import slack_notification logger = logging.getLogger(__name__) + def file_hash_matches_data(file_path, data): file_hash = file_md5(file_path) data_md5 = hashlib.md5(data.encode()).hexdigest() return file_hash == data_md5 + def file_md5(file, chunk_size=4096): md5 = hashlib.md5() with open(file, "rb") as f: @@ -22,6 +24,7 @@ def file_md5(file, chunk_size=4096): return md5.hexdigest() + def validate_dates_in_month(symbol, date_range): """Compares `date_range` (month) with NYSE trading calendar. Returns `True` if there are no missing days. @@ -43,6 +46,7 @@ def validate_dates_in_month(symbol, date_range): missing_days) return missing_days.empty + def validate_historical_dates(symbol, date_range): """Compares `date_range` (any time range) with trading calendar. Returns `True` if there are no missing days. @@ -63,9 +67,12 @@ def validate_historical_dates(symbol, date_range): return missing_days.empty + def validate_columns(expected, received): """Verify that the `received` columns scraped are equal to `expected`""" - valid = all(expected == received) + expected = sorted(expected) + received = sorted(received) + valid = expected == received if not valid: expected_cols = ", ".join(expected) received_cols = ", ".join(received) @@ -76,6 +83,7 @@ def validate_columns(expected, received): slack_notification(msg, __name__) return valid + def validate_aggregate_file(aggregate_file, daily_files): """Compares `aggregate_file` with the data from `daily_files`.""" aggregate_df = pd.read_csv(aggregate_file)