diff --git a/Makefile b/Makefile index 8af5c6a..765b1f6 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ stop: docker-compose -f ./docker/docker-compose.yml down init: - pipenv --three && pipenv install + pipenv --three && pipenv install --dev env: pipenv shell diff --git a/Pipfile b/Pipfile index f5ac40f..83ab5a7 100644 --- a/Pipfile +++ b/Pipfile @@ -12,7 +12,9 @@ jupyter = "*" pandas-datareader = "*" pandas-market-calendars = "*" boto3 = "*" +tenacity = "*" [dev-packages] yapf = "*" flake8 = "*" +pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 9a27b1d..7a13145 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "cd6cfed74640668b74dcbedc2cdf4426f24c7e71ab3351bfc4c331558994685c" + "sha256": "dde30de5709e602b5bcc9b58767bedc79c6fcb8013a8340064b0b44bcd20eece" }, "pipfile-spec": 6, "requires": {}, @@ -16,11 +16,11 @@ "default": { "altair": { "hashes": [ - "sha256:63934563a7a7b7186335858206a0b9be6043163b8b54a26cd3b3299a9e5e391f", - "sha256:65e243afa6da5b746c411890fd7dfd0f187c0f8e581cf3c34b07339712cf6627" + "sha256:77bf6baee04135cc37a20edd07820163336e2d2bb240804473e80dbefcb4bffd", + "sha256:f711ca44cc569eaec5ae8eafa610146c18b23a0ba4f7b9900646b28c42c13b9f" ], "index": "pypi", - "version": "==3.0.1" + "version": "==3.1.0" }, "appnope": { "hashes": [ @@ -46,12 +46,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", - "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", - "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", + "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", + "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469" ], "index": "pypi", - "version": "==4.7.1" + "version": "==4.8.0" }, "bleach": { "hashes": [ @@ -62,25 +62,25 @@ }, "boto3": { "hashes": [ - "sha256:a653dd7e155f0fb7d950cef7e2be7fa12b7787f4c8835f324abcfd1dbe4bb73d", - "sha256:f2054b788e19a2d91a3cc76814ca4e44050da7c89781a50924a132acf24fad56" + "sha256:0fec671879bbddffb0e088d970051f159effa0b7ad2a04541121372c719efd75", + "sha256:53c2b5f00990435f5c2a3f3ce1f9543246d8300c6444b4d7da8ffd69063c4182" ], "index": "pypi", - "version": "==1.9.147" + "version": "==1.9.200" }, "botocore": { "hashes": [ - "sha256:57d351535ffe0f19a80cb408df4dfc43fe66815cad3320e6a02d2261b498a8f6", - "sha256:e49db28173955bd3674b3280911446e6bd4458a9c40abfec46bf7c378fd30b2b" + "sha256:9448deffdb58e852935eeaae90682b7a0b4ef3c45eb6f17fa4444873f45f9ae6", + "sha256:d2ead76e77e7d0cd4c6bf9445e0ff54e279aee3c727eb30255b4ec327f956d22" ], - "version": "==1.12.147" + "version": "==1.12.200" }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "chardet": { "hashes": [ @@ -127,18 +127,18 @@ }, "ipykernel": { "hashes": [ - "sha256:0aeb7ec277ac42cc2b59ae3d08b10909b2ec161dc6908096210527162b53675d", - "sha256:0fc0bf97920d454102168ec2008620066878848fcfca06c22b669696212e292f" + "sha256:346189536b88859937b5f4848a6fd85d1ad0729f01724a411de5cae9b618819c", + "sha256:f0e962052718068ad3b1d8bcc703794660858f58803c3798628817f492a8769c" ], - "version": "==5.1.0" + "version": "==5.1.1" }, "ipython": { "hashes": [ - "sha256:54c5a8aa1eadd269ac210b96923688ccf01ebb2d0f21c18c3c717909583579a8", - "sha256:e840810029224b56cd0d9e7719dc3b39cf84d577f8ac686547c8ba7a06eeab26" + "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", + "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" ], "markers": "python_version >= '3.3'", - "version": "==7.5.0" + "version": "==7.7.0" }, "ipython-genutils": { "hashes": [ @@ -149,17 +149,17 @@ }, "ipywidgets": { "hashes": [ - "sha256:0f2b5cde9f272cb49d52f3f0889fdd1a7ae1e74f37b48dac35a83152780d2b7b", - "sha256:a3e224f430163f767047ab9a042fc55adbcab0c24bbe6cf9f306c4f89fdf0ba3" + "sha256:13ffeca438e0c0f91ae583dc22f50379b9d6b28390ac7be8b757140e9a771516", + "sha256:e945f6e02854a74994c596d9db83444a1850c01648f1574adf144fbbabe05c97" ], - "version": "==7.4.2" + "version": "==7.5.1" }, "jedi": { "hashes": [ - "sha256:2bb0603e3506f708e792c7f4ad8fc2a7a9d9c2d292a358fbbd58da531695595b", - "sha256:2c6bcd9545c7d6440951b12b44d373479bf18123a401a52025cf98563fbd826c" + "sha256:53c850f1a7d3cfcd306cc513e2450a54bdf5cacd7604b74e42dd1f0758eaaf36", + "sha256:e07457174ef7cb2342ff94fa56484fe41cec7ef69b0059f01d3f812379cb6f7c" ], - "version": "==0.13.3" + "version": "==0.14.1" }, "jinja2": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", - "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "jupyter": { "hashes": [ @@ -193,10 +193,10 @@ }, "jupyter-client": { "hashes": [ - "sha256:b5f9cb06105c1d2d30719db5ffb3ea67da60919fb68deaefa583deccd8813551", - "sha256:c44411eb1463ed77548bc2d5ec0d744c9b81c4a542d9637c7a52824e2121b987" + "sha256:73a809a2964afa07adcc1521537fddb58c2ffbb7e84d53dc5901cf80480465b3", + "sha256:98e8af5edff5d24e4d31e73bc21043130ae9d955a91aa93fc0bc3b1d0f7b5880" ], - "version": "==5.2.4" + "version": "==5.3.1" }, "jupyter-console": { "hashes": [ @@ -207,41 +207,37 @@ }, "jupyter-core": { "hashes": [ - "sha256:927d713ffa616ea11972534411544589976b2493fc7e09ad946e010aa7eb9970", - "sha256:ba70754aa680300306c699790128f6fbd8c306ee5927976cbe48adacf240c0b7" + "sha256:2c6e7c1e9f2ac45b5c2ceea5730bc9008d92fe59d0725eac57b04c0edfba24f7", + "sha256:f4fa22d6cf25f34807c995f22d2923693575c70f02557bcbfbe59bd5ec8d8b84" ], - "version": "==4.4.0" + "version": "==4.5.0" }, "lxml": { "hashes": [ - "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5", - "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f", - "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b", - "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616", - "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b", - "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294", - "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3", - "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90", - "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e", - "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314", - "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f", - "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d", - "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8", - "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3", - "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33", - "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317", - "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63", - "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd", - "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f", - "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a", - "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f", - "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22", - "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d", - "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b", - "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f", - "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86" + "sha256:06e5599b9c54f797a3c0f384c67705a0d621031007aa2400a6c7d17300fdb995", + "sha256:092237cfe4ece074401b75001a2e525fa6e1fb9d40fee8b7b132b1947d3bd2f8", + "sha256:0b6d49d0a26fe8207df8dd27c40b75be4deb2277173903aa76ec3e82df77cbe7", + "sha256:0f77061c20b4f32b1cf39e8f661c74e966344084c996e7b23c3a94e472461df0", + "sha256:0fef86edfa2f146b4b0ae2c6c05c3e4a8f3388b3655eafbc4aab3247f4dabb24", + "sha256:2f163c8844db4ed06a230ef092e2461ad01830972a896b8f3cf8b5bac70ae85d", + "sha256:350333190052bbfbc3222b1805b59b7979d7276e57af2257367e15a2db27082d", + "sha256:3b57dc5ed7b6a7d852c961f2389ca99404c2b59fd2088baec6fbaca02f688be4", + "sha256:3e86e5df4a8edd6f725f3c76f1d45e046d4f3aa40478092e4f5f373ad1f526e2", + "sha256:43dac60d10341d3e56be089cd0798b70e70d45ce32279f4c3190d8cbd71350e4", + "sha256:4665ee84ac8ba11d58f1ed517e29ea8536b4ae4e0c6fb6c7d3dce70abcd279f0", + "sha256:5033cf606a7cb559db967689b1b2e743994000f783607ba4c484e90917395ad7", + "sha256:75d731af05bf40f808d7716e0d26b4b02913402f861c032ce8c36efca350ae72", + "sha256:7720174604c7647e357566ac9e4d135c137caed5e7b01223551a4c81c8dc8b9a", + "sha256:b33ec641309bcea40c76c1b105f988e4e8f9a2f1ee1486aa5c0eeef33956c9bb", + "sha256:d1135dc0ac197242028ede085b693ba1f2bff7f0f9b91080e2540348312bfa53", + "sha256:d5a61e9c2322b45f259909a02b76bc98c4641214e22a37191d00c151aa9cdb9a", + "sha256:da22c4b17bc17dad9c8faf6d94c8fe568ac71c867a56631ab874da418fc7f8f7", + "sha256:da5c48ec9f8d8b5df42d328b6d1fb8d9413cd664a2367ef4f6f7cc48ee5b82c0", + "sha256:db2794bad21b7b30b6849b4e1537171cae8a7087711d958d69c233470dc612e7", + "sha256:f1c2f67df727034f94ccb590142d1d110f3dd38f638a4f1567fdd9f39892ba05", + "sha256:f840dddded8b046edc774c88ed8d2442cdb231a68894c42c74e3a809450fae76" ], - "version": "==4.3.3" + "version": "==4.4.0" }, "markupsafe": { "hashes": [ @@ -299,72 +295,59 @@ }, "notebook": { "hashes": [ - "sha256:573e0ae650c5d76b18b6e564ba6d21bf321d00847de1d215b418acb64f056eb8", - "sha256:f64fa6624d2323fbef6210a621817d6505a45d0d4a9367f1843b20a38a4666ee" + "sha256:0be97e939cec73cde37fc4d2a606a6f497a9addf3afcf61a09a21b0c35e699c5", + "sha256:5c16dbf4fa824db19de43637ebfb24bcbd3b4f646e5d6a0414ed3a376d6bc951" ], - "version": "==5.7.8" + "version": "==6.0.0" }, "numpy": { "hashes": [ - "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c", - "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb", - "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40", - "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340", - "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441", - "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634", - "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84", - "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2", - "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc", - "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686", - "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15", - "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621", - "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf", - "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0", - "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900", - "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df", - "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d", - "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897", - "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e", - "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26", - "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb", - "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad", - "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e" + "sha256:03e311b0a4c9f5755da7d52161280c6a78406c7be5c5cc7facfbcebb641efb7e", + "sha256:0cdd229a53d2720d21175012ab0599665f8c9588b3b8ffa6095dd7b90f0691dd", + "sha256:312bb18e95218bedc3563f26fcc9c1c6bfaaf9d453d15942c0839acdd7e4c473", + "sha256:464b1c48baf49e8505b1bb754c47a013d2c305c5b14269b5c85ea0625b6a988a", + "sha256:5adfde7bd3ee4864536e230bcab1c673f866736698724d5d28c11a4d63672658", + "sha256:7724e9e31ee72389d522b88c0d4201f24edc34277999701ccd4a5392e7d8af61", + "sha256:8d36f7c53ae741e23f54793ffefb2912340b800476eb0a831c6eb602e204c5c4", + "sha256:910d2272403c2ea8a52d9159827dc9f7c27fb4b263749dca884e2e4a8af3b302", + "sha256:951fefe2fb73f84c620bec4e001e80a80ddaa1b84dce244ded7f1e0cbe0ed34a", + "sha256:9588c6b4157f493edeb9378788dcd02cb9e6a6aeaa518b511a1c79d06cbd8094", + "sha256:9ce8300950f2f1d29d0e49c28ebfff0d2f1e2a7444830fbb0b913c7c08f31511", + "sha256:be39cca66cc6806652da97103605c7b65ee4442c638f04ff064a7efd9a81d50a", + "sha256:c3ab2d835b95ccb59d11dfcd56eb0480daea57cdf95d686d22eff35584bc4554", + "sha256:eb0fc4a492cb896346c9e2c7a22eae3e766d407df3eb20f4ce027f23f76e4c54", + "sha256:ec0c56eae6cee6299f41e780a0280318a93db519bbb2906103c43f3e2be1206c", + "sha256:f4e4612de60a4f1c4d06c8c2857cdcb2b8b5289189a12053f37d3f41f06c60d0" ], - "version": "==1.16.3" + "version": "==1.17.0" }, "pandas": { "hashes": [ - "sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b", - "sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa", - "sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846", - "sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822", - "sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167", - "sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794", - "sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204", - "sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2", - "sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2", - "sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248", - "sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8", - "sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8", - "sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296", - "sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5", - "sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d", - "sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5", - "sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0", - "sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3", - "sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb", - "sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1" + "sha256:074a032f99bb55d178b93bd98999c971542f19317829af08c99504febd9e9b8b", + "sha256:20f1728182b49575c2f6f681b3e2af5fac9e84abdf29488e76d569a7969b362e", + "sha256:2745ba6e16c34d13d765c3657bb64fa20a0e2daf503e6216a36ed61770066179", + "sha256:32c44e5b628c48ba17703f734d59f369d4cdcb4239ef26047d6c8a8bfda29a6b", + "sha256:3b9f7dcee6744d9dcdd53bce19b91d20b4311bf904303fa00ef58e7df398e901", + "sha256:544f2033250980fb6f069ce4a960e5f64d99b8165d01dc39afd0b244eeeef7d7", + "sha256:58f9ef68975b9f00ba96755d5702afdf039dea9acef6a0cfd8ddcde32918a79c", + "sha256:9023972a92073a495eba1380824b197ad1737550fe1c4ef8322e65fe58662888", + "sha256:914341ad2d5b1ea522798efa4016430b66107d05781dbfe7cf05eba8f37df995", + "sha256:9d151bfb0e751e2c987f931c57792871c8d7ff292bcdfcaa7233012c367940ee", + "sha256:b932b127da810fef57d427260dde1ad54542c136c44b227a1e367551bb1a684b", + "sha256:cfb862aa37f4dd5be0730731fdb8185ac935aba8b51bf3bd035658111c9ee1c9", + "sha256:de7ecb4b120e98b91e8a2a21f186571266a8d1faa31d92421e979c7ca67d8e5c", + "sha256:df7e1933a0b83920769611c5d6b9a1bf301e3fa6a544641c6678c67621fe9843" ], "index": "pypi", - "version": "==0.24.2" + "version": "==0.25.0" }, "pandas-datareader": { "hashes": [ - "sha256:6a5ad8c9ca27af148d06ac8eb526914cc12d04ae1d93af423d173279e2226c46", - "sha256:7dee3fe6fa483c8c2ee4f1af91a65b542c5446d75a6fc25c832cad1ffca8ef0b" + "sha256:2942644ac14386b08661b50581790d2727d11fd67301f225da8b2f92a634fadf", + "sha256:898945ee91e5efbb765e9449a8f88abac0733220d7b27d5109da4f51fcff4955" ], "index": "pypi", - "version": "==0.7.0" + "version": "==0.7.4" }, "pandas-market-calendars": { "hashes": [ @@ -382,10 +365,10 @@ }, "parso": { "hashes": [ - "sha256:17cc2d7a945eb42c3569d4564cdf49bde221bc2b552af3eca9c1aad517dcdd33", - "sha256:2e9574cb12e7112a87253e14e2c380ce312060269d04bd018478a3c92ea9a376" + "sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc", + "sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c" ], - "version": "==0.4.0" + "version": "==0.5.1" }, "pexpect": { "hashes": [ @@ -404,9 +387,9 @@ }, "prometheus-client": { "hashes": [ - "sha256:1b38b958750f66f208bcd9ab92a633c0c994d8859c831f7abc1f46724fcee490" + "sha256:71cd24a2b3eb335cb800c7159f423df1bd4dcd5171b234be15e3f31ec9f622da" ], - "version": "==0.6.0" + "version": "==0.7.1" }, "prompt-toolkit": { "hashes": [ @@ -426,16 +409,16 @@ }, "pygments": { "hashes": [ - "sha256:31cba6ffb739f099a85e243eff8cb717089fdd3c7300767d9fc34cb8e1b065f5", - "sha256:5ad302949b3c98dd73f8d9fcdc7e9cb592f120e32a18e23efd7f3dc51194472b" + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pyrsistent": { "hashes": [ - "sha256:16692ee739d42cf5e39cef8d27649a8c1fdb7aa99887098f1460057c5eb75c3a" + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" ], - "version": "==0.15.2" + "version": "==0.15.4" }, "python-dateutil": { "hashes": [ @@ -447,62 +430,62 @@ }, "pytz": { "hashes": [ - "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", - "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" ], - "version": "==2019.1" + "version": "==2019.2" }, "pyzmq": { "hashes": [ - "sha256:1651e52ed91f0736afd6d94ef9f3259b5534ce8beddb054f3d5ca989c4ef7c4f", - "sha256:5ccb9b3d4cd20c000a9b75689d5add8cd3bce67fcbd0f8ae1b59345247d803af", - "sha256:5e120c4cd3872e332fb35d255ad5998ebcee32ace4387b1b337416b6b90436c7", - "sha256:5e2a3707c69a7281a9957f83718815fd74698cba31f6d69f9ed359921f662221", - "sha256:63d51add9af8d0442dc90f916baf98fdc04e3b0a32afec4bfc83f8d85e72959f", - "sha256:65c5a0bdc49e20f7d6b03a661f71e2fda7a99c51270cafe71598146d09810d0d", - "sha256:66828fabe911aa545d919028441a585edb7c9c77969a5fea6722ef6e6ece38ab", - "sha256:7d79427e82d9dad6e9b47c0b3e7ae5f9d489b1601e3a36ea629bb49501a4daf3", - "sha256:824ee5d3078c4eae737ffc500fbf32f2b14e6ec89b26b435b7834febd70120cf", - "sha256:89dc0a83cccec19ff3c62c091e43e66e0183d1e6b4658c16ee4e659518131494", - "sha256:8b319805f6f7c907b101c864c3ca6cefc9db8ce0791356f180b1b644c7347e4c", - "sha256:90facfb379ab47f94b19519c1ecc8ec8d10813b69d9c163117944948bdec5d15", - "sha256:a0a178c7420021fc0730180a914a4b4b3092ce9696ceb8e72d0f60f8ce1655dd", - "sha256:a7a89591ae315baccb8072f216614b3e59aed7385aef4393a6c741783d6ee9cf", - "sha256:ba2578f0ae582452c02ed9fac2dc477b08e80ce05d2c0885becf5fff6651ccb0", - "sha256:c69b0055c55702f5b0b6b354133e8325b9a56dbc80e1be2d240bead253fb9825", - "sha256:ca434e1858fe222380221ddeb81e86f45522773344c9da63c311d17161df5e06", - "sha256:d4b8ecfc3d92f114f04d5c40f60a65e5196198b827503341521dda12d8b14939", - "sha256:d706025c47b09a54f005953ebe206f6d07a22516776faa4f509aaff681cc5468", - "sha256:d8f27e958f8a2c0c8ffd4d8855c3ce8ac3fa1e105f0491ce31729aa2b3229740", - "sha256:dbd264298f76b9060ce537008eb989317ca787c857e23cbd1b3ddf89f190a9b1", - "sha256:e926d66f0df8fdbf03ba20583af0f215e475c667fb033d45fd031c66c63e34c9", - "sha256:efc3bd48237f973a749f7312f68062f1b4ca5c2032a0673ca3ea8e46aa77187b", - "sha256:f59bc782228777cbfe04555707a9c56d269c787ed25d6d28ed9d0fbb41cb1ad2", - "sha256:f8da5322f4ff5f667a0d5a27e871b560c6637153c81e318b35cb012b2a98835c" + "sha256:00dd015159eaeb1c0731ad49310e1f5d839c9a35a15e4f3267f5052233fad99b", + "sha256:03913b6beb8e7b417b9910b0ee1fd5d62e9626d218faefbe879d70714ceab1a2", + "sha256:13f17386df81d5e6efb9a4faea341d8de22cdc82e49a326dded26e33f42a3112", + "sha256:16c6281d96885db1e15f7047ddc1a8f48ff4ea35d31ca709f4d2eb39f246d356", + "sha256:17efab4a804e31f58361631256d660214204046f9e2b962738b171b9ad674ea7", + "sha256:2b79919ddeff3d3c96aa6087c21d294c8db1c01f6bfeee73324944683685f419", + "sha256:2f832e4711657bb8d16ea1feba860f676ec5f14fb9fe3b449b5953a60e89edae", + "sha256:31a11d37ac73107363b47e14c94547dbfc6a550029c3fe0530be443199026fc2", + "sha256:33a3e928e6c3138c675e1d6702dd11f6b7050177d7aab3fc322db6e1d2274490", + "sha256:34a38195a6d3a9646cbcdaf8eb245b4d935c7a57f7e1b3af467814bc1a92467e", + "sha256:42900054f1500acef6df7428edf806abbf641bf92eb9ceded24aa863397c3bae", + "sha256:4ccc7f3c63aa9d744dadb62c49eda2d0e7de55649b80c45d7c684d70161a69af", + "sha256:5b220c37c346e6575db8c88a940c1fc234f99ce8e0068c408919bb8896c4b6d2", + "sha256:6074848da5c8b44a1ca40adf75cf65aa92bc80f635e8249aa8f37a69b2b9b6f5", + "sha256:61a4155964bd4a14ef95bf46cb1651bcf8dcbbed8c0108e9c974c1fcbb57788f", + "sha256:62b5774688326600c52f587f7a033ca6b6284bef4c8b1b5fda32480897759eac", + "sha256:65a9ffa4f9f085d696f16fd7541f34b3c357d25fe99c90e3bce2ea59c3b5b4b6", + "sha256:76a077d2c30f8adc5e919a55985a784b96aeca69b53c1ea6fd5723d3ae2e6f53", + "sha256:8e5b4c51557071d6379d6dc1f54f35e9f6a137f5e84e102efb869c8d3c13c8ff", + "sha256:917f73e07cc04f0678a96d93e7bb8b1adcccdde9ccfe202e622814f4d1d1ecfd", + "sha256:91c75d3c4c357f9643e739db9e79ab9681b2f6ae8ec5678d6ef2ea0d01532596", + "sha256:923dd91618b100bb4c92ab9ed7b65825a595b8524a094ce03c7cb2aaae7d353b", + "sha256:9849054e0355e2bc7f4668766a25517ba76095031c9ff5e39ae8949cee5bb024", + "sha256:c9d453933f0e3f44b9759189f2a18aa765f7f1a4345c727c18ebe8ad0d748d26", + "sha256:cb7514936277abce64c2f4c56883e5704d85ed04d98d2d432d1c6764003bb003" ], - "version": "==18.0.1" + "version": "==18.0.2" }, "qtconsole": { "hashes": [ - "sha256:a667558c7b1e1442a2e5bcef1686c55e096efd0b58d8b2a0a8415f4579991ee3", - "sha256:fdfc6002d9d2834c88f9c92e0f6f590284ff3740fa53016f188a62d58bcca6d8" + "sha256:6a85456af7a98b0f554d140922b7b6a219757b039adb2b95e847cf115eaa20ae", + "sha256:767eb9ec3f9943bc84270198b5ff95d2d86d68d6b57792fafa4df4fc6b16cd7c" ], - "version": "==4.4.4" + "version": "==4.5.2" }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], "index": "pypi", - "version": "==2.21.0" + "version": "==2.22.0" }, "s3transfer": { "hashes": [ - "sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e", - "sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021" + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" ], - "version": "==0.2.0" + "version": "==0.2.1" }, "send2trash": { "hashes": [ @@ -520,10 +503,18 @@ }, "soupsieve": { "hashes": [ - "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece", - "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca" + "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946", + "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de" ], - "version": "==1.9.1" + "version": "==1.9.2" + }, + "tenacity": { + "hashes": [ + "sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b", + "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705" + ], + "index": "pypi", + "version": "==5.0.4" }, "terminado": { "hashes": [ @@ -541,21 +532,21 @@ }, "toolz": { "hashes": [ - "sha256:929f0a7ea7f61c178bd951bdae93920515d3fbdbafc8e6caf82d752b9b3b31c9" + "sha256:08fdd5ef7c96480ad11c12d472de21acd32359996f69a5259299b540feba4560" ], - "version": "==0.9.0" + "version": "==0.10.0" }, "tornado": { "hashes": [ - "sha256:1174dcb84d08887b55defb2cda1986faeeea715fff189ef3dc44cce99f5fca6b", - "sha256:2613fab506bd2aedb3722c8c64c17f8f74f4070afed6eea17f20b2115e445aec", - "sha256:44b82bc1146a24e5b9853d04c142576b4e8fa7a92f2e30bc364a85d1f75c4de2", - "sha256:457fcbee4df737d2defc181b9073758d73f54a6cfc1f280533ff48831b39f4a8", - "sha256:49603e1a6e24104961497ad0c07c799aec1caac7400a6762b687e74c8206677d", - "sha256:8c2f40b99a8153893793559919a355d7b74649a11e59f411b0b0a1793e160bc0", - "sha256:e1d897889c3b5a829426b7d52828fb37b28bc181cd598624e65c8be40ee3f7fa" + "sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c", + "sha256:398e0d35e086ba38a0427c3b37f4337327231942e731edaa6e9fd1865bbd6f60", + "sha256:4e73ef678b1a859f0cb29e1d895526a20ea64b5ffd510a2307b5998c7df24281", + "sha256:559bce3d31484b665259f50cd94c5c28b961b09315ccd838f284687245f416e5", + "sha256:abbe53a39734ef4aba061fca54e30c6b4639d3e1f59653f0da37a0003de148c7", + "sha256:c845db36ba616912074c5b1ee897f8e0124df269468f25e4fe21fe72f6edd7a9", + "sha256:c9399267c926a4e7c418baa5cbe91c7d1cf362d505a1ef898fde44a07c9dd8a5" ], - "version": "==6.0.2" + "version": "==6.0.3" }, "traitlets": { "hashes": [ @@ -566,11 +557,11 @@ }, "urllib3": { "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], "markers": "python_version >= '3.4'", - "version": "==1.24.3" + "version": "==1.25.3" }, "wcwidth": { "hashes": [ @@ -588,19 +579,33 @@ }, "widgetsnbextension": { "hashes": [ - "sha256:14b2c65f9940c9a7d3b70adbe713dbd38b5ec69724eebaba034d1036cf3d4740", - "sha256:fa618be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265" + "sha256:079f87d87270bce047512400efd70238820751a11d2d8cb137a5a5bdbaf255c7", + "sha256:bd314f8ceb488571a5ffea6cc5b9fc6cba0adaf88a9d2386b93a489751938bcd" ], - "version": "==3.4.2" + "version": "==3.5.1" }, "wrapt": { "hashes": [ - "sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533" + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], - "version": "==1.11.1" + "version": "==1.11.2" } }, "develop": { + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -610,11 +615,18 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" + }, + "importlib-metadata": { + "hashes": [ + "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", + "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" + ], + "version": "==0.19" }, "mccabe": { "hashes": [ @@ -623,6 +635,34 @@ ], "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, + "packaging": { + "hashes": [ + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + ], + "version": "==19.1" + }, + "pluggy": { + "hashes": [ + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + ], + "version": "==0.12.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, "pycodestyle": { "hashes": [ "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", @@ -637,13 +677,49 @@ ], "version": "==2.1.1" }, - "yapf": { + "pyparsing": { "hashes": [ - "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", - "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + ], + "version": "==2.4.2" + }, + "pytest": { + "hashes": [ + "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d", + "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77" ], "index": "pypi", - "version": "==0.27.0" + "version": "==5.0.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "yapf": { + "hashes": [ + "sha256:02ace10a00fa2e36c7ebd1df2ead91dbfbd7989686dc4ccbdc549e95d19f5780", + "sha256:6f94b6a176a7c114cfa6bad86d40f259bbe0f10cf2fa7f2f4b3596fc5802a41b" + ], + "index": "pypi", + "version": "==0.28.0" + }, + "zipp": { + "hashes": [ + "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", + "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + ], + "version": "==0.5.2" } } } diff --git a/data_scraper/cboe.py b/data_scraper/cboe.py index b01fdf1..fe4611a 100644 --- a/data_scraper/cboe.py +++ b/data_scraper/cboe.py @@ -7,6 +7,7 @@ from itertools import groupby from bs4 import BeautifulSoup import requests import pandas as pd +import tenacity from . import utils, validation from .notifications import send_report @@ -19,8 +20,8 @@ url = "http://www.cboe.com/delayedquote/quote-table-download" def fetch_data(symbols=None): """Fetches options data for a given list of symbols""" symbols = symbols or _get_all_listed_symbols() - options = utils.get_module_config("cboe") - + valid_symbols = _get_all_listed_symbols() + # options = utils.get_module_config("cboe") try: form_data = _form_data() except requests.ConnectionError as ce: @@ -41,7 +42,9 @@ def fetch_data(symbols=None): for symbol in symbols: form_data["ctl00$ContentTop$C005$txtTicker"] = symbol + try: + assert symbol in valid_symbols response = requests.post(url, data=form_data, headers=headers, @@ -50,8 +53,12 @@ def fetch_data(symbols=None): cookies=response.cookies, headers=headers) symbol_data = symbol_req.text + if symbol_data == "" or symbol_data.startswith(" 0: - msg1 = "Successfully {}d {}".format(op, _symbol_str(done)) + msg_success = "👍 Successfully {}d {}".format(op, _symbol_str(done)) + else: + msg_success = "No symbols where successfully scraped 😭" if len(failed) > 0: - msg2 = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)), - ", ".join(failed)) - - msg= msg1 + " and " + msg2 + msg_fail = "⚠️️ Failed to {} {}: {}".format(op, + _symbol_str(len(failed)), + ", ".join(failed)) + else: + msg_fail = "No symbols fail to scrape! 🤩" + + msg = msg_success + '\n' + msg_fail slack_notification(msg, scraper, status=Status.Warning) diff --git a/data_scraper/test/test_cboe.py b/data_scraper/test/test_cboe.py index 72b9c28..76b9da1 100644 --- a/data_scraper/test/test_cboe.py +++ b/data_scraper/test/test_cboe.py @@ -30,7 +30,7 @@ class TestCBOE(unittest.TestCase): if cls.save_data_path: os.environ["SAVE_DATA_PATH"] = cls.save_data_path - @patch("data_scraper.cboe.slack_notification", return_value=None) + @patch("data_scraper.cboe.send_report", return_value=None) def test_fetch_spy(self, mocked_notification): """Fetch todays SPY quote""" cboe.fetch_data(["SPY"]) @@ -48,14 +48,15 @@ class TestCBOE(unittest.TestCase): counts = spy_df["type"].value_counts() self.assertEqual(counts["put"] + counts["call"], len(spy_df)) - @patch("data_scraper.cboe.slack_notification", return_value=None) - def test_fetch_invalid_symbol(self, mocked_notification): + @patch("data_scraper.cboe.send_report", return_value=None) + def test_fetch_invalid_symbol(self, mocked_report): """Fetching invalid symbol should send notification""" cboe.fetch_data(["FOOBAR"]) - self.assertTrue(mocked_notification.called) + self.assertTrue(mocked_report.called) - @patch("data_scraper.cboe.url", new="http://www.aldkfjaskldfjsa.com") - @patch("data_scraper.cboe.slack_notification", return_value=None) + @patch("data_scraper.cboe.url", + new="http://non_existing_domain_expected_to_fail.com") + @patch("data_scraper.cboe.send_report", return_value=None) def test_no_connection(self, mocked_notification): """Raise ConnectionError and send notification when host is unreachable""" with self.assertRaises(ConnectionError): @@ -63,23 +64,32 @@ class TestCBOE(unittest.TestCase): self.assertTrue(mocked_notification.called) @patch("data_scraper.cboe.utils.remove_file", return_value=None) - @patch("data_scraper.cboe.slack_notification", return_value=None) - def test_data_aggregation(self, mocked_notification, mocked_remove): + @patch("data_scraper.cboe.send_report", return_value=None) + def test_data_aggregation(self, mocked_report, mocked_remove): """Test data aggregation happy path""" cboe.aggregate_monthly_data(["SPX"]) aggregate_file = os.path.join(TestCBOE.cboe_data_path, "SPX", "SPX_20190301_to_20190329.csv") self.addCleanup(TestCBOE.remove_files, os.path.dirname(aggregate_file)) self.assertTrue(mocked_remove.called) - self.assertFalse(mocked_notification.called) - + self.assertTrue(mocked_report.called) if self.assertTrue(os.path.exists(aggregate_file)): spx_df = pd.read_csv(TestCBOE.spx_data_path) aggregate_df = pd.read_csv(aggregate_file) self.assertTrue(spx_df.equals(aggregate_df)) + @patch("data_scraper.cboe.url", + new="http://non_existing_domain_expected_to_fail.com") + @patch("data_scraper.cboe.retry_failure", return_value=None) + def test_retry(self, mocked_retry): + """Raise ConnectionError and retry when host is unreachable""" + with self.assertRaises(ConnectionError): + cboe.fetch_data(["SPX"]) + self.assertTrue(mocked_retry.called) + self.assertTrue(mocked_retry.call_count == 10) + @patch("data_scraper.cboe.utils.remove_file", return_value=None) - @patch("data_scraper.cboe.slack_notification", return_value=None) + @patch("data_scraper.cboe.send_report", return_value=None) def test_aggregate_missing_days(self, mocked_notification, mocked_remove): """Data aggregation should send notification when there are missing days""" cboe.aggregate_monthly_data(["GOOG"]) @@ -87,7 +97,7 @@ class TestCBOE(unittest.TestCase): self.assertFalse(mocked_remove.called) @patch("data_scraper.cboe.utils.remove_file", return_value=None) - @patch("data_scraper.cboe.slack_notification", return_value=None) + @patch("data_scraper.cboe.send_report", return_value=None) def test_aggregate_invalid_symbol(self, mocked_notification, mocked_remove): """Data aggregation should fail and send notification on invalid symbol""" @@ -101,4 +111,4 @@ class TestCBOE(unittest.TestCase): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/data_scraper/test/test_tiingo.py b/data_scraper/test/test_tiingo.py index 34e1bc4..e9e974e 100644 --- a/data_scraper/test/test_tiingo.py +++ b/data_scraper/test/test_tiingo.py @@ -3,7 +3,6 @@ import unittest from unittest.mock import patch import os import shutil - import pandas as pd from data_scraper import tiingo @@ -28,13 +27,12 @@ class TestTiingo(unittest.TestCase): def tearDownClass(cls): if cls.save_data_path: os.environ["SAVE_DATA_PATH"] = cls.save_data_path - - @patch("data_scraper.tiingo.slack_notification", return_value=None) + @patch("data_scraper.tiingo.send_report", return_value=None) def test_fetch_gld(self, mocked_notification): """Fetch GLD data""" tiingo.fetch_data(["GLD"]) gld_dir = os.path.join(TestTiingo.tiingo_data_path, "GLD") - self.addCleanup(TestTiingo.remove_files, gld_dir) + self.addCleanup(TestTiingo.remove_files, os.path.dirname(gld_dir)) if self.assertTrue(os.path.exists(gld_dir)): self.assertTrue(mocked_notification.called) @@ -50,21 +48,31 @@ class TestTiingo(unittest.TestCase): ] self.assertEqual(gld_df.columns, expected_columns) - @patch("data_scraper.tiingo.slack_notification", return_value=None) - def test_fetch_invalid_symbol(self, mocked_notification): + @patch("data_scraper.tiingo.send_report", return_value=None) + def test_fetch_invalid_symbol(self, mocked_report): """Fetching invalid symbol data should send notification""" tiingo.fetch_data(["FOOBAR"]) - self.assertTrue(mocked_notification.called) + self.assertTrue(mocked_report.called) @patch("data_scraper.tiingo.pdr.get_data_tiingo") # mock pandas_datareader - @patch("data_scraper.tiingo.slack_notification", return_value=None) - def test_no_connection(self, mocked_notification, mocked_pdr): + @patch("data_scraper.tiingo.send_report", return_value=None) + def test_no_connection(self, mocked_report, mocked_pdr): """Raise ConnectionError and send notification when host is unreachable""" mocked_pdr.side_effect = ConnectionError("This is a test") with self.assertRaises(ConnectionError): tiingo.fetch_data(["IBM"]) - self.assertTrue(mocked_notification.called) + self.assertTrue(mocked_report.called) + + @patch("data_scraper.tiingo.pdr.get_data_tiingo") # mock pandas_datareader + @patch("data_scraper.tiingo.retry_failure", return_value=None) + def test_retry(self, mocked_retry, mocked_pdr): + """Raise ConnectionError and retry when host is unreachable""" + mocked_pdr.side_effect = ConnectionError("This is a test") + with self.assertRaises(ConnectionError): + tiingo.fetch_data(["IBM"]) + self.assertTrue(mocked_retry.called) + self.assertTrue(mocked_retry.call_count == 10) def remove_files(file_path): if os.path.exists(file_path): diff --git a/data_scraper/tiingo.py b/data_scraper/tiingo.py index 677adc9..27e14fb 100644 --- a/data_scraper/tiingo.py +++ b/data_scraper/tiingo.py @@ -4,6 +4,7 @@ from datetime import date import pandas as pd import pandas_datareader as pdr +import tenacity from . import utils, validation from .notifications import send_report @@ -23,12 +24,39 @@ assets = [ def fetch_data(symbols=assets): """Fetches historical data for given symbols from Tiingo""" api_key = utils.get_environment_var("TIINGO_API_KEY") - symbols = [symbol.upper() for symbol in symbols] done = 0 failed = [] for symbol in symbols: + try: + symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key) + except ConnectionError as ce: + msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format( + symbol) + logger.error(msg, exc_info=True) + raise ce + except TypeError: + # pandas_datareader raises TypeError when fetching invalid symbol + msg = "Attempted to fetch invalid symbol {}".format(symbol) + logger.error(msg, exc_info=True) + except Exception: + msg = "Error fetching symbol {}".format(symbol) + logger.error(msg, exc_info=True) + else: + _save_data(symbol, symbol_data.reset_index()) + done += 1 + retry_failure(failed, done) + send_report(done, failed, __name__) + + +# if a symbol failes to scrape try again exponentialy +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(IOError)) +def retry_failure(failed, done): + api_key = utils.get_environment_var("TIINGO_API_KEY") + for symbol in failed: try: symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key) except ConnectionError as ce: @@ -45,10 +73,9 @@ def fetch_data(symbols=assets): msg = "Error fetching symbol {}".format(symbol) logger.error(msg, exc_info=True) else: - _save_data(symbol, symbol_data.reset_index()) + _save_data(symbol, symbol_data) done += 1 - - send_report(done, failed, __name__) + failed.remove(symbol) def _save_data(symbol, symbol_df): @@ -69,9 +96,9 @@ def _save_data(symbol, symbol_df): logger.debug("File %s already downloaded", file_path) else: expected_columns = [ - "symbol", "date", "adjClose", "adjHigh", "adjLow", "adjOpen", - "adjVolume", "close", "divCash", "high", "low", "open", - "splitFactor", "volume" + "symbol", "date", "close", "high", "low", "open", "volume", + "adjClose", "adjHigh", "adjLow", "adjOpen", "adjVolume", "divCash", + "splitFactor" ] if validation.validate_historical_dates( @@ -87,10 +114,8 @@ def _save_data(symbol, symbol_df): def _merge(symbol, symbol_df): """Merge `symbol_df` with previous data file.""" - save_data_path = utils.get_save_data_path() symbol_dir = os.path.join(save_data_path, "tiingo", symbol) - files = os.listdir(symbol_dir) if len(files) == 0: return symbol_df diff --git a/data_scraper/validation.py b/data_scraper/validation.py index 69aac38..6bfc92f 100644 --- a/data_scraper/validation.py +++ b/data_scraper/validation.py @@ -9,13 +9,11 @@ from .notifications import slack_notification logger = logging.getLogger(__name__) - def file_hash_matches_data(file_path, data): file_hash = file_md5(file_path) data_md5 = hashlib.md5(data.encode()).hexdigest() return file_hash == data_md5 - def file_md5(file, chunk_size=4096): md5 = hashlib.md5() with open(file, "rb") as f: @@ -24,7 +22,6 @@ def file_md5(file, chunk_size=4096): return md5.hexdigest() - def validate_dates_in_month(symbol, date_range): """Compares `date_range` (month) with NYSE trading calendar. Returns `True` if there are no missing days. @@ -41,13 +38,11 @@ def validate_dates_in_month(symbol, date_range): # Remove timezone info trading_days = trading_days.tz_convert(tz=None) missing_days = trading_days.difference(date_range) - if not missing_days.empty: logger.error("Error validating monthly dates. Missing: %s", missing_days) return missing_days.empty - def validate_historical_dates(symbol, date_range): """Compares `date_range` (any time range) with trading calendar. Returns `True` if there are no missing days. @@ -68,11 +63,9 @@ def validate_historical_dates(symbol, date_range): return missing_days.empty - def validate_columns(expected, received): """Verify that the `received` columns scraped are equal to `expected`""" valid = all(expected == received) - if not valid: expected_cols = ", ".join(expected) received_cols = ", ".join(received) @@ -81,13 +74,10 @@ def validate_columns(expected, received): Received: {}""".format(expected_cols, received_cols) logger.error(msg) slack_notification(msg, __name__) - return valid - def validate_aggregate_file(aggregate_file, daily_files): """Compares `aggregate_file` with the data from `daily_files`.""" aggregate_df = pd.read_csv(aggregate_file) recreated_df = cboe.concatenate_files(daily_files) - - return aggregate_df.equals(recreated_df) + return aggregate_df.equals(recreated_df) \ No newline at end of file