From bdbb90c6a4ce50a8e54d4be5832ee71aaecbd426 Mon Sep 17 00:00:00 2001 From: David Marx Date: Mon, 6 Aug 2018 03:07:04 -0700 Subject: [PATCH] Fixed _impose_rate_limit behavior to apply backoff to the met request (asking what the ratelimit is) issued in __init__. --- psaw/PushshiftAPI.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 861defb..19ddb0b 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -4,6 +4,7 @@ import json import requests import time from datetime import datetime as dt +import warnings class RateLimitCache(object): def __init__(self, n, t=60): @@ -117,11 +118,10 @@ class PushshiftAPIMinimal(object): return thing def _impose_rate_limit(self, nth_request=0): - if not hasattr(self, '_rlcache'): - return interval = 0 - if self._rlcache.blocked: - interval = self._rlcache.interval + if hasattr(self, '_rlcache'): + if self._rlcache.blocked: + interval = self._rlcache.interval interval = max(interval, self.backoff*nth_request) interval = min(interval, self.max_sleep) if interval > 0: @@ -149,10 +149,17 @@ class PushshiftAPIMinimal(object): def _get(self, url, payload={}): i, success = 0, False while (not success) and (i 0: + warnings.warn("Unable to connect to pushshift.io. Retrying after backoff.") self._impose_rate_limit(i) - response = requests.get(url, params=payload) - success = response.status_code == 200 i+=1 + try: + response = requests.get(url, params=payload) + except requests.ConnectionError: + continue + success = response.status_code == 200 + if not success: + raise Exception("Unable to connect to pushshift.io. Max retries exceeded.") return json.loads(response.text) def _handle_paging(self, url):