From 1b10c11801c3a298cf50d3ed6f44598c9497b0cd Mon Sep 17 00:00:00 2001 From: David Marx Date: Wed, 16 May 2018 00:40:11 -0700 Subject: [PATCH 1/8] Added get_submission_comment_ids, big refactor to accomodate --- psaw/PushshiftAPI.py | 76 +++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 91f60cc..aca4827 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -41,8 +41,9 @@ class RateLimitCache(object): self.cache.append(time.time()) class PushshiftAPIMinimal(object): - base_url = {'search':'https://api.pushshift.io/reddit/{}/search/', - 'meta':'https://api.pushshift.io/meta/'} + #base_url = {'search':'https://api.pushshift.io/reddit/{}/search/', + # 'meta':'https://api.pushshift.io/meta/'} + _base_url = 'https://{domain}.pushshift.io/{{endpoint}}' _limited_args = ('aggs') def __init__(self, max_retries=20, @@ -51,7 +52,8 @@ class PushshiftAPIMinimal(object): rate_limit_per_minute=None, max_results_per_request=500, detect_local_tz=True, - utc_offset_secs=None + utc_offset_secs=None, + domain='api' ): assert max_results_per_request <= 500 assert backoff >= 1 @@ -64,20 +66,30 @@ class PushshiftAPIMinimal(object): self._utc_offset_secs = utc_offset_secs self._detect_local_tz = detect_local_tz + self.domain = domain + if rate_limit_per_minute is None: - rate_limit_per_minute = self._get(self.base_url['meta'])['server_ratelimit_per_minute'] + response = self._get(self.base_url.format(endpoint='meta')) + rate_limit_per_minute = response['server_ratelimit_per_minute'] self._rlcache = RateLimitCache(n=rate_limit_per_minute, t=60) + @property + def base_url(self): + return self._base_url.format(domain=self.domain) + @property def utc_offset_secs(self): - if self._utc_offset_secs is None: - if self._detect_local_tz: - try: - self._utc_offset_secs = dt.utcnow().astimezone().utcoffset().total_seconds() - except ValueError: - self._utc_offset_secs = 0 - else: + if self._utc_offset_secs is not None: + return self._utc_offset_secs + + if self._detect_local_tz: + try: + self._utc_offset_secs = dt.utcnow().astimezone().utcoffset().total_seconds() + except ValueError: self._utc_offset_secs = 0 + else: + self._utc_offset_secs = 0 + return self._utc_offset_secs def _limited(self, payload): @@ -131,26 +143,31 @@ class PushshiftAPIMinimal(object): i+=1 return json.loads(response.text) - def _query(self, kind, stop_condition=lambda x: False, **kwargs): - limit = kwargs.get('limit', None) - payload = copy.deepcopy(kwargs) + def _handle_paging(self, url): + limit = self.payload.get('limit', None) n = 0 while True: if limit is not None: if limit > self.max_results_per_request: - payload['limit'] = self.max_results_per_request + self.payload['limit'] = self.max_results_per_request limit -= self.max_results_per_request else: payload['limit'] = limit limit = 0 - self._add_nec_args(payload) - url = self.base_url['search'].format(kind) - results = self._get(url, payload) - if self._limited(payload): - yield results + self._add_nec_args(self.payload) + + yield self._get(url, self.payload) + + if (limit is not None) & (limit == 0): return - results = results['data'] + def _search(self, kind, stop_condition=lambda x: False, **kwargs): + self.payload = copy.deepcopy(kwargs) + endpoint = 'reddit/{}/search'.format(kind) + url = self.base_url.format(endpoint=endpoint) + + for response in self._handle_paging(url): + results = response['data'] if len(results) == 0: return for thing in results: @@ -159,14 +176,21 @@ class PushshiftAPIMinimal(object): yield thing if stop_condition(thing): return - payload['before'] = thing.created_utc - if (limit is not None) & (limit == 0): - return + + # For paging. + self.payload['before'] = thing.created_utc + def search_submissions(self, **kwargs): - return self._query(kind='submission', **kwargs) + return self._search(kind='submission', **kwargs) def search_comments(self, **kwargs): - return self._query(kind='comment', **kwargs) + return self._search(kind='comment', **kwargs) + + def get_submission_comment_ids(self, submission_id, **kwargs): + self.payload = copy.deepcopy(kwargs) + endpoint = 'reddit/submission/comment_ids' + url = self.base_url.format(endpoint=endpoint) + class PushshiftAPI(PushshiftAPIMinimal): # Fill out this class with more user-friendly features later From b550f20e97a4ed0ae97c231639a2c1eb67ef54ea Mon Sep 17 00:00:00 2001 From: David Marx Date: Thu, 17 May 2018 23:05:40 -0700 Subject: [PATCH 2/8] Fixed some bugs introduced in last commit, added batch_return, started work on praw utility class --- psaw/PushshiftAPI.py | 48 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index aca4827..553d5b5 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -145,14 +145,14 @@ class PushshiftAPIMinimal(object): def _handle_paging(self, url): limit = self.payload.get('limit', None) - n = 0 + #n = 0 while True: if limit is not None: if limit > self.max_results_per_request: self.payload['limit'] = self.max_results_per_request limit -= self.max_results_per_request else: - payload['limit'] = limit + self.payload['limit'] = limit limit = 0 self._add_nec_args(self.payload) @@ -161,7 +161,11 @@ class PushshiftAPIMinimal(object): if (limit is not None) & (limit == 0): return - def _search(self, kind, stop_condition=lambda x: False, **kwargs): + def _search(self, + kind, + stop_condition=lambda x: False, + return_batch=False, + **kwargs): self.payload = copy.deepcopy(kwargs) endpoint = 'reddit/{}/search'.format(kind) url = self.base_url.format(endpoint=endpoint) @@ -170,15 +174,30 @@ class PushshiftAPIMinimal(object): results = response['data'] if len(results) == 0: return + if return_batch: + #print("initializing batch after request") + batch = [] + for thing in results: - n+=1 + #n+=1 thing = self._wrap_thing(thing, kind) - yield thing + + if return_batch: + #print("appending to batch") + batch.append(thing) + else: + yield thing + if stop_condition(thing): + if return_batch: + return batch return - # For paging. - self.payload['before'] = thing.created_utc + if return_batch: + yield batch + + # For paging. + self.payload['before'] = thing.created_utc def search_submissions(self, **kwargs): return self._search(kind='submission', **kwargs) @@ -195,3 +214,18 @@ class PushshiftAPIMinimal(object): class PushshiftAPI(PushshiftAPIMinimal): # Fill out this class with more user-friendly features later pass + +class WithPraw(PushshiftAPIMinimal): + def __init__(self, r, *args, **kwargs): + super().__init__(*args, **kwargs) + self.r = r + + def search_comments(self, **kwargs): + return self._praw_get(kind='comment', **kwargs) + + def search_submissions(self, **kwargs): + return self._praw_get(kind='submission', **kwargs) + + def _praw_get(self, **kwargs): + for c in self._search(**kwargs): + yield c From ded03e51222a599973b8088fed2a5669d65a6edc Mon Sep 17 00:00:00 2001 From: David Marx Date: Thu, 17 May 2018 23:37:08 -0700 Subject: [PATCH 3/8] Added support for getting ids from pushshift and pulling things from praw --- .gitignore | 1 + psaw/PushshiftAPI.py | 30 +++++++++++++++++++++++++----- psaw/__init__.py | 2 +- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 33c8692..62c8c72 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ _build/ build/ dist/ +praw.ini diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 553d5b5..5efcc37 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -40,11 +40,20 @@ class RateLimitCache(object): raise Exception("RateLimitCache is blocked.") self.cache.append(time.time()) + class PushshiftAPIMinimal(object): #base_url = {'search':'https://api.pushshift.io/reddit/{}/search/', # 'meta':'https://api.pushshift.io/meta/'} _base_url = 'https://{domain}.pushshift.io/{{endpoint}}' _limited_args = ('aggs') + _thing_prefix = { + 'Comment':'t1_', + 'Account':'t2_', + 'Link':'t3_', + 'Message':'t4_', + 'Subreddit':'t5_', + 'Award':'t6_' + } def __init__(self, max_retries=20, max_sleep=3600, @@ -221,11 +230,22 @@ class WithPraw(PushshiftAPIMinimal): self.r = r def search_comments(self, **kwargs): - return self._praw_get(kind='comment', **kwargs) + return self._praw_search(kind='comment', **kwargs) def search_submissions(self, **kwargs): - return self._praw_get(kind='submission', **kwargs) + return self._praw_search(kind='submission', **kwargs) - def _praw_get(self, **kwargs): - for c in self._search(**kwargs): - yield c + def _praw_search(self, **kwargs): + prefix = self._thing_prefix[kwargs['kind'].title()] + client_return_batch = kwargs.get('return_batch') + if client_return_batch is False: + kwargs.pop('return_batch') + + for batch in self._search(return_batch=True, **kwargs): + fullnames = [prefix + thing.id for thing in batch] + praw_batch = self.r.info(fullnames=fullnames) + if client_return_batch: + yield praw_batch + else: + for praw_thing in praw_batch: + yield praw_thing diff --git a/psaw/__init__.py b/psaw/__init__.py index 36508c8..f9bd879 100644 --- a/psaw/__init__.py +++ b/psaw/__init__.py @@ -4,6 +4,6 @@ Pushshift.io API Wrapper (for reddit.com public comment/submission search) https://github.com/dmarx/psaw """ -from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal +from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal, WithPraw __version__ = '0.0.2' From 80651b11afa4f0dca30ffaf3ea02dc4f53f476b1 Mon Sep 17 00:00:00 2001 From: David Marx Date: Thu, 17 May 2018 23:43:34 -0700 Subject: [PATCH 4/8] Moved praw functionality into PushshiftAPI class --- psaw/PushshiftAPI.py | 19 ++++++++++++------- psaw/__init__.py | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 5efcc37..008af38 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -220,20 +220,23 @@ class PushshiftAPIMinimal(object): url = self.base_url.format(endpoint=endpoint) -class PushshiftAPI(PushshiftAPIMinimal): +#class PushshiftAPI(PushshiftAPIMinimal): # Fill out this class with more user-friendly features later - pass +# pass -class WithPraw(PushshiftAPIMinimal): - def __init__(self, r, *args, **kwargs): +class PushshiftAPI(PushshiftAPIMinimal): + def __init__(self, r=None, *args, **kwargs): super().__init__(*args, **kwargs) self.r = r + self._search_func = self._search + if r is not None: + self._search_func = self._praw_search def search_comments(self, **kwargs): - return self._praw_search(kind='comment', **kwargs) + return self._search_func(kind='comment', **kwargs) def search_submissions(self, **kwargs): - return self._praw_search(kind='submission', **kwargs) + return self._search_func(kind='submission', **kwargs) def _praw_search(self, **kwargs): prefix = self._thing_prefix[kwargs['kind'].title()] @@ -241,7 +244,9 @@ class WithPraw(PushshiftAPIMinimal): if client_return_batch is False: kwargs.pop('return_batch') - for batch in self._search(return_batch=True, **kwargs): + if 'filter' in kwargs: + kwargs.pop('filter') + for batch in self._search(return_batch=True, filter='id', **kwargs): fullnames = [prefix + thing.id for thing in batch] praw_batch = self.r.info(fullnames=fullnames) if client_return_batch: diff --git a/psaw/__init__.py b/psaw/__init__.py index f9bd879..36508c8 100644 --- a/psaw/__init__.py +++ b/psaw/__init__.py @@ -4,6 +4,6 @@ Pushshift.io API Wrapper (for reddit.com public comment/submission search) https://github.com/dmarx/psaw """ -from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal, WithPraw +from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal __version__ = '0.0.2' From 9bbec1d7001691fa328b87e2a2c37139da502bdb Mon Sep 17 00:00:00 2001 From: David Marx Date: Fri, 18 May 2018 00:11:02 -0700 Subject: [PATCH 5/8] Added support for /reddit/submission/comment_ids/ endpoint --- psaw/PushshiftAPI.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 008af38..ef25d8b 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -143,7 +143,7 @@ class PushshiftAPIMinimal(object): if 'created_utc' not in payload['filter']: payload['filter'].append('created_utc') - def _get(self, url, payload={}, endpoint='search'): + def _get(self, url, payload={}): i, success = 0, False while (not success) and (i Date: Fri, 18 May 2018 00:43:16 -0700 Subject: [PATCH 6/8] using submission/comment_ids under the hood in _praw_search when submission_id provided. Need to clean up debugging print statements --- psaw/PushshiftAPI.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index ef25d8b..d639525 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -219,9 +219,11 @@ class PushshiftAPI(PushshiftAPIMinimal): self.r = r self._search_func = self._search if r is not None: + print("using praw") self._search_func = self._praw_search def search_comments(self, **kwargs): + print(kwargs.keys()) return self._search_func(kind='comment', **kwargs) def search_submissions(self, **kwargs): @@ -232,24 +234,39 @@ class PushshiftAPI(PushshiftAPIMinimal): endpoint = 'reddit/submission/comment_ids/{}'.format(submission_id) url = self.base_url.format(endpoint=endpoint) print(url) - return self._get(url, self.payload) + return self._get(url, self.payload)['data'] def _praw_search(self, **kwargs): + print(kwargs.keys()) prefix = self._thing_prefix[kwargs['kind'].title()] + + self.payload = copy.deepcopy(kwargs) + client_return_batch = kwargs.get('return_batch') if client_return_batch is False: - kwargs.pop('return_batch') + self.payload.pop('return_batch') if 'filter' in kwargs: - kwargs.pop('filter') + self.payload.pop('filter') - gen = self._search(return_batch=True, filter='id', **kwargs) - if kwargs.get('kind') == 'comment' and kwargs.get('submission_id'): - gen = self._get_submission_comment_ids(**kwargs) + gen = self._search(return_batch=True, filter='id', **self.payload) + print(kwargs.get('kind'), kwargs.get('submission_id')) + using_gsci = False + if kwargs.get('kind') == 'comment' and self.payload.get('submission_id'): + print("using _get_submission_comment_ids") + using_gsci = True + gen = [self._get_submission_comment_ids(**kwargs)] + + print("gen constructed") for batch in gen: - fullnames = [prefix + thing.id for thing in batch] + print(type(batch[0])) + if using_gsci: + fullnames = [prefix + base36id for base36id in batch] + print(fullnames) + else: + fullnames = [prefix + c.id for c in batch] praw_batch = self.r.info(fullnames=fullnames) if client_return_batch: yield praw_batch From c012add0674c23df72f99890f8a9eca66714cf4f Mon Sep 17 00:00:00 2001 From: David Marx Date: Fri, 18 May 2018 00:46:07 -0700 Subject: [PATCH 7/8] Removed debugging code --- psaw/PushshiftAPI.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index d639525..2522535 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -184,15 +184,12 @@ class PushshiftAPIMinimal(object): if len(results) == 0: return if return_batch: - #print("initializing batch after request") batch = [] for thing in results: - #n+=1 thing = self._wrap_thing(thing, kind) if return_batch: - #print("appending to batch") batch.append(thing) else: yield thing @@ -219,11 +216,9 @@ class PushshiftAPI(PushshiftAPIMinimal): self.r = r self._search_func = self._search if r is not None: - print("using praw") self._search_func = self._praw_search def search_comments(self, **kwargs): - print(kwargs.keys()) return self._search_func(kind='comment', **kwargs) def search_submissions(self, **kwargs): @@ -233,11 +228,9 @@ class PushshiftAPI(PushshiftAPIMinimal): self.payload = copy.deepcopy(kwargs) endpoint = 'reddit/submission/comment_ids/{}'.format(submission_id) url = self.base_url.format(endpoint=endpoint) - print(url) return self._get(url, self.payload)['data'] def _praw_search(self, **kwargs): - print(kwargs.keys()) prefix = self._thing_prefix[kwargs['kind'].title()] self.payload = copy.deepcopy(kwargs) @@ -251,20 +244,14 @@ class PushshiftAPI(PushshiftAPIMinimal): gen = self._search(return_batch=True, filter='id', **self.payload) - print(kwargs.get('kind'), kwargs.get('submission_id')) using_gsci = False if kwargs.get('kind') == 'comment' and self.payload.get('submission_id'): - - print("using _get_submission_comment_ids") using_gsci = True gen = [self._get_submission_comment_ids(**kwargs)] - print("gen constructed") for batch in gen: - print(type(batch[0])) if using_gsci: fullnames = [prefix + base36id for base36id in batch] - print(fullnames) else: fullnames = [prefix + c.id for c in batch] praw_batch = self.r.info(fullnames=fullnames) From 51d7876f929c5cb649d497c08f192b4327df34b8 Mon Sep 17 00:00:00 2001 From: David Marx Date: Fri, 18 May 2018 01:03:51 -0700 Subject: [PATCH 8/8] Added change log --- CHANGES.rst | 21 +++++++++++++++++++++ psaw/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 CHANGES.rst diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..56d0112 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,21 @@ +Change Log +========== + +0.0.3 (2018/05/18) +------------------ + +* Added praw support. If ``praw.Reddit`` object provided to ``PushshiftAPI``, + gets ids from pushshift and passes them to praw. +* Added support for ``/reddit/submission/comment_ids/`` endpoint. +* Added change log. + +0.0.2 (2018/05/13) +------------------ + +* Improved rate limit handling +* Misc bug fixes. + +0.0.1 (2018/04/14) +------------------ + +* Dirty support for ``reddit/comment/search`` and ``reddit/comment/search``. diff --git a/psaw/__init__.py b/psaw/__init__.py index 36508c8..a9bdcba 100644 --- a/psaw/__init__.py +++ b/psaw/__init__.py @@ -6,4 +6,4 @@ https://github.com/dmarx/psaw from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal -__version__ = '0.0.2' +__version__ = '0.0.3'