diff --git a/.gitignore b/.gitignore index 33c8692..62c8c72 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ _build/ build/ dist/ +praw.ini diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 553d5b5..5efcc37 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -40,11 +40,20 @@ class RateLimitCache(object): raise Exception("RateLimitCache is blocked.") self.cache.append(time.time()) + class PushshiftAPIMinimal(object): #base_url = {'search':'https://api.pushshift.io/reddit/{}/search/', # 'meta':'https://api.pushshift.io/meta/'} _base_url = 'https://{domain}.pushshift.io/{{endpoint}}' _limited_args = ('aggs') + _thing_prefix = { + 'Comment':'t1_', + 'Account':'t2_', + 'Link':'t3_', + 'Message':'t4_', + 'Subreddit':'t5_', + 'Award':'t6_' + } def __init__(self, max_retries=20, max_sleep=3600, @@ -221,11 +230,22 @@ class WithPraw(PushshiftAPIMinimal): self.r = r def search_comments(self, **kwargs): - return self._praw_get(kind='comment', **kwargs) + return self._praw_search(kind='comment', **kwargs) def search_submissions(self, **kwargs): - return self._praw_get(kind='submission', **kwargs) + return self._praw_search(kind='submission', **kwargs) - def _praw_get(self, **kwargs): - for c in self._search(**kwargs): - yield c + def _praw_search(self, **kwargs): + prefix = self._thing_prefix[kwargs['kind'].title()] + client_return_batch = kwargs.get('return_batch') + if client_return_batch is False: + kwargs.pop('return_batch') + + for batch in self._search(return_batch=True, **kwargs): + fullnames = [prefix + thing.id for thing in batch] + praw_batch = self.r.info(fullnames=fullnames) + if client_return_batch: + yield praw_batch + else: + for praw_thing in praw_batch: + yield praw_thing diff --git a/psaw/__init__.py b/psaw/__init__.py index 36508c8..f9bd879 100644 --- a/psaw/__init__.py +++ b/psaw/__init__.py @@ -4,6 +4,6 @@ Pushshift.io API Wrapper (for reddit.com public comment/submission search) https://github.com/dmarx/psaw """ -from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal +from .PushshiftAPI import PushshiftAPI, PushshiftAPIMinimal, WithPraw __version__ = '0.0.2'