From 4e7731f3a4ebbf575f6688d9ef14202ce15ca713 Mon Sep 17 00:00:00 2001 From: David Marx Date: Mon, 6 Aug 2018 01:16:01 -0700 Subject: [PATCH] Added aggs support to _search --- CHANGES.rst | 8 ++++++++ psaw/PushshiftAPI.py | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9a7a750..31bc5fa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Change Log ========== +x.x.x (????/??/??) +------------------ +* Fixed (added?) `aggs` support. + * If an aggs parameter is provided to a search method, it will be returned as the first result. + * Subsequent results will be yielded as though there was no aggs parameter. + * To the best of my understanding, PushShift only returns 100 results per agg, and there is not + way to page for additional results. + 0.0.5 (2018/08/05) ------------------ * New parameter documentation added to README. diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 1e26b80..8aa1465 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -178,8 +178,13 @@ class PushshiftAPIMinimal(object): self.payload = copy.deepcopy(kwargs) endpoint = 'reddit/{}/search'.format(kind) url = self.base_url.format(endpoint=endpoint) - for response in self._handle_paging(url): + if 'aggs' in response: + yield response['aggs'] + # Aggs responses are unreliable in subsequent batches with + # current search paging implementation. Enforce aggs result + # is only returned once. + self.payload.pop('aggs') results = response['data'] if len(results) == 0: return