diff --git a/README.rst b/README.rst index c74d033..cb898c9 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,7 @@ Features * Extremely simple interface to pass query arguments to the API. The API is sparsely documented, so it's often fruitful to just try an argument and see if it works. * Limited support for pushshift's ``aggs`` argument. +* A ``stop_condition`` argument to make it simple to stop yielding results given arbitrary user-defined criteria Demo usage ---------- @@ -126,7 +127,6 @@ API requests returning 500 comments each. Alternatively, the generator can be qu Using the ``aggs`` argument to count comments mentioning trump each hour in past week ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Replicating the example from the pushshift documentation: https://api.pushshift.io/reddit/search/comment/?q=trump&after=7d&aggs=created_utc&frequency=hour&size=0 @@ -148,6 +148,19 @@ the result to a namedtuple for dot notation attribute access). result = next(gen) +Using the ``stop_condition`` argument to get the most recent submission by a bot account +------------------------------------------------------------------------------------------- + +.. code-block:: python + + gen = api.search_submissions(stop_condition=lambda x: 'bot' in x.author) + + for subm in enumerate(gen): + pass + + print(subm.author) + + License ------- diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index f0d70b4..2202a1d 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -30,7 +30,6 @@ class PushshiftAPIMinimal(object): @property def utc_offset_secs(self): if not self._utc_offset_secs: - #self._utc_offset_secs = dt.datetime.utcnow().astimezone().utcoffset().total_seconds() self._utc_offset_secs = dt.utcnow().astimezone().utcoffset().total_seconds() return self._utc_offset_secs @@ -58,7 +57,7 @@ class PushshiftAPIMinimal(object): self._last_request_time = time.time() def _add_nec_args(self, payload): - #if 'aggs' in payload: + """Adds 'limit' and 'created_utc' arguments to the payload as necessary.""" if self._limited(payload): # Do nothing I guess? Not sure how paging works on this endpoint... return @@ -80,12 +79,11 @@ class PushshiftAPIMinimal(object): i+=1 response_json = json.loads(response.text) outv = response_json['data'] - #if 'aggs' in payload: if self._limited(payload): outv = response_json return outv - def _query(self, kind, stop_condition=lambda **x: False, **kwargs): + def _query(self, kind, stop_condition=lambda x: False, **kwargs): limit = kwargs.get('limit', None) payload = copy.deepcopy(kwargs) n = 0 @@ -99,7 +97,6 @@ class PushshiftAPIMinimal(object): limit = 0 results = self._get(kind, payload) - #if 'aggs' in payload: if self._limited(payload): yield results return @@ -108,10 +105,10 @@ class PushshiftAPIMinimal(object): return for thing in results: n+=1 - if stop_condition(**thing): - return thing = self._wrap_thing(thing, kind) yield thing + if stop_condition(thing): + return payload['before'] = thing.created_utc if (limit is not None) & (limit == 0): return