Fixed stop_condition behavior, added stop_condition demo, cleaned out deprecated code

README.rst

@@ -45,6 +45,7 @@ Features
 * Extremely simple interface to pass query arguments to the API. The API is sparsely documented,
   so it's often fruitful to just try an argument and see if it works.
 * Limited support for pushshift's ``aggs`` argument.
+* A ``stop_condition`` argument to make it simple to stop yielding results given arbitrary user-defined criteria
 
 Demo usage
 ----------
@@ -126,7 +127,6 @@ API requests returning 500 comments each. Alternatively, the generator can be qu
 Using the ``aggs`` argument to count comments mentioning trump each hour in past week
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-
 Replicating the example from the pushshift documentation:
 
 https://api.pushshift.io/reddit/search/comment/?q=trump&after=7d&aggs=created_utc&frequency=hour&size=0
@@ -148,6 +148,19 @@ the result to a namedtuple for dot notation attribute access).
 
     result = next(gen)
 
+Using the ``stop_condition`` argument to get the most recent submission by a bot account
+-------------------------------------------------------------------------------------------
+
+.. code-block:: python
+
+    gen = api.search_submissions(stop_condition=lambda x: 'bot' in x.author)
+
+    for subm in enumerate(gen):
+        pass
+
+    print(subm.author)
+
+
 License
 -------
 

psaw/PushshiftAPI.py

@@ -30,7 +30,6 @@ class PushshiftAPIMinimal(object):
     @property
     def utc_offset_secs(self):
         if not self._utc_offset_secs:
-            #self._utc_offset_secs = dt.datetime.utcnow().astimezone().utcoffset().total_seconds()
             self._utc_offset_secs =  dt.utcnow().astimezone().utcoffset().total_seconds()
         return self._utc_offset_secs
 
@@ -58,7 +57,7 @@ class PushshiftAPIMinimal(object):
         self._last_request_time = time.time()
 
     def _add_nec_args(self, payload):
-        #if 'aggs' in payload:
+        """Adds 'limit' and 'created_utc' arguments to the payload as necessary."""
         if self._limited(payload):
             # Do nothing I guess? Not sure how paging works on this endpoint...
             return
@@ -80,12 +79,11 @@ class PushshiftAPIMinimal(object):
             i+=1
         response_json = json.loads(response.text)
         outv = response_json['data']
-        #if 'aggs' in payload:
         if self._limited(payload):
             outv = response_json
         return outv
 
-    def _query(self, kind, stop_condition=lambda **x: False, **kwargs):
+    def _query(self, kind, stop_condition=lambda x: False, **kwargs):
         limit = kwargs.get('limit', None)
         payload = copy.deepcopy(kwargs)
         n = 0
@@ -99,7 +97,6 @@ class PushshiftAPIMinimal(object):
                     limit = 0
 
             results = self._get(kind, payload)
-            #if 'aggs' in payload:
             if self._limited(payload):
                 yield results
                 return
@@ -108,10 +105,10 @@ class PushshiftAPIMinimal(object):
                 return
             for thing in results:
                 n+=1
-                if stop_condition(**thing):
-                    return
                 thing = self._wrap_thing(thing, kind)
                 yield thing
+                if stop_condition(thing):
+                    return
             payload['before'] = thing.created_utc
             if (limit is not None) & (limit == 0):
                 return