From 5691b1a6926a4809843bc1096c83bfb873746854 Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Thu, 30 Oct 2014 18:40:02 +0100 Subject: [PATCH 1/3] new test for bug: search returns at most 10 items --- test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test.py b/test.py index 4572a2e..a320d7f 100644 --- a/test.py +++ b/test.py @@ -113,4 +113,17 @@ class Tests(TestCase): self.assertIn(self.e2, found) self.assertIn(self.e4, found) + def test_more_items(self): + expected_count = 20 + self.entry_list = [ + self.Entry(title=u'foobar_{}'.format(x), content=u'xxxx', user=self.u1 ) + for x in range(20) + ] + + self.db.session.add_all(self.entry_list) + self.db.session.commit() + + found = self.Entry.query.whooshee_search('foobar').all() + assert len(found) == expected_count + # TODO: more :) From 82c4dd4e16674949d729c607cbf424069b55932a Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Thu, 30 Oct 2014 19:28:34 +0100 Subject: [PATCH 2/3] Pass `limit` parameter to a searcher. --- flask_whooshee.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/flask_whooshee.py b/flask_whooshee.py index c0ada36..31959f8 100644 --- a/flask_whooshee.py +++ b/flask_whooshee.py @@ -15,7 +15,8 @@ class WhoosheeQuery(BaseQuery): """An override for SQLAlchemy query used to do fulltext search.""" # TODO: add an option to override used Whoosheer - def whooshee_search(self, search_string, group=whoosh.qparser.OrGroup, match_substrings=True): + def whooshee_search(self, search_string, group=whoosh.qparser.OrGroup, + match_substrings=True, limit=None): """Do a fulltext search on the query. Args: @@ -23,6 +24,7 @@ class WhoosheeQuery(BaseQuery): group: whoosh group to use for searching, defaults to OrGroup (searches for all words in all columns) match_substrings: True if you want to match substrings, False otherwise + limit: number of the top records to be returned, default None returns all records Returns: query filtered with results of the fulltext search @@ -52,7 +54,8 @@ class WhoosheeQuery(BaseQuery): res = whoosheer.search(search_string=search_string, values_of=uniq, group=group, - match_substrings=match_substrings) + match_substrings=match_substrings, + limit=None) if not res: return self.filter('null') @@ -80,7 +83,7 @@ class AbstractWhoosheer(object): """ @classmethod - def search(cls, search_string, values_of='', group=whoosh.qparser.OrGroup, match_substrings=True): + def search(cls, search_string, values_of='', group=whoosh.qparser.OrGroup, match_substrings=True, limit=None): """Actually searches the fields for given search_string. Args: @@ -90,6 +93,8 @@ class AbstractWhoosheer(object): group: whoosh group to use for searching, defaults to OrGroup (searches for all words in all columns) match_substrings: True if you want to match substrings, False otherwise + limit: number of the top records to be returned, default None returns all records + Returns: Found records if 'not values_of', else values of given column """ @@ -97,7 +102,7 @@ class AbstractWhoosheer(object): with cls.index.searcher() as searcher: parser = whoosh.qparser.MultifieldParser(cls.schema.names(), cls.index.schema, group=group) query = parser.parse(prepped_string) - results = searcher.search(query) + results = searcher.search(query, limit=limit) if values_of: return [x[values_of] for x in results] return results From 73594f7b47d5672c288266c90118a019c08beecc Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Thu, 30 Oct 2014 19:43:44 +0100 Subject: [PATCH 3/3] slightly more complex test for search of larse sets --- test.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test.py b/test.py index a320d7f..016ee23 100644 --- a/test.py +++ b/test.py @@ -114,16 +114,22 @@ class Tests(TestCase): self.assertIn(self.e4, found) def test_more_items(self): - expected_count = 20 - self.entry_list = [ - self.Entry(title=u'foobar_{}'.format(x), content=u'xxxx', user=self.u1 ) - for x in range(20) - ] + expected_count = 0 + # couldn't test for large set due to some bugs either in sqlite or whoosh or SA + # got: OperationalError: (OperationalError) too many SQL variables u'SELECT entry.id + # ... FROM entry \nWHERE entry.id IN (?, ?, .... when whooshee_search is invoked + for batch_size in [2, 5, 7, 20, 50, 300, 500]: # , 1000]: + expected_count += batch_size + self.entry_list = [ + self.Entry(title=u'foobar_{}_{}'.format(expected_count, x), + content=u'xxxx', user=self.u1) + for x in range(batch_size) + ] - self.db.session.add_all(self.entry_list) - self.db.session.commit() + self.db.session.add_all(self.entry_list) + self.db.session.commit() - found = self.Entry.query.whooshee_search('foobar').all() - assert len(found) == expected_count + found = self.Entry.query.whooshee_search('foobar').all() + assert len(found) == expected_count # TODO: more :)