zaclys
/
searxng
mirror of https://github.com/searxng/searxng.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

[fix] use queues to answers/suggestions/infoboxes

This commit is contained in:
Adam Tauber 2014-12-14 02:15:04 +01:00
parent a5324d9d82
commit d93d703703
1 changed files with 18 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
searx/search.py
View File

@ -73,7 +73,6 @@ def make_callback(engine_name,
# creating a callback wrapper for the search engine results # creating a callback wrapper for the search engine results
def process_callback(response, **kwargs): def process_callback(response, **kwargs):
cb_res = []
response.search_params = params response.search_params = params
# callback # callback
@ -82,7 +81,6 @@ def make_callback(engine_name,
except Exception, e: except Exception, e:
# increase errors stats # increase errors stats
engines[engine_name].stats['errors'] += 1 engines[engine_name].stats['errors'] += 1
results_queue.put_nowait((engine_name, cb_res))
# print engine name and specific error message # print engine name and specific error message
print '[E] Error with engine "{0}":\n\t{1}'.format( print '[E] Error with engine "{0}":\n\t{1}'.format(
@ -93,26 +91,7 @@ def make_callback(engine_name,
for result in search_results: for result in search_results:
result['engine'] = engine_name result['engine'] = engine_name
# if it is a suggestion, add it to list of suggestions results_queue.put_nowait((engine_name, search_results))
if 'suggestion' in result:
# TODO type checks
suggestions.add(result['suggestion'])
continue
# if it is an answer, add it to list of answers
if 'answer' in result:
answers.add(result['answer'])
continue
# if it is an infobox, add it to list of infoboxes
if 'infobox' in result:
infoboxes.append(result)
continue
# append result
cb_res.append(result)
results_queue.put_nowait((engine_name, cb_res))
# update stats with current page-load-time # update stats with current page-load-time
engines[engine_name].stats['page_load_time'] += \ engines[engine_name].stats['page_load_time'] += \
@ -511,8 +490,25 @@ class Search(object):
threaded_requests(requests) threaded_requests(requests)
results = {} results = {}
while not results_queue.empty(): while not results_queue.empty():
engine_name, engine_results = results_queue.get_nowait() engine_name, engine_results = results_queue.get_nowait()
# TODO type checks
[suggestions.add(x['suggestion'])
for x in list(engine_results)
if 'suggestion' in x
and engine_results.remove(x) is None]
[answers.add(x['answer'])
for x in list(engine_results)
if 'answer' in x
and engine_results.remove(x) is None]
infoboxes.extend(x for x in list(engine_results)
if 'infobox' in x
and engine_results.remove(x) is None)
results[engine_name] = engine_results results[engine_name] = engine_results
# update engine-specific stats # update engine-specific stats