From 4cb8cbc6c3347a3dcd20cd1c6c58cea21826f2be Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 10 Oct 2023 08:13:07 +0200 Subject: [PATCH] [fix] duckduckgo lite engine: set HTTP header 'Referer' We have had problems with this before, the bot protection from ddg-lite seems to have included this referer in the rating [1][2]. From reverse engineering: - The Referer ``https://google.com/`` was set in commt 257dc7d6c4 --> DDG lite does not like this referer anymore! - The 'Referer' header is only set on second and follow up pages but not on the first page - The vqd value is not needed on the first page, the ddg-lite client sets this value only on follow up pages / this can help to reduce the vqd requests from SearXNG. Related to 'Referer' header & ddg requests: [1] https://github.com/searxng/searxng/pull/2161 [2] https://github.com/searxng/searxng/pull/2081 Closes: https://github.com/searxng/searxng/issues/2796 Signed-off-by: Markus Heiser --- searx/engines/duckduckgo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index d0e818faf..e421b9cfd 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -222,7 +222,6 @@ def request(query, params): # link again and again .. params['headers']['Content-Type'] = 'application/x-www-form-urlencoded' - params['headers']['Referer'] = 'https://google.com/' # initial page does not have an offset if params['pageno'] == 2: @@ -248,6 +247,10 @@ def request(query, params): params['data']['nextParams'] = form_data.get('nextParams', '') params['data']['v'] = form_data.get('v', 'l') + # request needs a vqd argument + params['data']['vqd'] = get_vqd(query) + params['headers']['Referer'] = 'https://lite.duckduckgo.com/' + params['data']['kl'] = eng_region params['cookies']['kl'] = eng_region