Merge pull request #1487 from searxng/google-consent

bypass google consent with ucbcb=1

searx/engines/google.py

@@ -287,6 +287,7 @@ def request(query, params):
                 'oe': "utf8",
                 'start': offset,
                 'filter': '0',
+                'ucbcb': 1,
                 **additional_parameters,
             }
         )

searx/engines/google_images.py

@@ -133,14 +133,7 @@ def request(query, params):
         + '/search'
         + "?"
         + urlencode(
-            {
-                'q': query,
-                'tbm': "isch",
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
-                'num': 30,
-            }
+            {'q': query, 'tbm': "isch", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'num': 30, 'ucbcb': 1}
         )
     )
 

searx/engines/google_news.py

@@ -14,7 +14,6 @@ ignores some parameters from the common :ref:`google API`:
 # pylint: disable=invalid-name
 
 import binascii
-from datetime import datetime
 import re
 from urllib.parse import urlencode
 from base64 import b64decode
@@ -99,13 +98,7 @@ def request(query, params):
         + '/search'
         + "?"
         + urlencode(
-            {
-                'q': query,
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
-                'gl': lang_info['country'],
-            }
+            {'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'gl': lang_info['country'], 'ucbcb': 1}
         )
         + ('&ceid=%s' % ceid)
     )  # ceid includes a ':' character which must not be urlencoded
@@ -113,7 +106,6 @@ def request(query, params):
 
     params['headers'].update(lang_info['headers'])
     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-    params['headers']['Cookie'] = "CONSENT=YES+cb.%s-14-p0.en+F+941;" % datetime.now().strftime("%Y%m%d")
 
     return params
 

searx/engines/google_play_apps.py

@@ -22,7 +22,7 @@ about = {
 }
 
 categories = ["files", "apps"]
-search_url = "https://play.google.com/store/search?{query}&c=apps"
+search_url = "https://play.google.com/store/search?{query}&c=apps&ucbcb=1"
 
 
 def request(query, params):

searx/engines/google_scholar.py

@@ -85,15 +85,7 @@ def request(query, params):
         + lang_info['subdomain']
         + '/scholar'
         + "?"
-        + urlencode(
-            {
-                'q': query,
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
-                'start': offset,
-            }
-        )
+        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset, 'ucbcb': 1})
     )
 
     query_url += time_range_url(params)

searx/engines/google_videos.py

@@ -118,15 +118,7 @@ def request(query, params):
         + lang_info['subdomain']
         + '/search'
         + "?"
-        + urlencode(
-            {
-                'q': query,
-                'tbm': "vid",
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
-            }
-        )
+        + urlencode({'q': query, 'tbm': "vid", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'ucbcb': 1})
     )
 
     if params['time_range'] in time_range_dict:

searx/engines/youtube_noapi.py

@@ -3,7 +3,6 @@
  Youtube (Videos)
 """
 
-from datetime import datetime
 from functools import reduce
 from json import loads, dumps
 from urllib.parse import quote_plus
@@ -26,7 +25,7 @@ time_range_support = True
 
 # search-url
 base_url = 'https://www.youtube.com/results'
-search_url = base_url + '?search_query={query}&page={page}'
+search_url = base_url + '?search_query={query}&page={page}&ucbcb=1'
 time_range_url = '&sp=EgII{time_range}%253D%253D'
 # the key seems to be constant
 next_page_url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
@@ -52,7 +51,6 @@ def request(query, params):
         )
         params['headers']['Content-Type'] = 'application/json'
 
-    params['headers']['Cookie'] = "CONSENT=YES+cb.%s-17-p0.en+F+941;" % datetime.now().strftime("%Y%m%d")
     return params