[mod] add option max_page

Related: https://github.com/searxng/searxng/issues/2982
Closes: https://github.com/searxng/searxng/issues/2972

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/__init__.py

@@ -45,6 +45,7 @@ ENGINE_DEFAULT_ARGS = {
     "using_tor_proxy": False,
     "send_accept_language_header": False,
     "tokens": [],
+    "max_page": 0,
 }
 # set automatically when an engine does not have any tab category
 DEFAULT_CATEGORY = 'other'

searx/engines/google.py

@@ -48,6 +48,7 @@ about = {
 # engine dependent config
 categories = ['general', 'web']
 paging = True
+max_page = 50
 time_range_support = True
 safesearch = True
 

searx/engines/google_images.py

@@ -47,6 +47,7 @@ about = {
 # engine dependent config
 categories = ['images', 'web']
 paging = True
+max_page = 50
 time_range_support = True
 safesearch = True
 send_accept_language_header = True

searx/engines/google_scholar.py

@@ -51,6 +51,7 @@ about = {
 # engine dependent config
 categories = ['science', 'scientific publications']
 paging = True
+max_page = 50
 language_support = True
 time_range_support = True
 safesearch = False

searx/engines/google_videos.py

@@ -57,6 +57,7 @@ about = {
 
 categories = ['videos', 'web']
 paging = True
+max_page = 50
 language_support = True
 time_range_support = True
 safesearch = True

searx/search/processors/abstract.py

@@ -150,6 +150,11 @@ class EngineProcessor(ABC):
         if search_query.pageno > 1 and not self.engine.paging:
             return None
 
+        # if max page is reached, skip
+        max_page = self.engine.max_page or settings['search']['max_page']
+        if max_page and max_page < search_query.pageno:
+            return None
+
         # if time_range is not supported, skip
         if search_query.time_range and not self.engine.time_range_support:
             return None

searx/settings.yml

@@ -32,6 +32,7 @@ search:
   # Default search language - leave blank to detect from browser information or
   # use codes from 'languages.py'
   default_lang: "auto"
+  # max_page: 0  # if engine supports paging, 0 means unlimited numbers of pages
   # Available languages
   # languages:
   #   - all

searx/settings_defaults.py

@@ -169,6 +169,7 @@ SCHEMA = {
             'recaptcha_SearxEngineCaptcha': SettingsValue(numbers.Real, 604800),
         },
         'formats': SettingsValue(list, OUTPUT_FORMATS),
+        'max_page': SettingsValue(int, 0),
     },
     'server': {
         'port': SettingsValue((int, str), 8888, 'SEARXNG_PORT'),