Merge 870fd2823c into c96ba25f5b

docs/dev/engines/engine_overview.rst

@@ -50,6 +50,7 @@ Engine File
    categories              list        categories, in which the engine is working
    paging                  boolean     support multiple pages
    time_range_support      boolean     support search time range
+   license_filter_support  boolean     support search with content license filter
    engine_type             str         - ``online`` :ref:`[ref] <online engines>` by
                                          default, other possibles values are:
                                        - ``offline`` :ref:`[ref] <offline engines>`
@@ -152,6 +153,7 @@ parameters with default value can be redefined for special purposes.
    category               str            current category, like ``'general'``
    safesearch             int            ``0``, between ``0`` and ``2`` (normal, moderate, strict)
    time_range             Optional[str]  ``None``, can be ``day``, ``week``, ``month``, ``year``
+   license_filter         Optional[str]  ``None``, can be ``public`` (copyleft content), ``freetouse`` (Free to share, modify and use), ``commercial`` (not allowed to use without further permission by the creator)
    pageno                 int            current pagenumber
    searxng_locale         str            SearXNG's locale selected by user.  Specific language code like
                                          ``'en'``, ``'en-US'``, or ``'all'`` if unspecified.

searx/engines/__init__.py

@@ -31,6 +31,7 @@ ENGINE_DEFAULT_ARGS = {
     "engine_type": "online",
     "paging": False,
     "time_range_support": False,
+    "license_filter_support": False,
     "safesearch": False,
     # settings.yml
     "categories": ["general"],

searx/engines/bing_images.py

@@ -37,6 +37,7 @@ categories = ['images', 'web']
 paging = True
 safesearch = True
 time_range_support = True
+license_filter_support = True
 
 base_url = 'https://www.bing.com/images/async'
 """Bing (Images) search URL"""
@@ -47,6 +48,7 @@ time_map = {
     'month': 60 * 24 * 31,
     'year': 60 * 24 * 365,
 }
+license_map = {'public': 'L1', 'freetouse': 'L2_L3_L5_L6', 'commercial': ''}
 
 
 def request(query, params):
@@ -69,8 +71,12 @@ def request(query, params):
     # time range
     # - example: one year (525600 minutes) 'qft=+filterui:age-lt525600'
 
+    query_params['qft'] = ''
     if params['time_range']:
-        query_params['qft'] = 'filterui:age-lt%s' % time_map[params['time_range']]
+        query_params['qft'] += f"+filterui:age-lt{time_map[params['time_range']]}"
+
+    if params['license_filter']:
+        query_params['qft'] += f"+filterui:license-{license_map[params['license_filter']]}"
 
     params['url'] = base_url + '?' + urlencode(query_params)
 

searx/engines/duckduckgo_extra.py

@@ -45,6 +45,7 @@ safesearch_cookies = {0: '-2', 1: None, 2: '1'}
 safesearch_args = {0: '1', 1: None, 2: '1'}
 
 search_path_map = {'images': 'i', 'videos': 'v', 'news': 'news'}
+license_map = {'public': 'Public', 'freetouse': 'Modify', 'commercial': ''}
 
 
 def request(query, params):
@@ -59,12 +60,16 @@ def request(query, params):
     eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
     eng_lang = get_ddg_lang(traits, params['searxng_locale'])
 
+    f_arg = ''
+    if ddg_category == 'images' and params['license_filter']:
+        f_arg = 'license:' + license_map[params['license_filter']]
+
     args = {
         'q': query,
         'o': 'json',
         # 'u': 'bing',
         'l': eng_region,
-        'f': ',,,,,',
+        'f': ',,,,,' + f_arg,
         'vqd': vqd,
     }
 

searx/engines/google_images.py

@@ -48,10 +48,12 @@ categories = ['images', 'web']
 paging = True
 max_page = 50
 time_range_support = True
+license_filter_support = True
 safesearch = True
 send_accept_language_header = True
 
 filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
+license_map = {'public': 'cl', 'freetouse': 'cl', 'commercial': 'ol'}
 
 
 def request(query, params):
@@ -70,8 +72,14 @@ def request(query, params):
         + f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
     )
 
+    tbs_args = []
     if params['time_range'] in time_range_dict:
-        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
+        tbs_args.append('qdr:' + time_range_dict[params['time_range']])
+    if params['license_filter']:
+        tbs_args.append('sur:' + license_map[params['license_filter']])
+    if tbs_args:
+        query_url += '&' + urlencode({'tbs': ','.join(tbs_args)})
+
     if params['safesearch']:
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url

searx/search/checker/impl.py

@@ -131,6 +131,7 @@ def _search_query_to_dict(search_query: SearchQuery) -> typing.Dict[str, typing.
         'pageno': search_query.pageno,
         'safesearch': search_query.safesearch,
         'time_range': search_query.time_range,
+        'license_filter': search_query.license_filter,
     }
 
 

searx/search/models.py

@@ -35,6 +35,7 @@ class SearchQuery:
         'safesearch',
         'pageno',
         'time_range',
+        'license_filter',
         'timeout_limit',
         'external_bang',
         'engine_data',
@@ -49,6 +50,7 @@ class SearchQuery:
         safesearch: int = 0,
         pageno: int = 1,
         time_range: typing.Optional[str] = None,
+        license_filter: typing.Optional[str] = None,
         timeout_limit: typing.Optional[float] = None,
         external_bang: typing.Optional[str] = None,
         engine_data: typing.Optional[typing.Dict[str, str]] = None,
@@ -60,6 +62,7 @@ class SearchQuery:
         self.safesearch = safesearch
         self.pageno = pageno
         self.time_range = time_range
+        self.license_filter = license_filter
         self.timeout_limit = timeout_limit
         self.external_bang = external_bang
         self.engine_data = engine_data or {}
@@ -77,13 +80,14 @@ class SearchQuery:
         return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
 
     def __repr__(self):
-        return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
+        return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
             self.query,
             self.engineref_list,
             self.lang,
             self.safesearch,
             self.pageno,
             self.time_range,
+            self.license_filter,
             self.timeout_limit,
             self.external_bang,
             self.redirect_to_first_result,
@@ -97,6 +101,7 @@ class SearchQuery:
             and self.safesearch == other.safesearch
             and self.pageno == other.pageno
             and self.time_range == other.time_range
+            and self.license_filter == other.license_filter
             and self.timeout_limit == other.timeout_limit
             and self.external_bang == other.external_bang
             and self.redirect_to_first_result == other.redirect_to_first_result
@@ -111,6 +116,7 @@ class SearchQuery:
                 self.safesearch,
                 self.pageno,
                 self.time_range,
+                self.license_filter,
                 self.timeout_limit,
                 self.external_bang,
                 self.redirect_to_first_result,
@@ -125,6 +131,7 @@ class SearchQuery:
             self.safesearch,
             self.pageno,
             self.time_range,
+            self.license_filter,
             self.timeout_limit,
             self.external_bang,
             self.engine_data,

searx/search/processors/abstract.py

@@ -157,11 +157,15 @@ class EngineProcessor(ABC):
         if search_query.time_range and not self.engine.time_range_support:
             return None
 
+        if search_query.license_filter and not self.engine.license_filter_support:
+            return None
+
         params = {}
         params['category'] = engine_category
         params['pageno'] = search_query.pageno
         params['safesearch'] = search_query.safesearch
         params['time_range'] = search_query.time_range
+        params['license_filter'] = search_query.license_filter
         params['engine_data'] = search_query.engine_data.get(self.engine_name, {})
         params['searxng_locale'] = search_query.lang
 

searx/settings.yml

@@ -685,6 +685,7 @@ engines:
   - name: duckduckgo images
     engine: duckduckgo_extra
     categories: [images, web]
+    license_filter_support: true
     ddg_category: images
     shortcut: ddi
     disabled: true

searx/static/themes/simple/src/js/main/search.js

@@ -164,6 +164,7 @@
       searxng.on(d.getElementById('safesearch'), 'change', submitIfQuery);
       searxng.on(d.getElementById('time_range'), 'change', submitIfQuery);
       searxng.on(d.getElementById('language'), 'change', submitIfQuery);
+      searxng.on(d.getElementById('license_filter'), 'change', submitIfQuery);
     }
 
     // most common browsers at the time of writing this support :has, except for Firefox

searx/templates/simple/elements/apis.html

@@ -12,6 +12,7 @@
             <input type="hidden" name="pageno" value="{{ pageno }}">
             <input type="hidden" name="language" value="{{ current_language }}">
             <input type="hidden" name="time_range" value="{{ time_range }}">
+            <input type="hidden" name="license_filter" value="{{ license_filter }}">
             <input type="hidden" name="safesearch" value="{{ safesearch }}">
             <input type="hidden" name="format" value="{{ output_type }}">
             {%- if timeout_limit -%}

searx/templates/simple/filters/license.html

@@ -0,0 +1,14 @@
+<select name="license_filter" id="license_filter" class="license_filter" aria-label="{{ _('License') }}">{{- '' -}}
+    <option id="license-any" value="" {{ "selected" if license=="" or not license else ""}}>
+        {{- _('None') -}}
+    </option>{{- '' -}}
+    <option id="license-public" value="public" {{ "selected" if license=="public" else ""}}>
+        {{- _('Public domain') -}}
+    </option>{{- '' -}}
+    <option id="license-freetouse" value="freetouse" {{ "selected" if license=="freetouse" else ""}}>
+        {{- _('Free to use') -}}
+    </option>{{- '' -}}
+    <option id="license-commercial" value="commercial" {{ "selected" if license=="commercial" else ""}}>
+        {{- _('Commercial') -}}
+    </option>{{- '' -}}
+</select>

searx/templates/simple/results.html

@@ -82,6 +82,7 @@
           <input type="hidden" name="q" value="{{ correction.url }}">
           <input type="hidden" name="language" value="{{ current_language }}">
           <input type="hidden" name="time_range" value="{{ time_range }}">
+          <input type="hidden" name="license_filter" value="{{ license_filter }}">
           <input type="hidden" name="safesearch" value="{{ safesearch }}">
           <input type="hidden" name="theme" value="{{ theme }}">
           {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit }}" >{% endif %}
@@ -118,6 +119,7 @@
                   <input type="hidden" name="pageno" value="{{ pageno-1 }}" >
                   <input type="hidden" name="language" value="{{ current_language }}" >
                   <input type="hidden" name="time_range" value="{{ time_range }}" >
+                  <input type="hidden" name="license_filter" value="{{ license_filter }}">
                   <input type="hidden" name="safesearch" value="{{ safesearch }}" >
                   <input type="hidden" name="theme" value="{{ theme }}" >
                   {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}
@@ -136,6 +138,7 @@
                 <input type="hidden" name="pageno" value="{{ pageno+1 }}" >
                 <input type="hidden" name="language" value="{{ current_language }}" >
                 <input type="hidden" name="time_range" value="{{ time_range }}" >
+                <input type="hidden" name="license_filter" value="{{ license_filter }}">
                 <input type="hidden" name="safesearch" value="{{ safesearch }}" >
                 <input type="hidden" name="theme" value="{{ theme }}" >
                 {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}
@@ -161,6 +164,7 @@
                 <input type="hidden" name="pageno" value="{{ x }}" >
                 <input type="hidden" name="language" value="{{ current_language }}" >
                 <input type="hidden" name="time_range" value="{{ time_range }}" >
+                <input type="hidden" name="license_filter" value="{{ license_filter }}">
                 <input type="hidden" name="safesearch" value="{{ safesearch }}" >
                 <input type="hidden" name="theme" value="{{ theme }}" >
                 {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

searx/templates/simple/search.html

@@ -17,7 +17,10 @@
   <div class="search_filters">
     {% include 'simple/filters/languages.html' %}
     {% include 'simple/filters/time_range.html' %}
-    {% include 'simple/filters/safesearch.html' %}
+    {% if 'images' in selected_categories %}
+      {% include 'simple/filters/safesearch.html' %}
+    {% endif %}
+    {% include 'simple/filters/license.html' %}
   </div>
   <input type="hidden" name="theme" value="{{ theme }}" >
   {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

searx/webadapter.py

@@ -102,6 +102,15 @@ def parse_time_range(form: Dict[str, str]) -> Optional[str]:
     return query_time_range
 
 
+def parse_license_filter(form: Dict[str, str]) -> Optional[str]:
+    license_filter = form.get('license_filter')
+
+    if license_filter in ('public', 'freetouse', 'commercial'):
+        return license_filter
+
+    return None
+
+
 def parse_timeout(form: Dict[str, str], raw_text_query: RawTextQuery) -> Optional[float]:
     timeout_limit = raw_text_query.timeout_limit
     if timeout_limit is None:
@@ -258,6 +267,7 @@ def get_search_query_from_webapp(
     query_pageno = parse_pageno(form)
     query_safesearch = parse_safesearch(preferences, form)
     query_time_range = parse_time_range(form)
+    query_license = parse_license_filter(form)
     query_timeout = parse_timeout(form, raw_text_query)
     external_bang = raw_text_query.external_bang
     redirect_to_first_result = raw_text_query.redirect_to_first_result
@@ -292,6 +302,7 @@ def get_search_query_from_webapp(
             query_safesearch,
             query_pageno,
             query_time_range,
+            query_license,
             query_timeout,
             external_bang=external_bang,
             engine_data=engine_data,

searx/webapp.py

@@ -781,6 +781,7 @@ def search():
         selected_categories = search_query.categories,
         pageno = search_query.pageno,
         time_range = search_query.time_range or '',
+        license_filter = search_query.license_filter or '',
         number_of_results = format_decimal(result_container.number_of_results),
         suggestions = suggestion_urls,
         answers = result_container.answers,

searxng_extra/standalone_searx.py

@@ -67,6 +67,7 @@ def get_search_query(
         "pageno": str(args.pageno),
         "language": args.lang,
         "time_range": args.timerange,
+        'license_filter': args.license_filter,
     }
     preferences = searx.preferences.Preferences(['simple'], engine_categories, searx.engines.engines, [])
     preferences.key_value_settings['safesearch'].parse(args.safesearch)
@@ -106,7 +107,8 @@ def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]:
             "pageno": search_query.pageno,
             "lang": search_query.lang,
             "safesearch": search_query.safesearch,
-            "timerange": search_query.time_range,
+            "time_range": search_query.time_range,
+            "license": search_query.license,
         },
         "results": no_parsed_url(result_container.get_ordered_results()),
         "infoboxes": result_container.infoboxes,
@@ -160,6 +162,13 @@ def parse_argument(
     parser.add_argument(
         '--timerange', type=str, nargs='?', choices=['day', 'week', 'month', 'year'], help='Filter by time range'
     )
+    parser.add_argument(
+        '--license_filter',
+        type=str,
+        nargs='?',
+        choices=['any', 'public', 'freetouse', 'commercial'],
+        help='Filter by license',
+    )
     return parser.parse_args(args)
 
 

tests/unit/processors/test_online.py

@@ -36,7 +36,9 @@ class TestOnlineProcessor(SearxTestCase):  # pylint: disable=missing-class-docst
     def test_get_params_default_params(self):
         engine = engines.engines[TEST_ENGINE_NAME]
         online_processor = online.OnlineProcessor(engine, TEST_ENGINE_NAME)
-        search_query = SearchQuery('test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None)
+        search_query = SearchQuery(
+            'test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None, None
+        )
         params = self._get_params(online_processor, search_query, 'general')
         self.assertIn('method', params)
         self.assertIn('headers', params)
@@ -48,6 +50,8 @@ class TestOnlineProcessor(SearxTestCase):  # pylint: disable=missing-class-docst
     def test_get_params_useragent(self):
         engine = engines.engines[TEST_ENGINE_NAME]
         online_processor = online.OnlineProcessor(engine, TEST_ENGINE_NAME)
-        search_query = SearchQuery('test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None)
+        search_query = SearchQuery(
+            'test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None, None
+        )
         params = self._get_params(online_processor, search_query, 'general')
         self.assertIn('User-Agent', params['headers'])

tests/unit/test_search.py

@@ -29,7 +29,7 @@ class SearchQueryTestCase(SearxTestCase):  # pylint: disable=missing-class-docst
     def test_repr(self):
         s = SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g')
         self.assertEqual(
-            repr(s), "SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g', None)"
+            repr(s), "SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g', None, None)"
         )  # noqa
 
     def test_eq(self):
@@ -73,7 +73,7 @@ class SearchTestCase(SearxTestCase):  # pylint: disable=missing-class-docstring
     def test_timeout_query_above_default_nomax(self):
         settings['outgoing']['max_request_timeout'] = None
         search_query = SearchQuery(
-            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0
+            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 5.0
         )
         search = searx.search.Search(search_query)
         with self.app.test_request_context('/search'):
@@ -83,7 +83,7 @@ class SearchTestCase(SearxTestCase):  # pylint: disable=missing-class-docstring
     def test_timeout_query_below_default_nomax(self):
         settings['outgoing']['max_request_timeout'] = None
         search_query = SearchQuery(
-            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 1.0
+            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 1.0
         )
         search = searx.search.Search(search_query)
         with self.app.test_request_context('/search'):
@@ -93,7 +93,7 @@ class SearchTestCase(SearxTestCase):  # pylint: disable=missing-class-docstring
     def test_timeout_query_below_max(self):
         settings['outgoing']['max_request_timeout'] = 10.0
         search_query = SearchQuery(
-            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0
+            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 5.0
         )
         search = searx.search.Search(search_query)
         with self.app.test_request_context('/search'):
@@ -103,7 +103,7 @@ class SearchTestCase(SearxTestCase):  # pylint: disable=missing-class-docstring
     def test_timeout_query_above_max(self):
         settings['outgoing']['max_request_timeout'] = 10.0
         search_query = SearchQuery(
-            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 15.0
+            'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 15.0
         )
         search = searx.search.Search(search_query)
         with self.app.test_request_context('/search'):