zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge 870fd2823c into c96ba25f5b

Browse source
This commit is contained in:
Bnyro 2024-10-15 15:28:57 +02:00 • committed by GitHub
commit f616dd3d07
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 102 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/dev/engines/engine_overview.rst
View file

@ -50,6 +50,7 @@ Engine File
categories list categories, in which the engine is working
paging boolean support multiple pages
time_range_support boolean support search time range
license_filter_support boolean support search with content license filter
engine_type str - ``online`` :ref:`[ref] <online engines>` by
default, other possibles values are:
- ``offline`` :ref:`[ref] <offline engines>`
@ -152,6 +153,7 @@ parameters with default value can be redefined for special purposes.
category str current category, like ``'general'``
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
license_filter Optional[str] ``None``, can be ``public`` (copyleft content), ``freetouse`` (Free to share, modify and use), ``commercial`` (not allowed to use without further permission by the creator)
pageno int current pagenumber
searxng_locale str SearXNG's locale selected by user. Specific language code like
``'en'``, ``'en-US'``, or ``'all'`` if unspecified.

1
searx/engines/__init__.py
View file

@ -31,6 +31,7 @@ ENGINE_DEFAULT_ARGS = {
"engine_type": "online",
"paging": False,
"time_range_support": False,
"license_filter_support": False,
"safesearch": False,
# settings.yml
"categories": ["general"],

8
searx/engines/bing_images.py
View file

@ -37,6 +37,7 @@ categories = ['images', 'web']
paging = True
safesearch = True
time_range_support = True
license_filter_support = True
base_url = 'https://www.bing.com/images/async'
"""Bing (Images) search URL"""
@ -47,6 +48,7 @@ time_map = {
'month': 60 * 24 * 31,
'year': 60 * 24 * 365,
}
license_map = {'public': 'L1', 'freetouse': 'L2_L3_L5_L6', 'commercial': ''}
def request(query, params):
@ -69,8 +71,12 @@ def request(query, params):
# time range
# - example: one year (525600 minutes) 'qft=+filterui:age-lt525600'
query_params['qft'] = ''
if params['time_range']:
query_params['qft'] = 'filterui:age-lt%s' % time_map[params['time_range']]
query_params['qft'] += f"+filterui:age-lt{time_map[params['time_range']]}"
if params['license_filter']:
query_params['qft'] += f"+filterui:license-{license_map[params['license_filter']]}"
params['url'] = base_url + '?' + urlencode(query_params)

7
searx/engines/duckduckgo_extra.py
View file

@ -45,6 +45,7 @@ safesearch_cookies = {0: '-2', 1: None, 2: '1'}
safesearch_args = {0: '1', 1: None, 2: '1'}
search_path_map = {'images': 'i', 'videos': 'v', 'news': 'news'}
license_map = {'public': 'Public', 'freetouse': 'Modify', 'commercial': ''}
def request(query, params):
@ -59,12 +60,16 @@ def request(query, params):
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
f_arg = ''
if ddg_category == 'images' and params['license_filter']:
f_arg = 'license:' + license_map[params['license_filter']]
args = {
'q': query,
'o': 'json',
# 'u': 'bing',
'l': eng_region,
'f': ',,,,,',
'f': ',,,,,' + f_arg,
'vqd': vqd,
}

10
searx/engines/google_images.py
View file

@ -48,10 +48,12 @@ categories = ['images', 'web']
paging = True
max_page = 50
time_range_support = True
license_filter_support = True
safesearch = True
send_accept_language_header = True
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
license_map = {'public': 'cl', 'freetouse': 'cl', 'commercial': 'ol'}
def request(query, params):
@ -70,8 +72,14 @@ def request(query, params):
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
tbs_args = []
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
tbs_args.append('qdr:' + time_range_dict[params['time_range']])
if params['license_filter']:
tbs_args.append('sur:' + license_map[params['license_filter']])
if tbs_args:
query_url += '&' + urlencode({'tbs': ','.join(tbs_args)})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url

1
searx/search/checker/impl.py
View file

@ -131,6 +131,7 @@ def _search_query_to_dict(search_query: SearchQuery) -> typing.Dict[str, typing.
'pageno': search_query.pageno,
'safesearch': search_query.safesearch,
'time_range': search_query.time_range,
'license_filter': search_query.license_filter,
}

9
searx/search/models.py
View file

@ -35,6 +35,7 @@ class SearchQuery:
'safesearch',
'pageno',
'time_range',
'license_filter',
'timeout_limit',
'external_bang',
'engine_data',
@ -49,6 +50,7 @@ class SearchQuery:
safesearch: int = 0,
pageno: int = 1,
time_range: typing.Optional[str] = None,
license_filter: typing.Optional[str] = None,
timeout_limit: typing.Optional[float] = None,
external_bang: typing.Optional[str] = None,
engine_data: typing.Optional[typing.Dict[str, str]] = None,
@ -60,6 +62,7 @@ class SearchQuery:
self.safesearch = safesearch
self.pageno = pageno
self.time_range = time_range
self.license_filter = license_filter
self.timeout_limit = timeout_limit
self.external_bang = external_bang
self.engine_data = engine_data or {}
@ -77,13 +80,14 @@ class SearchQuery:
return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
def __repr__(self):
return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
self.query,
self.engineref_list,
self.lang,
self.safesearch,
self.pageno,
self.time_range,
self.license_filter,
self.timeout_limit,
self.external_bang,
self.redirect_to_first_result,
@ -97,6 +101,7 @@ class SearchQuery:
and self.safesearch == other.safesearch
and self.pageno == other.pageno
and self.time_range == other.time_range
and self.license_filter == other.license_filter
and self.timeout_limit == other.timeout_limit
and self.external_bang == other.external_bang
and self.redirect_to_first_result == other.redirect_to_first_result
@ -111,6 +116,7 @@ class SearchQuery:
self.safesearch,
self.pageno,
self.time_range,
self.license_filter,
self.timeout_limit,
self.external_bang,
self.redirect_to_first_result,
@ -125,6 +131,7 @@ class SearchQuery:
self.safesearch,
self.pageno,
self.time_range,
self.license_filter,
self.timeout_limit,
self.external_bang,
self.engine_data,

4
searx/search/processors/abstract.py
View file

@ -157,11 +157,15 @@ class EngineProcessor(ABC):
if search_query.time_range and not self.engine.time_range_support:
return None
if search_query.license_filter and not self.engine.license_filter_support:
return None
params = {}
params['category'] = engine_category
params['pageno'] = search_query.pageno
params['safesearch'] = search_query.safesearch
params['time_range'] = search_query.time_range
params['license_filter'] = search_query.license_filter
params['engine_data'] = search_query.engine_data.get(self.engine_name, {})
params['searxng_locale'] = search_query.lang

1
searx/settings.yml
View file

@ -685,6 +685,7 @@ engines:
- name: duckduckgo images
engine: duckduckgo_extra
categories: [images, web]
license_filter_support: true
ddg_category: images
shortcut: ddi
disabled: true

2
searx/static/themes/simple/css/searxng-rtl.min.css vendored
View file

File diff suppressed because one or more lines are too long

2
searx/static/themes/simple/css/searxng-rtl.min.css.map
View file

File diff suppressed because one or more lines are too long

2
searx/static/themes/simple/css/searxng.min.css vendored
View file

File diff suppressed because one or more lines are too long

2
searx/static/themes/simple/css/searxng.min.css.map
View file

File diff suppressed because one or more lines are too long

2
searx/static/themes/simple/js/searxng.min.js vendored
View file

File diff suppressed because one or more lines are too long

2
searx/static/themes/simple/js/searxng.min.js.map
View file

File diff suppressed because one or more lines are too long

1
searx/static/themes/simple/src/js/main/search.js
View file

@ -164,6 +164,7 @@
searxng.on(d.getElementById('safesearch'), 'change', submitIfQuery);
searxng.on(d.getElementById('time_range'), 'change', submitIfQuery);
searxng.on(d.getElementById('language'), 'change', submitIfQuery);
searxng.on(d.getElementById('license_filter'), 'change', submitIfQuery);
}
// most common browsers at the time of writing this support :has, except for Firefox

1
searx/templates/simple/elements/apis.html
View file

@ -12,6 +12,7 @@
<input type="hidden" name="pageno" value="{{ pageno }}">
<input type="hidden" name="language" value="{{ current_language }}">
<input type="hidden" name="time_range" value="{{ time_range }}">
<input type="hidden" name="license_filter" value="{{ license_filter }}">
<input type="hidden" name="safesearch" value="{{ safesearch }}">
<input type="hidden" name="format" value="{{ output_type }}">
{%- if timeout_limit -%}

14
searx/templates/simple/filters/license.html Normal file
View file

@ -0,0 +1,14 @@
<select name="license_filter" id="license_filter" class="license_filter" aria-label="{{ _('License') }}">{{- '' -}}
<option id="license-any" value="" {{ "selected" if license=="" or not license else ""}}>
{{- _('None') -}}
</option>{{- '' -}}
<option id="license-public" value="public" {{ "selected" if license=="public" else ""}}>
{{- _('Public domain') -}}
</option>{{- '' -}}
<option id="license-freetouse" value="freetouse" {{ "selected" if license=="freetouse" else ""}}>
{{- _('Free to use') -}}
</option>{{- '' -}}
<option id="license-commercial" value="commercial" {{ "selected" if license=="commercial" else ""}}>
{{- _('Commercial') -}}
</option>{{- '' -}}
</select>

4
searx/templates/simple/results.html
View file

@ -82,6 +82,7 @@
<input type="hidden" name="q" value="{{ correction.url }}">
<input type="hidden" name="language" value="{{ current_language }}">
<input type="hidden" name="time_range" value="{{ time_range }}">
<input type="hidden" name="license_filter" value="{{ license_filter }}">
<input type="hidden" name="safesearch" value="{{ safesearch }}">
<input type="hidden" name="theme" value="{{ theme }}">
{% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit }}" >{% endif %}
@ -118,6 +119,7 @@
<input type="hidden" name="pageno" value="{{ pageno-1 }}" >
<input type="hidden" name="language" value="{{ current_language }}" >
<input type="hidden" name="time_range" value="{{ time_range }}" >
<input type="hidden" name="license_filter" value="{{ license_filter }}">
<input type="hidden" name="safesearch" value="{{ safesearch }}" >
<input type="hidden" name="theme" value="{{ theme }}" >
{% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}
@ -136,6 +138,7 @@
<input type="hidden" name="pageno" value="{{ pageno+1 }}" >
<input type="hidden" name="language" value="{{ current_language }}" >
<input type="hidden" name="time_range" value="{{ time_range }}" >
<input type="hidden" name="license_filter" value="{{ license_filter }}">
<input type="hidden" name="safesearch" value="{{ safesearch }}" >
<input type="hidden" name="theme" value="{{ theme }}" >
{% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}
@ -161,6 +164,7 @@
<input type="hidden" name="pageno" value="{{ x }}" >
<input type="hidden" name="language" value="{{ current_language }}" >
<input type="hidden" name="time_range" value="{{ time_range }}" >
<input type="hidden" name="license_filter" value="{{ license_filter }}">
<input type="hidden" name="safesearch" value="{{ safesearch }}" >
<input type="hidden" name="theme" value="{{ theme }}" >
{% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

5
searx/templates/simple/search.html
View file

@ -17,7 +17,10 @@
<div class="search_filters">
{% include 'simple/filters/languages.html' %}
{% include 'simple/filters/time_range.html' %}
{% include 'simple/filters/safesearch.html' %}
{% if 'images' in selected_categories %}
{% include 'simple/filters/safesearch.html' %}
{% endif %}
{% include 'simple/filters/license.html' %}
</div>
<input type="hidden" name="theme" value="{{ theme }}" >
{% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

11
searx/webadapter.py
View file

@ -102,6 +102,15 @@ def parse_time_range(form: Dict[str, str]) -> Optional[str]:
return query_time_range
def parse_license_filter(form: Dict[str, str]) -> Optional[str]:
license_filter = form.get('license_filter')
if license_filter in ('public', 'freetouse', 'commercial'):
return license_filter
return None
def parse_timeout(form: Dict[str, str], raw_text_query: RawTextQuery) -> Optional[float]:
timeout_limit = raw_text_query.timeout_limit
if timeout_limit is None:
@ -258,6 +267,7 @@ def get_search_query_from_webapp(
query_pageno = parse_pageno(form)
query_safesearch = parse_safesearch(preferences, form)
query_time_range = parse_time_range(form)
query_license = parse_license_filter(form)
query_timeout = parse_timeout(form, raw_text_query)
external_bang = raw_text_query.external_bang
redirect_to_first_result = raw_text_query.redirect_to_first_result
@ -292,6 +302,7 @@ def get_search_query_from_webapp(
query_safesearch,
query_pageno,
query_time_range,
query_license,
query_timeout,
external_bang=external_bang,
engine_data=engine_data,

1
searx/webapp.py
View file

@ -781,6 +781,7 @@ def search():
selected_categories = search_query.categories,
pageno = search_query.pageno,
time_range = search_query.time_range or '',
license_filter = search_query.license_filter or '',
number_of_results = format_decimal(result_container.number_of_results),
suggestions = suggestion_urls,
answers = result_container.answers,

11
searxng_extra/standalone_searx.py
View file

@ -67,6 +67,7 @@ def get_search_query(
"pageno": str(args.pageno),
"language": args.lang,
"time_range": args.timerange,
'license_filter': args.license_filter,
}
preferences = searx.preferences.Preferences(['simple'], engine_categories, searx.engines.engines, [])
preferences.key_value_settings['safesearch'].parse(args.safesearch)
@ -106,7 +107,8 @@ def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]:
"pageno": search_query.pageno,
"lang": search_query.lang,
"safesearch": search_query.safesearch,
"timerange": search_query.time_range,
"time_range": search_query.time_range,
"license": search_query.license,
},
"results": no_parsed_url(result_container.get_ordered_results()),
"infoboxes": result_container.infoboxes,
@ -160,6 +162,13 @@ def parse_argument(
parser.add_argument(
'--timerange', type=str, nargs='?', choices=['day', 'week', 'month', 'year'], help='Filter by time range'
)
parser.add_argument(
'--license_filter',
type=str,
nargs='?',
choices=['any', 'public', 'freetouse', 'commercial'],
help='Filter by license',
)
return parser.parse_args(args)

8
tests/unit/processors/test_online.py
View file

@ -36,7 +36,9 @@ class TestOnlineProcessor(SearxTestCase): # pylint: disable=missing-class-docst
def test_get_params_default_params(self):
engine = engines.engines[TEST_ENGINE_NAME]
online_processor = online.OnlineProcessor(engine, TEST_ENGINE_NAME)
search_query = SearchQuery('test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None)
search_query = SearchQuery(
'test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None, None
)
params = self._get_params(online_processor, search_query, 'general')
self.assertIn('method', params)
self.assertIn('headers', params)
@ -48,6 +50,8 @@ class TestOnlineProcessor(SearxTestCase): # pylint: disable=missing-class-docst
def test_get_params_useragent(self):
engine = engines.engines[TEST_ENGINE_NAME]
online_processor = online.OnlineProcessor(engine, TEST_ENGINE_NAME)
search_query = SearchQuery('test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None)
search_query = SearchQuery(
'test', [EngineRef(TEST_ENGINE_NAME, 'general')], 'all', 0, 1, None, None, None, None
)
params = self._get_params(online_processor, search_query, 'general')
self.assertIn('User-Agent', params['headers'])

10
tests/unit/test_search.py
View file

@ -29,7 +29,7 @@ class SearchQueryTestCase(SearxTestCase): # pylint: disable=missing-class-docst
def test_repr(self):
s = SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g')
self.assertEqual(
repr(s), "SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g', None)"
repr(s), "SearchQuery('test', [EngineRef('bing', 'general')], 'all', 0, 1, '1', 5.0, 'g', None, None)"
) # noqa
def test_eq(self):
@ -73,7 +73,7 @@ class SearchTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
def test_timeout_query_above_default_nomax(self):
settings['outgoing']['max_request_timeout'] = None
search_query = SearchQuery(
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 5.0
)
search = searx.search.Search(search_query)
with self.app.test_request_context('/search'):
@ -83,7 +83,7 @@ class SearchTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
def test_timeout_query_below_default_nomax(self):
settings['outgoing']['max_request_timeout'] = None
search_query = SearchQuery(
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 1.0
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 1.0
)
search = searx.search.Search(search_query)
with self.app.test_request_context('/search'):
@ -93,7 +93,7 @@ class SearchTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
def test_timeout_query_below_max(self):
settings['outgoing']['max_request_timeout'] = 10.0
search_query = SearchQuery(
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 5.0
)
search = searx.search.Search(search_query)
with self.app.test_request_context('/search'):
@ -103,7 +103,7 @@ class SearchTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
def test_timeout_query_above_max(self):
settings['outgoing']['max_request_timeout'] = 10.0
search_query = SearchQuery(
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 15.0
'test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None, 15.0
)
search = searx.search.Search(search_query)
with self.app.test_request_context('/search'):