mirror of
				https://github.com/searxng/searxng
				synced 2024-01-01 19:24:07 +01:00 
			
		
		
		
	[format.python] initial formatting of the python code
This patch was generated by black [1]::
    make format.python
[1] https://github.com/psf/black
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
			
			
This commit is contained in:
		
							parent
							
								
									fcdc2c2cd2
								
							
						
					
					
						commit
						3d96a9839a
					
				
					 184 changed files with 2800 additions and 2836 deletions
				
			
		|  | @ -29,6 +29,7 @@ if settings is not None: | |||
| 
 | ||||
| _unset = object() | ||||
| 
 | ||||
| 
 | ||||
| def get_setting(name, default=_unset): | ||||
|     """Returns the value to which ``name`` point.  If there is no such name in the | ||||
|     settings and the ``default`` is unset, a :py:obj:`KeyError` is raised. | ||||
|  | @ -80,14 +81,9 @@ def logging_config_debug(): | |||
|             'levelname': {'color': 8}, | ||||
|             'name': {'color': 8}, | ||||
|             'programname': {'color': 'cyan'}, | ||||
|             'username': {'color': 'yellow'} | ||||
|             'username': {'color': 'yellow'}, | ||||
|         } | ||||
|         coloredlogs.install( | ||||
|             level=log_level, | ||||
|             level_styles=level_styles, | ||||
|             field_styles=field_styles, | ||||
|             fmt=LOG_FORMAT_DEBUG | ||||
|         ) | ||||
|         coloredlogs.install(level=log_level, level_styles=level_styles, field_styles=field_styles, fmt=LOG_FORMAT_DEBUG) | ||||
|     else: | ||||
|         logging.basicConfig(level=logging.getLevelName(log_level), format=LOG_FORMAT_DEBUG) | ||||
| 
 | ||||
|  |  | |||
|  | @ -13,8 +13,7 @@ random_string_letters = string.ascii_lowercase + string.digits + string.ascii_up | |||
| 
 | ||||
| 
 | ||||
| def random_characters(): | ||||
|     return [random.choice(random_string_letters) | ||||
|             for _ in range(random.randint(8, 32))] | ||||
|     return [random.choice(random_string_letters) for _ in range(random.randint(8, 32))] | ||||
| 
 | ||||
| 
 | ||||
| def random_string(): | ||||
|  | @ -39,11 +38,13 @@ def random_uuid(): | |||
|     return str(uuid.uuid4()) | ||||
| 
 | ||||
| 
 | ||||
| random_types = {'string': random_string, | ||||
| random_types = { | ||||
|     'string': random_string, | ||||
|     'int': random_int, | ||||
|     'float': random_float, | ||||
|     'sha256': random_sha256, | ||||
|                 'uuid': random_uuid} | ||||
|     'uuid': random_uuid, | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
| # required answerer function | ||||
|  | @ -62,6 +63,8 @@ def answer(query): | |||
| # required answerer function | ||||
| # returns information about the answerer | ||||
| def self_info(): | ||||
|     return {'name': gettext('Random value generator'), | ||||
|     return { | ||||
|         'name': gettext('Random value generator'), | ||||
|         'description': gettext('Generate different random values'), | ||||
|             'examples': ['random {}'.format(x) for x in random_types]} | ||||
|         'examples': ['random {}'.format(x) for x in random_types], | ||||
|     } | ||||
|  |  | |||
|  | @ -4,11 +4,7 @@ from operator import mul | |||
| from flask_babel import gettext | ||||
| 
 | ||||
| 
 | ||||
| keywords = ('min', | ||||
|             'max', | ||||
|             'avg', | ||||
|             'sum', | ||||
|             'prod') | ||||
| keywords = ('min', 'max', 'avg', 'sum', 'prod') | ||||
| 
 | ||||
| 
 | ||||
| # required answerer function | ||||
|  | @ -47,6 +43,8 @@ def answer(query): | |||
| # required answerer function | ||||
| # returns information about the answerer | ||||
| def self_info(): | ||||
|     return {'name': gettext('Statistics functions'), | ||||
|     return { | ||||
|         'name': gettext('Statistics functions'), | ||||
|         'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)), | ||||
|             'examples': ['avg 123 548 2.04 24.2']} | ||||
|         'examples': ['avg 123 548 2.04 24.2'], | ||||
|     } | ||||
|  |  | |||
|  | @ -120,13 +120,14 @@ def wikipedia(query, lang): | |||
|     return [] | ||||
| 
 | ||||
| 
 | ||||
| backends = {'dbpedia': dbpedia, | ||||
| backends = { | ||||
|     'dbpedia': dbpedia, | ||||
|     'duckduckgo': duckduckgo, | ||||
|     'google': google, | ||||
|     'startpage': startpage, | ||||
|     'swisscows': swisscows, | ||||
|     'qwant': qwant, | ||||
|             'wikipedia': wikipedia | ||||
|     'wikipedia': wikipedia, | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
|  |  | |||
|  | @ -23,10 +23,12 @@ from pathlib import Path | |||
| 
 | ||||
| data_dir = Path(__file__).parent | ||||
| 
 | ||||
| 
 | ||||
| def _load(filename): | ||||
|     with open(data_dir / filename, encoding='utf-8') as f: | ||||
|         return json.load(f) | ||||
| 
 | ||||
| 
 | ||||
| def ahmia_blacklist_loader(): | ||||
|     """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion | ||||
|     names.  The MD5 values are fetched by:: | ||||
|  | @ -39,6 +41,7 @@ def ahmia_blacklist_loader(): | |||
|     with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f: | ||||
|         return f.read().split() | ||||
| 
 | ||||
| 
 | ||||
| ENGINES_LANGUAGES = _load('engines_languages.json') | ||||
| CURRENCIES = _load('currencies.json') | ||||
| USER_AGENTS = _load('useragents.json') | ||||
|  |  | |||
|  | @ -43,11 +43,15 @@ def response(resp): | |||
|         filesize, filesize_multiplier = filesize_info.split() | ||||
|         filesize = get_torrent_size(filesize, filesize_multiplier) | ||||
| 
 | ||||
|         results.append({'url': href, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': href, | ||||
|                 'title': title, | ||||
|                 'seed': seed, | ||||
|                 'leech': leech, | ||||
|                 'filesize': filesize, | ||||
|                         'template': 'torrent.html'}) | ||||
|                 'template': 'torrent.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -57,6 +57,7 @@ engine_shortcuts = {} | |||
| 
 | ||||
| """ | ||||
| 
 | ||||
| 
 | ||||
| def load_engine(engine_data): | ||||
|     """Load engine from ``engine_data``. | ||||
| 
 | ||||
|  | @ -166,20 +167,19 @@ def set_language_attributes(engine): | |||
|         # settings.yml | ||||
|         if engine.language not in engine.supported_languages: | ||||
|             raise ValueError( | ||||
|                 "settings.yml - engine: '%s' / language: '%s' not supported" % ( | ||||
|                     engine.name, engine.language )) | ||||
|                 "settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language) | ||||
|             ) | ||||
| 
 | ||||
|         if isinstance(engine.supported_languages, dict): | ||||
|             engine.supported_languages = { | ||||
|                 engine.language : engine.supported_languages[engine.language] | ||||
|             } | ||||
|             engine.supported_languages = {engine.language: engine.supported_languages[engine.language]} | ||||
|         else: | ||||
|             engine.supported_languages = [engine.language] | ||||
| 
 | ||||
|     # find custom aliases for non standard language codes | ||||
|     for engine_lang in engine.supported_languages: | ||||
|         iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None) | ||||
|         if (iso_lang | ||||
|         if ( | ||||
|             iso_lang | ||||
|             and iso_lang != engine_lang | ||||
|             and not engine_lang.startswith(iso_lang) | ||||
|             and iso_lang not in engine.supported_languages | ||||
|  | @ -197,14 +197,12 @@ def set_language_attributes(engine): | |||
|         } | ||||
|         engine.fetch_supported_languages = ( | ||||
|             # pylint: disable=protected-access | ||||
|             lambda: engine._fetch_supported_languages( | ||||
|                 get(engine.supported_languages_url, headers=headers)) | ||||
|             lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers)) | ||||
|         ) | ||||
| 
 | ||||
| 
 | ||||
| def update_attributes_for_tor(engine): | ||||
|     if (settings['outgoing'].get('using_tor_proxy') | ||||
|         and hasattr(engine, 'onion_url') ): | ||||
|     if settings['outgoing'].get('using_tor_proxy') and hasattr(engine, 'onion_url'): | ||||
|         engine.search_url = engine.onion_url + getattr(engine, 'search_path', '') | ||||
|         engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0) | ||||
| 
 | ||||
|  | @ -217,9 +215,7 @@ def is_missing_required_attributes(engine): | |||
|     missing = False | ||||
|     for engine_attr in dir(engine): | ||||
|         if not engine_attr.startswith('_') and getattr(engine, engine_attr) is None: | ||||
|             logger.error( | ||||
|                 'Missing engine config attribute: "{0}.{1}"' | ||||
|                 .format(engine.name, engine_attr)) | ||||
|             logger.error('Missing engine config attribute: "{0}.{1}"'.format(engine.name, engine_attr)) | ||||
|             missing = True | ||||
|     return missing | ||||
| 
 | ||||
|  | @ -230,8 +226,7 @@ def is_engine_active(engine): | |||
|         return False | ||||
| 
 | ||||
|     # exclude onion engines if not using tor | ||||
|     if ('onions' in engine.categories | ||||
|         and not settings['outgoing'].get('using_tor_proxy') ): | ||||
|     if 'onions' in engine.categories and not settings['outgoing'].get('using_tor_proxy'): | ||||
|         return False | ||||
| 
 | ||||
|     return True | ||||
|  | @ -253,8 +248,7 @@ def register_engine(engine): | |||
| 
 | ||||
| 
 | ||||
| def load_engines(engine_list): | ||||
|     """usage: ``engine_list = settings['engines']`` | ||||
|     """ | ||||
|     """usage: ``engine_list = settings['engines']``""" | ||||
|     engines.clear() | ||||
|     engine_shortcuts.clear() | ||||
|     categories.clear() | ||||
|  |  | |||
|  | @ -25,9 +25,7 @@ page_size = 10 | |||
| # search url | ||||
| search_url = 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/search/?{query}' | ||||
| time_range_support = True | ||||
| time_range_dict = {'day': 1, | ||||
|                    'week': 7, | ||||
|                    'month': 30} | ||||
| time_range_dict = {'day': 1, 'week': 7, 'month': 30} | ||||
| 
 | ||||
| # xpaths | ||||
| results_xpath = '//li[@class="result"]' | ||||
|  | @ -65,10 +63,7 @@ def response(resp): | |||
|         title = extract_text(eval_xpath(result, title_xpath)) | ||||
|         content = extract_text(eval_xpath(result, content_xpath)) | ||||
| 
 | ||||
|         results.append({'url': cleaned_url, | ||||
|                         'title': title, | ||||
|                         'content': content, | ||||
|                         'is_onion': True}) | ||||
|         results.append({'url': cleaned_url, 'title': title, 'content': content, 'is_onion': True}) | ||||
| 
 | ||||
|     # get spelling corrections | ||||
|     for correction in eval_xpath_list(dom, correction_xpath): | ||||
|  |  | |||
|  | @ -55,11 +55,7 @@ def response(resp): | |||
|         url = base_url + link.attrib.get('href') + '#downloads' | ||||
|         title = extract_text(link) | ||||
|         img_src = base_url + eval_xpath_getindex(result, './/img/@src', 0) | ||||
|         res = { | ||||
|             'url': url, | ||||
|             'title': title, | ||||
|             'img_src': img_src | ||||
|         } | ||||
|         res = {'url': url, 'title': title, 'img_src': img_src} | ||||
| 
 | ||||
|         results.append(res) | ||||
| 
 | ||||
|  |  | |||
|  | @ -97,7 +97,7 @@ main_langs = { | |||
|     'sl': 'Slovenský', | ||||
|     'th': 'ไทย', | ||||
|     'uk': 'Українська', | ||||
|     'zh': '简体中文' | ||||
|     'zh': '简体中文', | ||||
| } | ||||
| supported_languages = dict(lang_urls, **main_langs) | ||||
| 
 | ||||
|  | @ -141,7 +141,6 @@ def response(resp): | |||
|         href = urljoin(base_url, link.attrib.get('href')) | ||||
|         title = extract_text(link) | ||||
| 
 | ||||
|         results.append({'url': href, | ||||
|                         'title': title}) | ||||
|         results.append({'url': href, 'title': title}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -27,19 +27,23 @@ nb_per_page = 20 | |||
| search_api = 'https://api.artic.edu/api/v1/artworks/search?' | ||||
| image_api = 'https://www.artic.edu/iiif/2/' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     args = urlencode({ | ||||
|     args = urlencode( | ||||
|         { | ||||
|             'q': query, | ||||
|             'page': params['pageno'], | ||||
|             'fields': 'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles', | ||||
|             'limit': nb_per_page, | ||||
|         }) | ||||
|         } | ||||
|     ) | ||||
|     params['url'] = search_api + args | ||||
| 
 | ||||
|     logger.debug("query_url --> %s", params['url']) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     results = [] | ||||
|  | @ -50,14 +54,16 @@ def response(resp): | |||
|         if not result['image_id']: | ||||
|             continue | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': 'https://artic.edu/artworks/%(id)s' % result, | ||||
|                 'title': result['title'] + " (%(date_display)s) //  %(artist_display)s" % result, | ||||
|                 'content': result['medium_display'], | ||||
|                 'author': ', '.join(result['artist_titles']), | ||||
|                 'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result, | ||||
|                 'img_format': result['dimensions'], | ||||
|             'template': 'images.html' | ||||
|         }) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -20,8 +20,9 @@ about = { | |||
| categories = ['science'] | ||||
| paging = True | ||||
| 
 | ||||
| base_url = 'https://export.arxiv.org/api/query?search_query=all:'\ | ||||
|            + '{query}&start={offset}&max_results={number_of_results}' | ||||
| base_url = ( | ||||
|     'https://export.arxiv.org/api/query?search_query=all:' + '{query}&start={offset}&max_results={number_of_results}' | ||||
| ) | ||||
| 
 | ||||
| # engine dependent config | ||||
| number_of_results = 10 | ||||
|  | @ -31,9 +32,7 @@ def request(query, params): | |||
|     # basic search | ||||
|     offset = (params['pageno'] - 1) * number_of_results | ||||
| 
 | ||||
|     string_args = dict(query=query, | ||||
|                        offset=offset, | ||||
|                        number_of_results=number_of_results) | ||||
|     string_args = dict(query=query, offset=offset, number_of_results=number_of_results) | ||||
| 
 | ||||
|     params['url'] = base_url.format(**string_args) | ||||
| 
 | ||||
|  | @ -65,10 +64,7 @@ def response(resp): | |||
| 
 | ||||
|         publishedDate = datetime.strptime(eval_xpath_getindex(entry, './/published', 0).text, '%Y-%m-%dT%H:%M:%SZ') | ||||
| 
 | ||||
|         res_dict = {'url': url, | ||||
|                     'title': title, | ||||
|                     'publishedDate': publishedDate, | ||||
|                     'content': content} | ||||
|         res_dict = {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content} | ||||
| 
 | ||||
|         results.append(res_dict) | ||||
| 
 | ||||
|  |  | |||
|  | @ -44,9 +44,7 @@ def request(query, params): | |||
|       pageno  : 1 # number of the requested page | ||||
|     ''' | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query=urlencode({'q': query}), | ||||
|         page=params['pageno']) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), page=params['pageno']) | ||||
| 
 | ||||
|     params['url'] = base_url + search_path | ||||
| 
 | ||||
|  |  | |||
|  | @ -21,8 +21,10 @@ about = { | |||
| 
 | ||||
| categories = ['science'] | ||||
| 
 | ||||
| base_url = 'https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi'\ | ||||
| base_url = ( | ||||
|     'https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi' | ||||
|     + '?func=PerformSearch&{query}&boost=oa&hits={hits}&offset={offset}' | ||||
| ) | ||||
| 
 | ||||
| # engine dependent config | ||||
| paging = True | ||||
|  | @ -47,7 +49,7 @@ shorcut_dict = { | |||
|     'source:': 'dcsource:', | ||||
|     'subject:': 'dcsubject:', | ||||
|     'title:': 'dctitle:', | ||||
|     'type:': 'dcdctype:' | ||||
|     'type:': 'dcdctype:', | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
|  | @ -59,9 +61,7 @@ def request(query, params): | |||
|     # basic search | ||||
|     offset = (params['pageno'] - 1) * number_of_results | ||||
| 
 | ||||
|     string_args = dict(query=urlencode({'query': query}), | ||||
|                        offset=offset, | ||||
|                        hits=number_of_results) | ||||
|     string_args = dict(query=urlencode({'query': query}), offset=offset, hits=number_of_results) | ||||
| 
 | ||||
|     params['url'] = base_url.format(**string_args) | ||||
| 
 | ||||
|  | @ -103,14 +103,9 @@ def response(resp): | |||
|                 pass | ||||
| 
 | ||||
|         if publishedDate is not None: | ||||
|             res_dict = {'url': url, | ||||
|                         'title': title, | ||||
|                         'publishedDate': publishedDate, | ||||
|                         'content': content} | ||||
|             res_dict = {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content} | ||||
|         else: | ||||
|             res_dict = {'url': url, | ||||
|                         'title': title, | ||||
|                         'content': content} | ||||
|             res_dict = {'url': url, 'title': title, 'content': content} | ||||
| 
 | ||||
|         results.append(res_dict) | ||||
| 
 | ||||
|  |  | |||
|  | @ -36,9 +36,11 @@ inital_query = 'search?{query}&search=&form=QBLH' | |||
| # following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE | ||||
| page_query = 'search?{query}&search=&first={offset}&FORM=PERE' | ||||
| 
 | ||||
| 
 | ||||
| def _get_offset_from_pageno(pageno): | ||||
|     return (pageno - 1) * 10 + 1 | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     offset = _get_offset_from_pageno(params.get('pageno', 1)) | ||||
|  | @ -53,17 +55,11 @@ def request(query, params): | |||
|     if params['language'] == 'all': | ||||
|         lang = 'EN' | ||||
|     else: | ||||
|         lang = match_language( | ||||
|             params['language'], supported_languages, language_aliases | ||||
|         ) | ||||
|         lang = match_language(params['language'], supported_languages, language_aliases) | ||||
| 
 | ||||
|     query = 'language:{} {}'.format( | ||||
|         lang.split('-')[0].upper(), query | ||||
|     ) | ||||
|     query = 'language:{} {}'.format(lang.split('-')[0].upper(), query) | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query = urlencode({'q': query}), | ||||
|         offset = offset) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), offset=offset) | ||||
| 
 | ||||
|     if offset > 1: | ||||
|         referer = base_url + inital_query.format(query=urlencode({'q': query})) | ||||
|  | @ -72,11 +68,10 @@ def request(query, params): | |||
| 
 | ||||
|     params['url'] = base_url + search_path | ||||
|     params['headers']['Accept-Language'] = "en-US,en;q=0.5" | ||||
|     params['headers']['Accept'] = ( | ||||
|         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     ) | ||||
|     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     results = [] | ||||
|  | @ -95,11 +90,7 @@ def response(resp): | |||
|         content = extract_text(eval_xpath(result, './/p')) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({ | ||||
|             'url': url, | ||||
|             'title': title, | ||||
|             'content': content | ||||
|         }) | ||||
|         results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     # parse results again if nothing is found yet | ||||
|     for result in eval_xpath(dom, '//li[@class="b_algo"]'): | ||||
|  | @ -110,11 +101,7 @@ def response(resp): | |||
|         content = extract_text(eval_xpath(result, './/p')) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({ | ||||
|             'url': url, | ||||
|             'title': title, | ||||
|             'content': content | ||||
|         }) | ||||
|         results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     try: | ||||
|         result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()')) | ||||
|  |  | |||
|  | @ -9,7 +9,10 @@ from json import loads | |||
| from searx.utils import match_language | ||||
| 
 | ||||
| from searx.engines.bing import language_aliases | ||||
| from searx.engines.bing import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import | ||||
| from searx.engines.bing import ( | ||||
|     _fetch_supported_languages, | ||||
|     supported_languages_url, | ||||
| )  # NOQA # pylint: disable=unused-import | ||||
| 
 | ||||
| # about | ||||
| about = { | ||||
|  | @ -31,39 +34,25 @@ number_of_results = 28 | |||
| 
 | ||||
| # search-url | ||||
| base_url = 'https://www.bing.com/' | ||||
| search_string = 'images/search'\ | ||||
|     '?{query}'\ | ||||
|     '&count={count}'\ | ||||
|     '&first={first}'\ | ||||
|     '&tsc=ImageHoverTitle' | ||||
| search_string = 'images/search' '?{query}' '&count={count}' '&first={first}' '&tsc=ImageHoverTitle' | ||||
| time_range_string = '&qft=+filterui:age-lt{interval}' | ||||
| time_range_dict = {'day': '1440', | ||||
|                    'week': '10080', | ||||
|                    'month': '43200', | ||||
|                    'year': '525600'} | ||||
| time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'} | ||||
| 
 | ||||
| # safesearch definitions | ||||
| safesearch_types = {2: 'STRICT', | ||||
|                     1: 'DEMOTE', | ||||
|                     0: 'OFF'} | ||||
| safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     offset = ((params['pageno'] - 1) * number_of_results) + 1 | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query=urlencode({'q': query}), | ||||
|         count=number_of_results, | ||||
|         first=offset) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset) | ||||
| 
 | ||||
|     language = match_language(params['language'], supported_languages, language_aliases).lower() | ||||
| 
 | ||||
|     params['cookies']['SRCHHPGUSR'] = \ | ||||
|         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE') | ||||
|     params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE') | ||||
| 
 | ||||
|     params['cookies']['_EDGE_S'] = 'mkt=' + language +\ | ||||
|         '&ui=' + language + '&F=1' | ||||
|     params['cookies']['_EDGE_S'] = 'mkt=' + language + '&ui=' + language + '&F=1' | ||||
| 
 | ||||
|     params['url'] = base_url + search_path | ||||
|     if params['time_range'] in time_range_dict: | ||||
|  | @ -92,14 +81,18 @@ def response(resp): | |||
|             # strip 'Unicode private use area' highlighting, they render to Tux | ||||
|             # the Linux penguin and a standing diamond on my machine... | ||||
|             title = m.get('t', '').replace('\ue000', '').replace('\ue001', '') | ||||
|             results.append({'template': 'images.html', | ||||
|             results.append( | ||||
|                 { | ||||
|                     'template': 'images.html', | ||||
|                     'url': m['purl'], | ||||
|                     'thumbnail_src': m['turl'], | ||||
|                     'img_src': m['murl'], | ||||
|                     'content': '', | ||||
|                     'title': title, | ||||
|                     'source': source, | ||||
|                             'img_format': img_format}) | ||||
|                     'img_format': img_format, | ||||
|                 } | ||||
|             ) | ||||
|         except: | ||||
|             continue | ||||
| 
 | ||||
|  |  | |||
|  | @ -13,10 +13,7 @@ from datetime import datetime | |||
| from dateutil import parser | ||||
| from lxml import etree | ||||
| from lxml.etree import XPath | ||||
| from searx.utils import ( | ||||
|     match_language, | ||||
|     eval_xpath_getindex | ||||
| ) | ||||
| from searx.utils import match_language, eval_xpath_getindex | ||||
| from searx.engines.bing import (  # pylint: disable=unused-import | ||||
|     language_aliases, | ||||
|     _fetch_supported_languages, | ||||
|  | @ -42,11 +39,8 @@ time_range_support = True | |||
| base_url = 'https://www.bing.com/' | ||||
| search_string = 'news/search?{query}&first={offset}&format=RSS' | ||||
| search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS' | ||||
| time_range_dict = { | ||||
|     'day': '7', | ||||
|     'week': '8', | ||||
|     'month': '9' | ||||
| } | ||||
| time_range_dict = {'day': '7', 'week': '8', 'month': '9'} | ||||
| 
 | ||||
| 
 | ||||
| def url_cleanup(url_string): | ||||
|     """remove click""" | ||||
|  | @ -57,6 +51,7 @@ def url_cleanup(url_string): | |||
|         url_string = query.get('url', None) | ||||
|     return url_string | ||||
| 
 | ||||
| 
 | ||||
| def image_url_cleanup(url_string): | ||||
|     """replace the http://*bing.com/th?id=... by https://www.bing.com/th?id=...""" | ||||
| 
 | ||||
|  | @ -66,6 +61,7 @@ def image_url_cleanup(url_string): | |||
|         url_string = "https://www.bing.com/th?id=" + quote(query.get('id')) | ||||
|     return url_string | ||||
| 
 | ||||
| 
 | ||||
| def _get_url(query, language, offset, time_range): | ||||
|     if time_range in time_range_dict: | ||||
|         search_path = search_string_with_time.format( | ||||
|  | @ -91,6 +87,7 @@ def _get_url(query, language, offset, time_range): | |||
|         ) | ||||
|     return base_url + search_path | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     if params['time_range'] and params['time_range'] not in time_range_dict: | ||||
|  | @ -105,6 +102,7 @@ def request(query, params): | |||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     results = [] | ||||
|  | @ -127,26 +125,16 @@ def response(resp): | |||
|             publishedDate = datetime.now() | ||||
| 
 | ||||
|         # thumbnail | ||||
|         thumbnail = eval_xpath_getindex( | ||||
|             item, XPath('./News:Image/text()', namespaces=namespaces), 0, default=None) | ||||
|         thumbnail = eval_xpath_getindex(item, XPath('./News:Image/text()', namespaces=namespaces), 0, default=None) | ||||
|         if thumbnail is not None: | ||||
|             thumbnail = image_url_cleanup(thumbnail) | ||||
| 
 | ||||
|         # append result | ||||
|         if thumbnail is not None: | ||||
|             results.append({ | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'publishedDate': publishedDate, | ||||
|                 'content': content, | ||||
|                 'img_src': thumbnail | ||||
|             }) | ||||
|             results.append( | ||||
|                 {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content, 'img_src': thumbnail} | ||||
|             ) | ||||
|         else: | ||||
|             results.append({ | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'publishedDate': publishedDate, | ||||
|                 'content': content | ||||
|             }) | ||||
|             results.append({'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -9,7 +9,10 @@ from urllib.parse import urlencode | |||
| from searx.utils import match_language | ||||
| 
 | ||||
| from searx.engines.bing import language_aliases | ||||
| from searx.engines.bing import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import | ||||
| from searx.engines.bing import ( | ||||
|     _fetch_supported_languages, | ||||
|     supported_languages_url, | ||||
| )  # NOQA # pylint: disable=unused-import | ||||
| 
 | ||||
| # about | ||||
| about = { | ||||
|  | @ -28,36 +31,22 @@ time_range_support = True | |||
| number_of_results = 28 | ||||
| 
 | ||||
| base_url = 'https://www.bing.com/' | ||||
| search_string = 'videos/search'\ | ||||
|     '?{query}'\ | ||||
|     '&count={count}'\ | ||||
|     '&first={first}'\ | ||||
|     '&scope=video'\ | ||||
|     '&FORM=QBLH' | ||||
| search_string = 'videos/search' '?{query}' '&count={count}' '&first={first}' '&scope=video' '&FORM=QBLH' | ||||
| time_range_string = '&qft=+filterui:videoage-lt{interval}' | ||||
| time_range_dict = {'day': '1440', | ||||
|                    'week': '10080', | ||||
|                    'month': '43200', | ||||
|                    'year': '525600'} | ||||
| time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'} | ||||
| 
 | ||||
| # safesearch definitions | ||||
| safesearch_types = {2: 'STRICT', | ||||
|                     1: 'DEMOTE', | ||||
|                     0: 'OFF'} | ||||
| safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     offset = ((params['pageno'] - 1) * number_of_results) + 1 | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query=urlencode({'q': query}), | ||||
|         count=number_of_results, | ||||
|         first=offset) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset) | ||||
| 
 | ||||
|     # safesearch cookie | ||||
|     params['cookies']['SRCHHPGUSR'] = \ | ||||
|         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE') | ||||
|     params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE') | ||||
| 
 | ||||
|     # language cookie | ||||
|     language = match_language(params['language'], supported_languages, language_aliases).lower() | ||||
|  | @ -89,11 +78,15 @@ def response(resp): | |||
|             info = ' - '.join(result.xpath('.//div[@class="mc_vtvc_meta_block"]//span/text()')).strip() | ||||
|             content = '{0} - {1}'.format(metadata['du'], info) | ||||
|             thumbnail = '{0}th?id={1}'.format(base_url, metadata['thid']) | ||||
|             results.append({'url': metadata['murl'], | ||||
|             results.append( | ||||
|                 { | ||||
|                     'url': metadata['murl'], | ||||
|                     'thumbnail': thumbnail, | ||||
|                     'title': metadata.get('vt', ''), | ||||
|                     'content': content, | ||||
|                             'template': 'videos.html'}) | ||||
|                     'template': 'videos.html', | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|         except: | ||||
|             continue | ||||
|  |  | |||
|  | @ -11,10 +11,7 @@ from searx.utils import extract_text, get_torrent_size | |||
| about = { | ||||
|     "website": 'https://btdig.com', | ||||
|     "wikidata_id": 'Q4836698', | ||||
|     "official_api_documentation": { | ||||
|         'url': 'https://btdig.com/contacts', | ||||
|         'comment': 'on demand' | ||||
|     }, | ||||
|     "official_api_documentation": {'url': 'https://btdig.com/contacts', 'comment': 'on demand'}, | ||||
|     "use_official_api": False, | ||||
|     "require_api_key": False, | ||||
|     "results": 'HTML', | ||||
|  | @ -31,8 +28,7 @@ search_url = url + '/search?q={search_term}&p={pageno}' | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     params['url'] = search_url.format(search_term=quote(query), | ||||
|                                       pageno=params['pageno'] - 1) | ||||
|     params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'] - 1) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -77,13 +73,17 @@ def response(resp): | |||
|         magnetlink = result.xpath('.//div[@class="torrent_magnet"]//a')[0].attrib['href'] | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': href, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': href, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'filesize': filesize, | ||||
|                 'files': files, | ||||
|                 'magnetlink': magnetlink, | ||||
|                         'template': 'torrent.html'}) | ||||
|                 'template': 'torrent.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results sorted by seeder | ||||
|     return results | ||||
|  |  | |||
|  | @ -29,10 +29,7 @@ search_string = '&page={page}&page_size={nb_per_page}&format=json&{query}' | |||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query=urlencode({'q': query}), | ||||
|         nb_per_page=nb_per_page, | ||||
|         page=params['pageno']) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), nb_per_page=nb_per_page, page=params['pageno']) | ||||
| 
 | ||||
|     params['url'] = base_url + search_path | ||||
| 
 | ||||
|  | @ -45,9 +42,13 @@ def response(resp): | |||
|     json_data = loads(resp.text) | ||||
| 
 | ||||
|     for result in json_data['results']: | ||||
|         results.append({'url': result['foreign_landing_url'], | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': result['foreign_landing_url'], | ||||
|                 'title': result['title'], | ||||
|                 'img_src': result['url'], | ||||
|                         'template': 'images.html'}) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -28,6 +28,7 @@ api_key = 'unset' | |||
| base_url = 'https://core.ac.uk:443/api-v2/search/' | ||||
| search_string = '{query}?page={page}&pageSize={nb_per_page}&apiKey={apikey}' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     if api_key == 'unset': | ||||
|  | @ -44,6 +45,7 @@ def request(query, params): | |||
|     logger.debug("query_url --> %s", params['url']) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     results = [] | ||||
|     json_data = loads(resp.text) | ||||
|  | @ -66,12 +68,14 @@ def response(resp): | |||
|             metadata.append(source['doi']) | ||||
|         metadata = ' / '.join(metadata) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': source['urls'][0].replace('http://', 'https://', 1), | ||||
|                 'title': source['title'], | ||||
|                 'content': source['description'], | ||||
|                 'publishedDate': date, | ||||
|                 'metadata': metadata, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -47,7 +47,8 @@ def response(resp): | |||
|     ) | ||||
| 
 | ||||
|     url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'.format( | ||||
|         resp.search_params['from'].upper(), resp.search_params['to']) | ||||
|         resp.search_params['from'].upper(), resp.search_params['to'] | ||||
|     ) | ||||
| 
 | ||||
|     results.append({'answer': answer, 'url': url}) | ||||
| 
 | ||||
|  |  | |||
|  | @ -25,8 +25,10 @@ paging = True | |||
| # search-url | ||||
| # see http://www.dailymotion.com/doc/api/obj-video.html | ||||
| search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,description,duration,url,thumbnail_360_url,id&sort=relevance&limit=5&page={pageno}&{query}'  # noqa | ||||
| embedded_url = '<iframe frameborder="0" width="540" height="304" ' +\ | ||||
|     'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>' | ||||
| embedded_url = ( | ||||
|     '<iframe frameborder="0" width="540" height="304" ' | ||||
|     + 'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>' | ||||
| ) | ||||
| 
 | ||||
| supported_languages_url = 'https://api.dailymotion.com/languages' | ||||
| 
 | ||||
|  | @ -39,8 +41,8 @@ def request(query, params): | |||
|         locale = match_language(params['language'], supported_languages) | ||||
| 
 | ||||
|     params['url'] = search_url.format( | ||||
|         query=urlencode({'search': query, 'localization': locale}), | ||||
|         pageno=params['pageno']) | ||||
|         query=urlencode({'search': query, 'localization': locale}), pageno=params['pageno'] | ||||
|     ) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -67,13 +69,17 @@ def response(resp): | |||
|         # http to https | ||||
|         thumbnail = thumbnail.replace("http://", "https://") | ||||
| 
 | ||||
|         results.append({'template': 'videos.html', | ||||
|         results.append( | ||||
|             { | ||||
|                 'template': 'videos.html', | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'publishedDate': publishedDate, | ||||
|                 'embedded': embedded, | ||||
|                         'thumbnail': thumbnail}) | ||||
|                 'thumbnail': thumbnail, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -24,9 +24,11 @@ paging = True | |||
| url = 'https://api.deezer.com/' | ||||
| search_url = url + 'search?{query}&index={offset}' | ||||
| 
 | ||||
| embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' +\ | ||||
|     'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" ' +\ | ||||
|     'width="540" height="80"></iframe>' | ||||
| embedded_url = ( | ||||
|     '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' | ||||
|     + 'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" ' | ||||
|     + 'width="540" height="80"></iframe>' | ||||
| ) | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
|  | @ -53,18 +55,12 @@ def response(resp): | |||
|             if url.startswith('http://'): | ||||
|                 url = 'https' + url[4:] | ||||
| 
 | ||||
|             content = '{} - {} - {}'.format( | ||||
|                 result['artist']['name'], | ||||
|                 result['album']['title'], | ||||
|                 result['title']) | ||||
|             content = '{} - {} - {}'.format(result['artist']['name'], result['album']['title'], result['title']) | ||||
| 
 | ||||
|             embedded = embedded_url.format(audioid=result['id']) | ||||
| 
 | ||||
|             # append result | ||||
|             results.append({'url': url, | ||||
|                             'title': title, | ||||
|                             'embedded': embedded, | ||||
|                             'content': content}) | ||||
|             results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -31,6 +31,7 @@ about = { | |||
| # if there is a need for globals, use a leading underline | ||||
| _my_offline_engine = None | ||||
| 
 | ||||
| 
 | ||||
| def init(engine_settings=None): | ||||
|     """Initialization of the (offline) engine.  The origin of this demo engine is a | ||||
|     simple json string which is loaded in this example while the engine is | ||||
|  | @ -44,11 +45,10 @@ def init(engine_settings=None): | |||
|         ', {"value":"first item"}' | ||||
|         ', {"value":"second item"}' | ||||
|         ', {"value":"third item"}' | ||||
|         ']' | ||||
| 
 | ||||
|         % engine_settings.get('name') | ||||
|         ']' % engine_settings.get('name') | ||||
|     ) | ||||
| 
 | ||||
| 
 | ||||
| def search(query, request_params): | ||||
|     """Query (offline) engine and return results.  Assemble the list of results from | ||||
|     your local engine.  In this demo engine we ignore the 'query' term, usual | ||||
|  |  | |||
|  | @ -43,6 +43,7 @@ about = { | |||
| # if there is a need for globals, use a leading underline | ||||
| _my_online_engine = None | ||||
| 
 | ||||
| 
 | ||||
| def init(engine_settings): | ||||
|     """Initialization of the (online) engine.  If no initialization is needed, drop | ||||
|     this init function. | ||||
|  | @ -51,20 +52,24 @@ def init(engine_settings): | |||
|     global _my_online_engine  # pylint: disable=global-statement | ||||
|     _my_online_engine = engine_settings.get('name') | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     """Build up the ``params`` for the online request.  In this example we build a | ||||
|     URL to fetch images from `artic.edu <https://artic.edu>`__ | ||||
| 
 | ||||
|     """ | ||||
|     args = urlencode({ | ||||
|     args = urlencode( | ||||
|         { | ||||
|             'q': query, | ||||
|             'page': params['pageno'], | ||||
|             'fields': 'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles', | ||||
|             'limit': page_size, | ||||
|         }) | ||||
|         } | ||||
|     ) | ||||
|     params['url'] = search_api + args | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     """Parse out the result items from the response.  In this example we parse the | ||||
|     response from `api.artic.edu <https://artic.edu>`__ and filter out all | ||||
|  | @ -79,14 +84,16 @@ def response(resp): | |||
|         if not result['image_id']: | ||||
|             continue | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': 'https://artic.edu/artworks/%(id)s' % result, | ||||
|                 'title': result['title'] + " (%(date_display)s) //  %(artist_display)s" % result, | ||||
|                 'content': result['medium_display'], | ||||
|                 'author': ', '.join(result['artist_titles']), | ||||
|                 'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result, | ||||
|                 'img_format': result['dimensions'], | ||||
|             'template': 'images.html' | ||||
|         }) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -32,6 +32,7 @@ time_range_dict = { | |||
| # search-url | ||||
| base_url = 'https://www.deviantart.com' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     # https://www.deviantart.com/search/deviations?page=5&q=foo | ||||
|  | @ -47,6 +48,7 @@ def request(query, params): | |||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     results = [] | ||||
|  | @ -67,11 +69,13 @@ def response(resp): | |||
|                 continue | ||||
|             img_tag = img_tag[0] | ||||
| 
 | ||||
|             results.append({ | ||||
|             results.append( | ||||
|                 { | ||||
|                     'template': 'images.html', | ||||
|                     'url': a_tag.attrib.get('href'), | ||||
|                     'img_src': img_tag.attrib.get('src'), | ||||
|                     'title': img_tag.attrib.get('alt'), | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -27,9 +27,7 @@ https_support = True | |||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = url.format(from_lang=params['from_lang'][2], | ||||
|                                to_lang=params['to_lang'][2], | ||||
|                                query=params['query']) | ||||
|     params['url'] = url.format(from_lang=params['from_lang'][2], to_lang=params['to_lang'][2], query=params['query']) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -51,10 +49,12 @@ def response(resp): | |||
|             if t.strip(): | ||||
|                 to_results.append(to_result.text_content()) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': urljoin(str(resp.url), '?%d' % k), | ||||
|                 'title': from_result.text_content(), | ||||
|             'content': '; '.join(to_results) | ||||
|         }) | ||||
|                 'content': '; '.join(to_results), | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -48,13 +48,17 @@ def response(resp): | |||
|         filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER]) | ||||
|         magnetlink = result.xpath('.//div[@class="tail"]//a[@class="title"]/@href')[0] | ||||
| 
 | ||||
|         results.append({'url': url, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'filesize': filesize, | ||||
|                 'magnetlink': magnetlink, | ||||
|                 'seed': 'N/A', | ||||
|                 'leech': 'N/A', | ||||
|                         'template': 'torrent.html'}) | ||||
|                 'template': 'torrent.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -23,6 +23,7 @@ paging = True | |||
| base_url = "https://hub.docker.com/" | ||||
| search_url = base_url + "api/content/v1/products/search?{query}&type=image&page_size=25" | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     params['url'] = search_url.format(query=urlencode(dict(q=query, page=params["pageno"]))) | ||||
|  | @ -30,6 +31,7 @@ def request(query, params): | |||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     '''post-response callback | ||||
|     resp: requests response object | ||||
|  | @ -53,12 +55,8 @@ def response(resp): | |||
|                 result["url"] = base_url + "r/" + item.get('slug', "") | ||||
|             result["title"] = item.get("name") | ||||
|             result["content"] = item.get("short_description") | ||||
|             result["publishedDate"] = parser.parse( | ||||
|                 item.get("updated_at") or item.get("created_at") | ||||
|             ) | ||||
|             result["thumbnail"] = ( | ||||
|                 item["logo_url"].get("large") or item["logo_url"].get("small") | ||||
|             ) | ||||
|             result["publishedDate"] = parser.parse(item.get("updated_at") or item.get("created_at")) | ||||
|             result["thumbnail"] = item["logo_url"].get("large") or item["logo_url"].get("small") | ||||
|             results.append(result) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -25,8 +25,7 @@ number_of_results = 5 | |||
| # search-url | ||||
| # Doku is OpenSearch compatible | ||||
| base_url = 'http://localhost:8090' | ||||
| search_url = '/?do=search'\ | ||||
|              '&{query}' | ||||
| search_url = '/?do=search' '&{query}' | ||||
| # TODO             '&startRecord={offset}'\ | ||||
| # TODO             '&maximumRecords={limit}'\ | ||||
| 
 | ||||
|  | @ -34,8 +33,7 @@ search_url = '/?do=search'\ | |||
| # do search-request | ||||
| def request(query, params): | ||||
| 
 | ||||
|     params['url'] = base_url +\ | ||||
|         search_url.format(query=urlencode({'id': query})) | ||||
|     params['url'] = base_url + search_url.format(query=urlencode({'id': query})) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -60,9 +58,7 @@ def response(resp): | |||
|         title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title')) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'title': title, | ||||
|                         'content': "", | ||||
|                         'url': base_url + res_url}) | ||||
|         results.append({'title': title, 'content': "", 'url': base_url + res_url}) | ||||
| 
 | ||||
|     # Search results | ||||
|     for r in eval_xpath(doc, '//dl[@class="search_results"]/*'): | ||||
|  | @ -74,9 +70,7 @@ def response(resp): | |||
|                 content = extract_text(eval_xpath(r, '.')) | ||||
| 
 | ||||
|                 # append result | ||||
|                 results.append({'title': title, | ||||
|                                 'content': content, | ||||
|                                 'url': base_url + res_url}) | ||||
|                 results.append({'title': title, 'content': content, 'url': base_url + res_url}) | ||||
|         except: | ||||
|             continue | ||||
| 
 | ||||
|  |  | |||
|  | @ -39,15 +39,10 @@ language_aliases = { | |||
|     'ko': 'kr-KR', | ||||
|     'sl-SI': 'sl-SL', | ||||
|     'zh-TW': 'tzh-TW', | ||||
|     'zh-HK': 'tzh-HK' | ||||
|     'zh-HK': 'tzh-HK', | ||||
| } | ||||
| 
 | ||||
| time_range_dict = { | ||||
|     'day': 'd', | ||||
|     'week': 'w', | ||||
|     'month': 'm', | ||||
|     'year': 'y' | ||||
| } | ||||
| time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'} | ||||
| 
 | ||||
| # search-url | ||||
| url = 'https://lite.duckduckgo.com/lite' | ||||
|  | @ -118,6 +113,7 @@ def request(query, params): | |||
|     logger.debug("param cookies: %s", params['cookies']) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| # get response from search-request | ||||
| def response(resp): | ||||
| 
 | ||||
|  | @ -163,14 +159,17 @@ def response(resp): | |||
|         if td_content is None: | ||||
|             continue | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'title': a_tag.text_content(), | ||||
|                 'content': extract_text(td_content), | ||||
|                 'url': a_tag.get('href'), | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
| 
 | ||||
| # get supported languages from their site | ||||
| def _fetch_supported_languages(resp): | ||||
| 
 | ||||
|  |  | |||
|  | @ -10,7 +10,10 @@ from lxml import html | |||
| 
 | ||||
| from searx.data import WIKIDATA_UNITS | ||||
| from searx.engines.duckduckgo import language_aliases | ||||
| from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import | ||||
| from searx.engines.duckduckgo import ( | ||||
|     _fetch_supported_languages, | ||||
|     supported_languages_url, | ||||
| )  # NOQA # pylint: disable=unused-import | ||||
| from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function | ||||
| from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom | ||||
| 
 | ||||
|  | @ -24,13 +27,9 @@ about = { | |||
|     "results": 'JSON', | ||||
| } | ||||
| 
 | ||||
| URL = 'https://api.duckduckgo.com/'\ | ||||
|     + '?{query}&format=json&pretty=0&no_redirect=1&d=1' | ||||
| URL = 'https://api.duckduckgo.com/' + '?{query}&format=json&pretty=0&no_redirect=1&d=1' | ||||
| 
 | ||||
| WIKIDATA_PREFIX = [ | ||||
|     'http://www.wikidata.org/entity/', | ||||
|     'https://www.wikidata.org/entity/' | ||||
| ] | ||||
| WIKIDATA_PREFIX = ['http://www.wikidata.org/entity/', 'https://www.wikidata.org/entity/'] | ||||
| 
 | ||||
| replace_http_by_https = get_string_replaces_function({'http:': 'https:'}) | ||||
| 
 | ||||
|  | @ -61,11 +60,7 @@ def result_to_text(text, htmlResult): | |||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = URL.format(query=urlencode({'q': query})) | ||||
|     language = match_language( | ||||
|         params['language'], | ||||
|         supported_languages, | ||||
|         language_aliases | ||||
|     ) | ||||
|     language = match_language(params['language'], supported_languages, language_aliases) | ||||
|     language = language.split('-')[0] | ||||
|     params['headers']['Accept-Language'] = language | ||||
|     return params | ||||
|  | @ -127,23 +122,14 @@ def response(resp): | |||
|             firstURL = ddg_result.get('FirstURL') | ||||
|             text = ddg_result.get('Text') | ||||
|             if not is_broken_text(text): | ||||
|                 suggestion = result_to_text( | ||||
|                     text, | ||||
|                     ddg_result.get('Result') | ||||
|                 ) | ||||
|                 suggestion = result_to_text(text, ddg_result.get('Result')) | ||||
|                 if suggestion != heading and suggestion is not None: | ||||
|                     results.append({'suggestion': suggestion}) | ||||
|         elif 'Topics' in ddg_result: | ||||
|             suggestions = [] | ||||
|             relatedTopics.append({ | ||||
|                 'name': ddg_result.get('Name', ''), | ||||
|                 'suggestions': suggestions | ||||
|             }) | ||||
|             relatedTopics.append({'name': ddg_result.get('Name', ''), 'suggestions': suggestions}) | ||||
|             for topic_result in ddg_result.get('Topics', []): | ||||
|                 suggestion = result_to_text( | ||||
|                     topic_result.get('Text'), | ||||
|                     topic_result.get('Result') | ||||
|                 ) | ||||
|                 suggestion = result_to_text(topic_result.get('Text'), topic_result.get('Result')) | ||||
|                 if suggestion != heading and suggestion is not None: | ||||
|                     suggestions.append(suggestion) | ||||
| 
 | ||||
|  | @ -152,25 +138,15 @@ def response(resp): | |||
|     if abstractURL != '': | ||||
|         # add as result ? problem always in english | ||||
|         infobox_id = abstractURL | ||||
|         urls.append({ | ||||
|             'title': search_res.get('AbstractSource'), | ||||
|             'url': abstractURL, | ||||
|             'official': True | ||||
|         }) | ||||
|         results.append({ | ||||
|             'url': abstractURL, | ||||
|             'title': heading | ||||
|         }) | ||||
|         urls.append({'title': search_res.get('AbstractSource'), 'url': abstractURL, 'official': True}) | ||||
|         results.append({'url': abstractURL, 'title': heading}) | ||||
| 
 | ||||
|     # definition | ||||
|     definitionURL = search_res.get('DefinitionURL', '') | ||||
|     if definitionURL != '': | ||||
|         # add as result ? as answer ? problem always in english | ||||
|         infobox_id = definitionURL | ||||
|         urls.append({ | ||||
|             'title': search_res.get('DefinitionSource'), | ||||
|             'url': definitionURL | ||||
|         }) | ||||
|         urls.append({'title': search_res.get('DefinitionSource'), 'url': definitionURL}) | ||||
| 
 | ||||
|     # to merge with wikidata's infobox | ||||
|     if infobox_id: | ||||
|  | @ -198,10 +174,7 @@ def response(resp): | |||
|                 # * netflix_id | ||||
|                 external_url = get_external_url(data_type, data_value) | ||||
|                 if external_url is not None: | ||||
|                     urls.append({ | ||||
|                         'title': data_label, | ||||
|                         'url': external_url | ||||
|                     }) | ||||
|                     urls.append({'title': data_label, 'url': external_url}) | ||||
|                 elif data_type in ['instance', 'wiki_maps_trigger', 'google_play_artist_id']: | ||||
|                     # ignore instance: Wikidata value from "Instance Of" (Qxxxx) | ||||
|                     # ignore wiki_maps_trigger: reference to a javascript | ||||
|  | @ -211,11 +184,7 @@ def response(resp): | |||
|                     # There is already an URL for the website | ||||
|                     pass | ||||
|                 elif data_type == 'area': | ||||
|                     attributes.append({ | ||||
|                         'label': data_label, | ||||
|                         'value': area_to_str(data_value), | ||||
|                         'entity': 'P2046' | ||||
|                     }) | ||||
|                     attributes.append({'label': data_label, 'value': area_to_str(data_value), 'entity': 'P2046'}) | ||||
|                     osm_zoom = area_to_osm_zoom(data_value.get('amount')) | ||||
|                 elif data_type == 'coordinates': | ||||
|                     if data_value.get('globe') == 'http://www.wikidata.org/entity/Q2': | ||||
|  | @ -224,16 +193,9 @@ def response(resp): | |||
|                         coordinates = info | ||||
|                     else: | ||||
|                         # coordinate NOT on Earth | ||||
|                         attributes.append({ | ||||
|                             'label': data_label, | ||||
|                             'value': data_value, | ||||
|                             'entity': 'P625' | ||||
|                         }) | ||||
|                         attributes.append({'label': data_label, 'value': data_value, 'entity': 'P625'}) | ||||
|                 elif data_type == 'string': | ||||
|                     attributes.append({ | ||||
|                         'label': data_label, | ||||
|                         'value': data_value | ||||
|                     }) | ||||
|                     attributes.append({'label': data_label, 'value': data_value}) | ||||
| 
 | ||||
|             if coordinates: | ||||
|                 data_label = coordinates.get('label') | ||||
|  | @ -241,31 +203,24 @@ def response(resp): | |||
|                 latitude = data_value.get('latitude') | ||||
|                 longitude = data_value.get('longitude') | ||||
|                 url = get_earth_coordinates_url(latitude, longitude, osm_zoom) | ||||
|                 urls.append({ | ||||
|                     'title': 'OpenStreetMap', | ||||
|                     'url': url, | ||||
|                     'entity': 'P625' | ||||
|                 }) | ||||
|                 urls.append({'title': 'OpenStreetMap', 'url': url, 'entity': 'P625'}) | ||||
| 
 | ||||
|     if len(heading) > 0: | ||||
|         # TODO get infobox.meta.value where .label='article_title'    # pylint: disable=fixme | ||||
|         if image is None and len(attributes) == 0 and len(urls) == 1 and\ | ||||
|            len(relatedTopics) == 0 and len(content) == 0: | ||||
|             results.append({ | ||||
|                 'url': urls[0]['url'], | ||||
|                 'title': heading, | ||||
|                 'content': content | ||||
|             }) | ||||
|         if image is None and len(attributes) == 0 and len(urls) == 1 and len(relatedTopics) == 0 and len(content) == 0: | ||||
|             results.append({'url': urls[0]['url'], 'title': heading, 'content': content}) | ||||
|         else: | ||||
|             results.append({ | ||||
|             results.append( | ||||
|                 { | ||||
|                     'infobox': heading, | ||||
|                     'id': infobox_id, | ||||
|                     'content': content, | ||||
|                     'img_src': image, | ||||
|                     'attributes': attributes, | ||||
|                     'urls': urls, | ||||
|                 'relatedTopics': relatedTopics | ||||
|             }) | ||||
|                     'relatedTopics': relatedTopics, | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
|  |  | |||
|  | @ -7,7 +7,10 @@ from json import loads | |||
| from urllib.parse import urlencode | ||||
| from searx.exceptions import SearxEngineAPIException | ||||
| from searx.engines.duckduckgo import get_region_code | ||||
| from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import | ||||
| from searx.engines.duckduckgo import ( | ||||
|     _fetch_supported_languages, | ||||
|     supported_languages_url, | ||||
| )  # NOQA # pylint: disable=unused-import | ||||
| from searx.network import get | ||||
| 
 | ||||
| # about | ||||
|  | @ -61,10 +64,10 @@ def request(query, params): | |||
|     region_code = get_region_code(params['language'], lang_list=supported_languages) | ||||
|     if region_code: | ||||
|         params['url'] = images_url.format( | ||||
|             query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd) | ||||
|             query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd | ||||
|         ) | ||||
|     else: | ||||
|         params['url'] = images_url.format( | ||||
|             query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd) | ||||
|         params['url'] = images_url.format(query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -84,11 +87,15 @@ def response(resp): | |||
|         image = result['image'] | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'template': 'images.html', | ||||
|         results.append( | ||||
|             { | ||||
|                 'template': 'images.html', | ||||
|                 'title': title, | ||||
|                 'content': '', | ||||
|                 'thumbnail_src': thumbnail, | ||||
|                 'img_src': image, | ||||
|                         'url': url}) | ||||
|                 'url': url, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -38,7 +38,7 @@ def request(query, params): | |||
|       pageno  : 1 # number of the requested page | ||||
|     ''' | ||||
| 
 | ||||
|     offset = (params['pageno'] - 1) | ||||
|     offset = params['pageno'] - 1 | ||||
|     if offset == 0: | ||||
|         search_url_fmt = base_url + 'suchen/dudenonline/{query}' | ||||
|         params['url'] = search_url_fmt.format(query=quote(query)) | ||||
|  | @ -58,9 +58,9 @@ def response(resp): | |||
| 
 | ||||
|     dom = html.fromstring(resp.text) | ||||
| 
 | ||||
|     number_of_results_element =\ | ||||
|         eval_xpath_getindex(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()', | ||||
|                             0, default=None) | ||||
|     number_of_results_element = eval_xpath_getindex( | ||||
|         dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()', 0, default=None | ||||
|     ) | ||||
|     if number_of_results_element is not None: | ||||
|         number_of_results_string = re.sub('[^0-9]', '', number_of_results_element) | ||||
|         results.append({'number_of_results': int(number_of_results_string)}) | ||||
|  | @ -71,8 +71,6 @@ def response(resp): | |||
|         title = eval_xpath(result, 'string(.//h2/a)').strip() | ||||
|         content = extract_text(eval_xpath(result, './/p')) | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|                         'title': title, | ||||
|                         'content': content}) | ||||
|         results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -15,6 +15,8 @@ about = { | |||
| 
 | ||||
| 
 | ||||
| def search(query, request_params): | ||||
|     return [{ | ||||
|     return [ | ||||
|         { | ||||
|             'result': 'this is what you get', | ||||
|     }] | ||||
|         } | ||||
|     ] | ||||
|  |  | |||
|  | @ -58,7 +58,8 @@ def response(resp): | |||
|         if title == "": | ||||
|             continue | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|  | @ -67,7 +68,7 @@ def response(resp): | |||
|                 'source_country': source_country, | ||||
|                 'thumbnail': thumbnail, | ||||
|                 'template': 'products.html', | ||||
| 
 | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -119,9 +119,7 @@ def response(resp): | |||
|         r['template'] = 'key-value.html' | ||||
| 
 | ||||
|         if show_metadata: | ||||
|             r['metadata'] = {'index': result['_index'], | ||||
|                              'id': result['_id'], | ||||
|                              'score': result['_score']} | ||||
|             r['metadata'] = {'index': result['_index'], 'id': result['_id'], 'score': result['_score']} | ||||
| 
 | ||||
|         results.append(r) | ||||
| 
 | ||||
|  | @ -133,12 +131,10 @@ _available_query_types = { | |||
|     # https://www.elastic.co/guide/en/elasticsearch/reference/current/full-text-queries.html | ||||
|     'match': _match_query, | ||||
|     'simple_query_string': _simple_query_string_query, | ||||
| 
 | ||||
|     # Term-level queries | ||||
|     # https://www.elastic.co/guide/en/elasticsearch/reference/current/term-level-queries.html | ||||
|     'term': _term_query, | ||||
|     'terms': _terms_query, | ||||
| 
 | ||||
|     # Query JSON defined by the instance administrator. | ||||
|     'custom': _custom_query, | ||||
| } | ||||
|  |  | |||
|  | @ -22,10 +22,7 @@ paging = False | |||
| safesearch = True | ||||
| 
 | ||||
| base_url = 'https://www.etools.ch' | ||||
| search_path = '/searchAdvancedSubmit.do'\ | ||||
|     '?query={search_term}'\ | ||||
|     '&pageResults=20'\ | ||||
|     '&safeSearch={safesearch}' | ||||
| search_path = '/searchAdvancedSubmit.do' '?query={search_term}' '&pageResults=20' '&safeSearch={safesearch}' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|  | @ -49,8 +46,6 @@ def response(resp): | |||
|         title = extract_text(eval_xpath(result, './a//text()')) | ||||
|         content = extract_text(eval_xpath(result, './/div[@class="text"]//text()')) | ||||
| 
 | ||||
|         results.append({'url': url, | ||||
|                         'title': title, | ||||
|                         'content': content}) | ||||
|         results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -42,13 +42,13 @@ def response(resp): | |||
|     for app in dom.xpath('//a[@class="package-header"]'): | ||||
|         app_url = app.xpath('./@href')[0] | ||||
|         app_title = extract_text(app.xpath('./div/h4[@class="package-name"]/text()')) | ||||
|         app_content = extract_text(app.xpath('./div/div/span[@class="package-summary"]')).strip() \ | ||||
|             + ' - ' + extract_text(app.xpath('./div/div/span[@class="package-license"]')).strip() | ||||
|         app_content = ( | ||||
|             extract_text(app.xpath('./div/div/span[@class="package-summary"]')).strip() | ||||
|             + ' - ' | ||||
|             + extract_text(app.xpath('./div/div/span[@class="package-license"]')).strip() | ||||
|         ) | ||||
|         app_img_src = app.xpath('./img[@class="package-icon"]/@src')[0] | ||||
| 
 | ||||
|         results.append({'url': app_url, | ||||
|                         'title': app_title, | ||||
|                         'content': app_content, | ||||
|                         'img_src': app_img_src}) | ||||
|         results.append({'url': app_url, 'title': app_title, 'content': app_content, 'img_src': app_img_src}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -25,10 +25,12 @@ paging = True | |||
| api_key = None | ||||
| 
 | ||||
| 
 | ||||
| url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' +\ | ||||
|       '&api_key={api_key}&{text}&sort=relevance' +\ | ||||
|       '&extras=description%2C+owner_name%2C+url_o%2C+url_n%2C+url_z' +\ | ||||
|       '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}' | ||||
| url = ( | ||||
|     'https://api.flickr.com/services/rest/?method=flickr.photos.search' | ||||
|     + '&api_key={api_key}&{text}&sort=relevance' | ||||
|     + '&extras=description%2C+owner_name%2C+url_o%2C+url_n%2C+url_z' | ||||
|     + '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}' | ||||
| ) | ||||
| photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' | ||||
| 
 | ||||
| paging = True | ||||
|  | @ -39,10 +41,9 @@ def build_flickr_url(user_id, photo_id): | |||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = url.format(text=urlencode({'text': query}), | ||||
|                                api_key=api_key, | ||||
|                                nb_per_page=nb_per_page, | ||||
|                                page=params['pageno']) | ||||
|     params['url'] = url.format( | ||||
|         text=urlencode({'text': query}), api_key=api_key, nb_per_page=nb_per_page, page=params['pageno'] | ||||
|     ) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -80,13 +81,17 @@ def response(resp): | |||
|         url = build_flickr_url(photo['owner'], photo['id']) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': photo['title'], | ||||
|                 'img_src': img_src, | ||||
|                 'thumbnail_src': thumbnail_src, | ||||
|                 'content': photo['description']['_content'], | ||||
|                 'author': photo['ownername'], | ||||
|                         'template': 'images.html'}) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -30,10 +30,12 @@ image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') | |||
| 
 | ||||
| paging = True | ||||
| time_range_support = True | ||||
| time_range_dict = {'day': 60 * 60 * 24, | ||||
| time_range_dict = { | ||||
|     'day': 60 * 60 * 24, | ||||
|     'week': 60 * 60 * 24 * 7, | ||||
|     'month': 60 * 60 * 24 * 7 * 4, | ||||
|                    'year': 60 * 60 * 24 * 7 * 52} | ||||
|     'year': 60 * 60 * 24 * 7 * 52, | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
| def build_flickr_url(user_id, photo_id): | ||||
|  | @ -47,8 +49,9 @@ def _get_time_range_url(time_range): | |||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = (search_url.format(query=urlencode({'text': query}), page=params['pageno']) | ||||
|                      + _get_time_range_url(params['time_range'])) | ||||
|     params['url'] = search_url.format(query=urlencode({'text': query}), page=params['pageno']) + _get_time_range_url( | ||||
|         params['time_range'] | ||||
|     ) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -83,10 +86,9 @@ def response(resp): | |||
|         for image_size in image_sizes: | ||||
|             if image_size in photo['sizes']: | ||||
|                 img_src = photo['sizes'][image_size]['url'] | ||||
|                 img_format = 'jpg ' \ | ||||
|                     + str(photo['sizes'][image_size]['width']) \ | ||||
|                     + 'x' \ | ||||
|                     + str(photo['sizes'][image_size]['height']) | ||||
|                 img_format = ( | ||||
|                     'jpg ' + str(photo['sizes'][image_size]['width']) + 'x' + str(photo['sizes'][image_size]['height']) | ||||
|                 ) | ||||
|                 break | ||||
| 
 | ||||
|         if not img_src: | ||||
|  | @ -113,7 +115,7 @@ def response(resp): | |||
|             'thumbnail_src': thumbnail_src, | ||||
|             'source': source, | ||||
|             'img_format': img_format, | ||||
|             'template': 'images.html' | ||||
|             'template': 'images.html', | ||||
|         } | ||||
|         result['author'] = author.encode(errors='ignore').decode() | ||||
|         result['source'] = source.encode(errors='ignore').decode() | ||||
|  |  | |||
|  | @ -35,9 +35,8 @@ content_xpath = './/div[@class="content"]//p' | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     offset = (params['pageno'] - 1) | ||||
|     params['url'] = search_url.format(query=urlencode({'keys': query}), | ||||
|                                       offset=offset) | ||||
|     offset = params['pageno'] - 1 | ||||
|     params['url'] = search_url.format(query=urlencode({'keys': query}), offset=offset) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -63,10 +62,7 @@ def response(resp): | |||
|         content = escape(extract_text(result.xpath(content_xpath))) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': href, | ||||
|                         'title': title, | ||||
|                         'img_src': thumbnail, | ||||
|                         'content': content}) | ||||
|         results.append({'url': href, 'title': title, 'img_src': thumbnail, 'content': content}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -26,8 +26,7 @@ paging = True | |||
| # search url | ||||
| url = "https://freesound.org/apiv2/" | ||||
| search_url = ( | ||||
|     url | ||||
|     + "search/text/?query={query}&page={page}&fields=name,url,download,created,description,type&token={api_key}" | ||||
|     url + "search/text/?query={query}&page={page}&fields=name,url,download,created,description,type&token={api_key}" | ||||
| ) | ||||
| 
 | ||||
| embedded_url = '<audio controls><source src="{uri}" type="audio/{ftype}"></audio>' | ||||
|  |  | |||
|  | @ -10,10 +10,7 @@ from urllib.parse import urlencode | |||
| about = { | ||||
|     "website": 'https://frinkiac.com', | ||||
|     "wikidata_id": 'Q24882614', | ||||
|     "official_api_documentation": { | ||||
|         'url': None, | ||||
|         'comment': 'see https://github.com/MitchellAW/CompuGlobal' | ||||
|     }, | ||||
|     "official_api_documentation": {'url': None, 'comment': 'see https://github.com/MitchellAW/CompuGlobal'}, | ||||
|     "use_official_api": False, | ||||
|     "require_api_key": False, | ||||
|     "results": 'JSON', | ||||
|  | @ -40,12 +37,15 @@ def response(resp): | |||
|         episode = result['Episode'] | ||||
|         timestamp = result['Timestamp'] | ||||
| 
 | ||||
|         results.append({'template': 'images.html', | ||||
|                         'url': RESULT_URL.format(base=BASE, | ||||
|                                                  query=urlencode({'p': 'caption', 'e': episode, 't': timestamp})), | ||||
|         results.append( | ||||
|             { | ||||
|                 'template': 'images.html', | ||||
|                 'url': RESULT_URL.format(base=BASE, query=urlencode({'p': 'caption', 'e': episode, 't': timestamp})), | ||||
|                 'title': episode, | ||||
|                 'content': '', | ||||
|                 'thumbnail_src': THUMB_URL.format(base=BASE, episode=episode, timestamp=timestamp), | ||||
|                         'img_src': IMAGE_URL.format(base=BASE, episode=episode, timestamp=timestamp)}) | ||||
|                 'img_src': IMAGE_URL.format(base=BASE, episode=episode, timestamp=timestamp), | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -37,15 +37,12 @@ def locale_to_lang_code(locale): | |||
| # wikis for some languages were moved off from the main site, we need to make | ||||
| # requests to correct URLs to be able to get results in those languages | ||||
| lang_urls = { | ||||
|     'en': { | ||||
|         'base': 'https://wiki.gentoo.org', | ||||
|         'search': '/index.php?title=Special:Search&offset={offset}&{query}' | ||||
|     }, | ||||
|     'en': {'base': 'https://wiki.gentoo.org', 'search': '/index.php?title=Special:Search&offset={offset}&{query}'}, | ||||
|     'others': { | ||||
|         'base': 'https://wiki.gentoo.org', | ||||
|         'search': '/index.php?title=Special:Search&offset={offset}&{query}\ | ||||
|                 &profile=translation&languagefilter={language}' | ||||
|     } | ||||
|                 &profile=translation&languagefilter={language}', | ||||
|     }, | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
|  | @ -78,7 +75,7 @@ main_langs = { | |||
|     'sl': 'Slovenský', | ||||
|     'th': 'ไทย', | ||||
|     'uk': 'Українська', | ||||
|     'zh': '简体中文' | ||||
|     'zh': '简体中文', | ||||
| } | ||||
| supported_languages = dict(lang_urls, **main_langs) | ||||
| 
 | ||||
|  | @ -101,8 +98,7 @@ def request(query, params): | |||
|     urls = get_lang_urls(language) | ||||
|     search_url = urls['base'] + urls['search'] | ||||
| 
 | ||||
|     params['url'] = search_url.format(query=query, offset=offset, | ||||
|                                       language=language) | ||||
|     params['url'] = search_url.format(query=query, offset=offset, language=language) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -123,7 +119,6 @@ def response(resp): | |||
|         href = urljoin(base_url, link.attrib.get('href')) | ||||
|         title = extract_text(link) | ||||
| 
 | ||||
|         results.append({'url': href, | ||||
|                         'title': title}) | ||||
|         results.append({'url': href, 'title': title}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -69,12 +69,7 @@ def fetch_extra_param(query_args, headers): | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params):  # pylint: disable=unused-argument | ||||
|     query_args = dict( | ||||
|         c = 'main' | ||||
|         , q = query | ||||
|         , dr = 1 | ||||
|         , showgoodimages = 0 | ||||
|     ) | ||||
|     query_args = dict(c='main', q=query, dr=1, showgoodimages=0) | ||||
| 
 | ||||
|     if params['language'] and params['language'] != 'all': | ||||
|         query_args['qlangcountry'] = params['language'] | ||||
|  | @ -93,6 +88,7 @@ def request(query, params):  # pylint: disable=unused-argument | |||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| # get response from search-request | ||||
| def response(resp): | ||||
|     results = [] | ||||
|  | @ -125,10 +121,6 @@ def response(resp): | |||
|         if len(subtitle) > 3 and subtitle != title: | ||||
|             title += " - " + subtitle | ||||
| 
 | ||||
|         results.append(dict( | ||||
|             url = url | ||||
|             , title = title | ||||
|             , content = content | ||||
|         )) | ||||
|         results.append(dict(url=url, title=title, content=content)) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -55,9 +55,7 @@ def response(resp): | |||
|             content = '' | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|                         'title': title, | ||||
|                         'content': content}) | ||||
|         results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -100,22 +100,13 @@ google_domains = { | |||
|     'UA': 'google.com.ua',  # Ukraine | ||||
|     'CN': 'google.com.hk',  # There is no google.cn, we use .com.hk for zh-CN | ||||
|     'HK': 'google.com.hk',  # Hong Kong | ||||
|     'TW': 'google.com.tw'   # Taiwan | ||||
|     'TW': 'google.com.tw',  # Taiwan | ||||
| } | ||||
| 
 | ||||
| time_range_dict = { | ||||
|     'day': 'd', | ||||
|     'week': 'w', | ||||
|     'month': 'm', | ||||
|     'year': 'y' | ||||
| } | ||||
| time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'} | ||||
| 
 | ||||
| # Filter results. 0: None, 1: Moderate, 2: Strict | ||||
| filter_mapping = { | ||||
|     0: 'off', | ||||
|     1: 'medium', | ||||
|     2: 'high' | ||||
| } | ||||
| filter_mapping = {0: 'off', 1: 'medium', 2: 'high'} | ||||
| 
 | ||||
| # specific xpath variables | ||||
| # ------------------------ | ||||
|  | @ -140,6 +131,7 @@ content_xpath = './/div[@class="IsZvec"]' | |||
| # from the links not the links itself. | ||||
| suggestion_xpath = '//div[contains(@class, "EIaa9b")]//a' | ||||
| 
 | ||||
| 
 | ||||
| def get_lang_info(params, lang_list, custom_aliases, supported_any_language): | ||||
|     """Composing various language properties for the google engines. | ||||
| 
 | ||||
|  | @ -250,15 +242,18 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language): | |||
|         ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language) | ||||
| 
 | ||||
|         # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5 | ||||
|         ret_val['headers']['Accept-Language'] = ','.join([ | ||||
|         ret_val['headers']['Accept-Language'] = ','.join( | ||||
|             [ | ||||
|                 lang_country, | ||||
|                 language + ';q=0.8,', | ||||
|                 'en;q=0.6', | ||||
|                 '*;q=0.5', | ||||
|         ]) | ||||
|             ] | ||||
|         ) | ||||
| 
 | ||||
|     return ret_val | ||||
| 
 | ||||
| 
 | ||||
| def detect_google_sorry(resp): | ||||
|     if resp.url.host == 'sorry.google.com' or resp.url.path.startswith('/sorry'): | ||||
|         raise SearxEngineCaptchaException() | ||||
|  | @ -269,9 +264,7 @@ def request(query, params): | |||
| 
 | ||||
|     offset = (params['pageno'] - 1) * 10 | ||||
| 
 | ||||
|     lang_info = get_lang_info( | ||||
|         params, supported_languages, language_aliases, True | ||||
|     ) | ||||
|     lang_info = get_lang_info(params, supported_languages, language_aliases, True) | ||||
| 
 | ||||
|     additional_parameters = {} | ||||
|     if use_mobile_ui: | ||||
|  | @ -281,7 +274,13 @@ def request(query, params): | |||
|         } | ||||
| 
 | ||||
|     # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium | ||||
|     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({ | ||||
|     query_url = ( | ||||
|         'https://' | ||||
|         + lang_info['subdomain'] | ||||
|         + '/search' | ||||
|         + "?" | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'q': query, | ||||
|                 **lang_info['params'], | ||||
|                 'ie': "utf8", | ||||
|  | @ -289,7 +288,9 @@ def request(query, params): | |||
|                 'start': offset, | ||||
|                 'filter': '0', | ||||
|                 **additional_parameters, | ||||
|     }) | ||||
|             } | ||||
|         ) | ||||
|     ) | ||||
| 
 | ||||
|     if params['time_range'] in time_range_dict: | ||||
|         query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]}) | ||||
|  | @ -301,9 +302,7 @@ def request(query, params): | |||
|     if use_mobile_ui: | ||||
|         params['headers']['Accept'] = '*/*' | ||||
|     else: | ||||
|         params['headers']['Accept'] = ( | ||||
|             'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|         ) | ||||
|         params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -355,11 +354,7 @@ def response(resp): | |||
|             if url is None: | ||||
|                 continue | ||||
|             content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True) | ||||
|             results.append({ | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content | ||||
|             }) | ||||
|             results.append({'url': url, 'title': title, 'content': content}) | ||||
|         except Exception as e:  # pylint: disable=broad-except | ||||
|             logger.error(e, exc_info=True) | ||||
|             # from lxml import etree | ||||
|  |  | |||
|  | @ -30,10 +30,8 @@ from searx.engines.google import ( | |||
| ) | ||||
| 
 | ||||
| # pylint: disable=unused-import | ||||
| from searx.engines.google import ( | ||||
|     supported_languages_url | ||||
|     ,  _fetch_supported_languages | ||||
| ) | ||||
| from searx.engines.google import supported_languages_url, _fetch_supported_languages | ||||
| 
 | ||||
| # pylint: enable=unused-import | ||||
| 
 | ||||
| # about | ||||
|  | @ -53,16 +51,11 @@ use_locale_domain = True | |||
| time_range_support = True | ||||
| safesearch = True | ||||
| 
 | ||||
| filter_mapping = { | ||||
|     0: 'images', | ||||
|     1: 'active', | ||||
|     2: 'active' | ||||
| } | ||||
| filter_mapping = {0: 'images', 1: 'active', 2: 'active'} | ||||
| 
 | ||||
| 
 | ||||
| def scrap_out_thumbs(dom): | ||||
|     """Scrap out thumbnail data from <script> tags. | ||||
|     """ | ||||
|     """Scrap out thumbnail data from <script> tags.""" | ||||
|     ret_val = {} | ||||
|     for script in eval_xpath(dom, '//script[contains(., "_setImgSrc(")]'): | ||||
|         _script = script.text | ||||
|  | @ -76,8 +69,7 @@ def scrap_out_thumbs(dom): | |||
| 
 | ||||
| 
 | ||||
| def scrap_img_by_id(script, data_id): | ||||
|     """Get full image URL by data-id in parent element | ||||
|     """ | ||||
|     """Get full image URL by data-id in parent element""" | ||||
|     img_url = '' | ||||
|     _script = script.split('\n') | ||||
|     for i, line in enumerate(_script): | ||||
|  | @ -91,20 +83,25 @@ def scrap_img_by_id(script, data_id): | |||
| def request(query, params): | ||||
|     """Google-Video search request""" | ||||
| 
 | ||||
|     lang_info = get_lang_info( | ||||
|         params, supported_languages, language_aliases, False | ||||
|     ) | ||||
|     logger.debug( | ||||
|         "HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
|     lang_info = get_lang_info(params, supported_languages, language_aliases, False) | ||||
|     logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
| 
 | ||||
|     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({ | ||||
|     query_url = ( | ||||
|         'https://' | ||||
|         + lang_info['subdomain'] | ||||
|         + '/search' | ||||
|         + "?" | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'q': query, | ||||
|                 'tbm': "isch", | ||||
|                 **lang_info['params'], | ||||
|                 'ie': "utf8", | ||||
|                 'oe': "utf8", | ||||
|                 'num': 30, | ||||
|     }) | ||||
|             } | ||||
|         ) | ||||
|     ) | ||||
| 
 | ||||
|     if params['time_range'] in time_range_dict: | ||||
|         query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]}) | ||||
|  | @ -113,9 +110,7 @@ def request(query, params): | |||
|     params['url'] = query_url | ||||
| 
 | ||||
|     params['headers'].update(lang_info['headers']) | ||||
|     params['headers']['Accept'] = ( | ||||
|         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     ) | ||||
|     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -128,8 +123,7 @@ def response(resp): | |||
|     # convert the text to dom | ||||
|     dom = html.fromstring(resp.text) | ||||
|     img_bas64_map = scrap_out_thumbs(dom) | ||||
|     img_src_script = eval_xpath_getindex( | ||||
|         dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text | ||||
|     img_src_script = eval_xpath_getindex(dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text | ||||
| 
 | ||||
|     # parse results | ||||
|     # | ||||
|  | @ -189,7 +183,8 @@ def response(resp): | |||
|         if not src_url: | ||||
|             src_url = thumbnail_src | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': img_alt, | ||||
|                 'content': pub_descr, | ||||
|  | @ -197,7 +192,8 @@ def response(resp): | |||
|                 'img_src': src_url, | ||||
|                 # 'img_format': img_format, | ||||
|                 'thumbnail_src': thumbnail_src, | ||||
|             'template': 'images.html' | ||||
|         }) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -32,6 +32,7 @@ from searx.engines.google import ( | |||
|     supported_languages_url, | ||||
|     _fetch_supported_languages, | ||||
| ) | ||||
| 
 | ||||
| # pylint: enable=unused-import | ||||
| 
 | ||||
| from searx.engines.google import ( | ||||
|  | @ -71,14 +72,12 @@ time_range_support = True | |||
| #  safesearch : results are identitical for safesearch=0 and safesearch=2 | ||||
| safesearch = False | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     """Google-News search request""" | ||||
| 
 | ||||
|     lang_info = get_lang_info( | ||||
|         params, supported_languages, language_aliases, False | ||||
|     ) | ||||
|     logger.debug( | ||||
|         "HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
|     lang_info = get_lang_info(params, supported_languages, language_aliases, False) | ||||
|     logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
| 
 | ||||
|     # google news has only one domain | ||||
|     lang_info['subdomain'] = 'news.google.com' | ||||
|  | @ -94,19 +93,26 @@ def request(query, params): | |||
|     if params['time_range']: | ||||
|         query += ' ' + time_range_dict[params['time_range']] | ||||
| 
 | ||||
|     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({ | ||||
|     query_url = ( | ||||
|         'https://' | ||||
|         + lang_info['subdomain'] | ||||
|         + '/search' | ||||
|         + "?" | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'q': query, | ||||
|                 **lang_info['params'], | ||||
|                 'ie': "utf8", | ||||
|                 'oe': "utf8", | ||||
|                 'gl': lang_info['country'], | ||||
|     }) + ('&ceid=%s' % ceid)  # ceid includes a ':' character which must not be urlencoded | ||||
|             } | ||||
|         ) | ||||
|         + ('&ceid=%s' % ceid) | ||||
|     )  # ceid includes a ':' character which must not be urlencoded | ||||
|     params['url'] = query_url | ||||
| 
 | ||||
|     params['headers'].update(lang_info['headers']) | ||||
|     params['headers']['Accept'] = ( | ||||
|         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|         ) | ||||
|     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     params['headers']['Cookie'] = "CONSENT=YES+cb.%s-14-p0.en+F+941;" % datetime.now().strftime("%Y%m%d") | ||||
| 
 | ||||
|     return params | ||||
|  | @ -178,12 +184,14 @@ def response(resp): | |||
| 
 | ||||
|         img_src = extract_text(result.xpath('preceding-sibling::a/figure/img/@src')) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'img_src': img_src, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -32,6 +32,7 @@ from searx.engines.google import ( | |||
|     supported_languages_url, | ||||
|     _fetch_supported_languages, | ||||
| ) | ||||
| 
 | ||||
| # pylint: enable=unused-import | ||||
| 
 | ||||
| # about | ||||
|  | @ -52,6 +53,7 @@ use_locale_domain = True | |||
| time_range_support = True | ||||
| safesearch = False | ||||
| 
 | ||||
| 
 | ||||
| def time_range_url(params): | ||||
|     """Returns a URL query component for a google-Scholar time range based on | ||||
|     ``params['time_range']``.  Google-Scholar does only support ranges in years. | ||||
|  | @ -72,34 +74,38 @@ def request(query, params): | |||
|     """Google-Scholar search request""" | ||||
| 
 | ||||
|     offset = (params['pageno'] - 1) * 10 | ||||
|     lang_info = get_lang_info( | ||||
|         params, supported_languages, language_aliases, False | ||||
|     ) | ||||
|     logger.debug( | ||||
|         "HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
|     lang_info = get_lang_info(params, supported_languages, language_aliases, False) | ||||
|     logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
| 
 | ||||
|     # subdomain is: scholar.google.xy | ||||
|     lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.") | ||||
| 
 | ||||
|     query_url = 'https://'+ lang_info['subdomain'] + '/scholar' + "?" + urlencode({ | ||||
|     query_url = ( | ||||
|         'https://' | ||||
|         + lang_info['subdomain'] | ||||
|         + '/scholar' | ||||
|         + "?" | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'q': query, | ||||
|                 **lang_info['params'], | ||||
|                 'ie': "utf8", | ||||
|                 'oe': "utf8", | ||||
|                 'start': offset, | ||||
|     }) | ||||
|             } | ||||
|         ) | ||||
|     ) | ||||
| 
 | ||||
|     query_url += time_range_url(params) | ||||
|     params['url'] = query_url | ||||
| 
 | ||||
|     params['headers'].update(lang_info['headers']) | ||||
|     params['headers']['Accept'] = ( | ||||
|         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     ) | ||||
|     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
| 
 | ||||
|     # params['google_subdomain'] = subdomain | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     """Get response from google's search request""" | ||||
|     results = [] | ||||
|  | @ -132,11 +138,13 @@ def response(resp): | |||
|         if pub_type: | ||||
|             title = title + " " + pub_type | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # parse suggestion | ||||
|     for suggestion in eval_xpath(dom, '//div[contains(@class, "gs_qsuggest_wrap")]//li//a'): | ||||
|  |  | |||
|  | @ -38,10 +38,8 @@ from searx.engines.google import ( | |||
| ) | ||||
| 
 | ||||
| # pylint: disable=unused-import | ||||
| from searx.engines.google import ( | ||||
|     supported_languages_url | ||||
|     ,  _fetch_supported_languages | ||||
| ) | ||||
| from searx.engines.google import supported_languages_url, _fetch_supported_languages | ||||
| 
 | ||||
| # pylint: enable=unused-import | ||||
| 
 | ||||
| # about | ||||
|  | @ -65,6 +63,7 @@ safesearch = True | |||
| 
 | ||||
| RE_CACHE = {} | ||||
| 
 | ||||
| 
 | ||||
| def _re(regexpr): | ||||
|     """returns compiled regular expression""" | ||||
|     RE_CACHE[regexpr] = RE_CACHE.get(regexpr, re.compile(regexpr)) | ||||
|  | @ -87,8 +86,7 @@ def scrap_out_thumbs_src(dom): | |||
| 
 | ||||
| 
 | ||||
| def scrap_out_thumbs(dom): | ||||
|     """Scrap out thumbnail data from <script> tags. | ||||
|     """ | ||||
|     """Scrap out thumbnail data from <script> tags.""" | ||||
|     ret_val = {} | ||||
|     thumb_name = 'dimg_' | ||||
| 
 | ||||
|  | @ -112,19 +110,24 @@ def scrap_out_thumbs(dom): | |||
| def request(query, params): | ||||
|     """Google-Video search request""" | ||||
| 
 | ||||
|     lang_info = get_lang_info( | ||||
|         params, supported_languages, language_aliases, False | ||||
|     ) | ||||
|     logger.debug( | ||||
|         "HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
|     lang_info = get_lang_info(params, supported_languages, language_aliases, False) | ||||
|     logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) | ||||
| 
 | ||||
|     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({ | ||||
|     query_url = ( | ||||
|         'https://' | ||||
|         + lang_info['subdomain'] | ||||
|         + '/search' | ||||
|         + "?" | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'q': query, | ||||
|                 'tbm': "vid", | ||||
|                 **lang_info['params'], | ||||
|                 'ie': "utf8", | ||||
|                 'oe': "utf8", | ||||
|     }) | ||||
|             } | ||||
|         ) | ||||
|     ) | ||||
| 
 | ||||
|     if params['time_range'] in time_range_dict: | ||||
|         query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]}) | ||||
|  | @ -133,9 +136,7 @@ def request(query, params): | |||
|     params['url'] = query_url | ||||
| 
 | ||||
|     params['headers'].update(lang_info['headers']) | ||||
|     params['headers']['Accept'] = ( | ||||
|         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|         ) | ||||
|     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -171,13 +172,13 @@ def response(resp): | |||
| 
 | ||||
|         title = extract_text(eval_xpath_getindex(result, title_xpath, 0)) | ||||
|         url = eval_xpath_getindex(result, './/div[@class="dXiKIc"]//a/@href', 0) | ||||
|         length = extract_text(eval_xpath( | ||||
|             result, './/div[contains(@class, "P7xzyf")]/span/span')) | ||||
|         length = extract_text(eval_xpath(result, './/div[contains(@class, "P7xzyf")]/span/span')) | ||||
|         c_node = eval_xpath_getindex(result, './/div[@class="Uroaid"]', 0) | ||||
|         content = extract_text(c_node) | ||||
|         pub_info = extract_text(eval_xpath(result, './/div[@class="Zg1NU"]')) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|  | @ -185,7 +186,8 @@ def response(resp): | |||
|                 'author': pub_info, | ||||
|                 'thumbnail': img_src, | ||||
|                 'template': 'videos.html', | ||||
|             }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # parse suggestion | ||||
|     for suggestion in eval_xpath_list(dom, suggestion_xpath): | ||||
|  |  | |||
|  | @ -27,7 +27,9 @@ about = { | |||
|     "results": 'HTML', | ||||
| } | ||||
| 
 | ||||
| categories = ['general', ] | ||||
| categories = [ | ||||
|     'general', | ||||
| ] | ||||
| paging = False | ||||
| 
 | ||||
| # suggestion_url = "https://sg.media-imdb.com/suggestion/{letter}/{query}.json" | ||||
|  | @ -35,13 +37,7 @@ suggestion_url = "https://v2.sg.media-imdb.com/suggestion/{letter}/{query}.json" | |||
| 
 | ||||
| href_base = 'https://imdb.com/{category}/{entry_id}' | ||||
| 
 | ||||
| search_categories = { | ||||
|     "nm": "name", | ||||
|     "tt": "title", | ||||
|     "kw": "keyword", | ||||
|     "co": "company", | ||||
|     "ep": "episode" | ||||
| } | ||||
| search_categories = {"nm": "name", "tt": "title", "kw": "keyword", "co": "company", "ep": "episode"} | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|  | @ -63,9 +59,7 @@ def response(resp): | |||
|         entry_id = entry['id'] | ||||
|         categ = search_categories.get(entry_id[:2]) | ||||
|         if categ is None: | ||||
|             logger.error( | ||||
|                 'skip unknown category tag %s in %s', entry_id[:2], entry_id | ||||
|             ) | ||||
|             logger.error('skip unknown category tag %s in %s', entry_id[:2], entry_id) | ||||
|             continue | ||||
| 
 | ||||
|         title = entry['l'] | ||||
|  | @ -95,11 +89,13 @@ def response(resp): | |||
|             if not image_url_name.endswith('_V1_'): | ||||
|                 magic = '_V1_' + magic | ||||
|             image_url = image_url_name + magic + '.' + image_url_prefix | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 "title": title, | ||||
|                 "url": href_base.format(category=categ, entry_id=entry_id), | ||||
|                 "content": content, | ||||
|                 "img_src": image_url, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -41,9 +41,7 @@ content_xpath = './/p[@class="media-body__summary"]' | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     params['url'] = search_url.format(ps=page_size, | ||||
|                                       start=params['pageno'] * page_size, | ||||
|                                       query=urlencode({'q': query})) | ||||
|     params['url'] = search_url.format(ps=page_size, start=params['pageno'] * page_size, query=urlencode({'q': query})) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -75,12 +73,16 @@ def response(resp): | |||
|         content = extract_text(result.xpath(content_xpath)) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'template': 'videos.html', | ||||
|                 'publishedDate': publishedDate, | ||||
|                         'thumbnail': thumbnail}) | ||||
|                 'thumbnail': thumbnail, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -46,14 +46,10 @@ def request(query, params): | |||
|         base_url_rand = base_url | ||||
| 
 | ||||
|     search_url = base_url_rand + "api/v1/search?q={query}" | ||||
|     params["url"] = search_url.format( | ||||
|         query=quote_plus(query) | ||||
|     ) + "&page={pageno}".format(pageno=params["pageno"]) | ||||
|     params["url"] = search_url.format(query=quote_plus(query)) + "&page={pageno}".format(pageno=params["pageno"]) | ||||
| 
 | ||||
|     if params["time_range"] in time_range_dict: | ||||
|         params["url"] += "&date={timerange}".format( | ||||
|             timerange=time_range_dict[params["time_range"]] | ||||
|         ) | ||||
|         params["url"] += "&date={timerange}".format(timerange=time_range_dict[params["time_range"]]) | ||||
| 
 | ||||
|     if params["language"] != "all": | ||||
|         lang = params["language"].split("-") | ||||
|  | @ -88,17 +84,13 @@ def response(resp): | |||
|             url = base_invidious_url + videoid | ||||
|             embedded = embedded_url.format(videoid=videoid) | ||||
|             thumbs = result.get("videoThumbnails", []) | ||||
|             thumb = next( | ||||
|                 (th for th in thumbs if th["quality"] == "sddefault"), None | ||||
|             ) | ||||
|             thumb = next((th for th in thumbs if th["quality"] == "sddefault"), None) | ||||
|             if thumb: | ||||
|                 thumbnail = thumb.get("url", "") | ||||
|             else: | ||||
|                 thumbnail = "" | ||||
| 
 | ||||
|             publishedDate = parser.parse( | ||||
|                 time.ctime(result.get("published", 0)) | ||||
|             ) | ||||
|             publishedDate = parser.parse(time.ctime(result.get("published", 0))) | ||||
|             length = time.gmtime(result.get("lengthSeconds")) | ||||
|             if length.tm_hour: | ||||
|                 length = time.strftime("%H:%M:%S", length) | ||||
|  |  | |||
|  | @ -119,22 +119,22 @@ def response(resp): | |||
|                 content = query(result, content_query)[0] | ||||
|             except: | ||||
|                 content = "" | ||||
|             results.append({ | ||||
|             results.append( | ||||
|                 { | ||||
|                     'url': to_string(url), | ||||
|                     'title': title_filter(to_string(title)), | ||||
|                     'content': content_filter(to_string(content)), | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
|     else: | ||||
|         for url, title, content in zip( | ||||
|             query(json, url_query), | ||||
|             query(json, title_query), | ||||
|             query(json, content_query) | ||||
|         ): | ||||
|             results.append({ | ||||
|         for url, title, content in zip(query(json, url_query), query(json, title_query), query(json, content_query)): | ||||
|             results.append( | ||||
|                 { | ||||
|                     'url': to_string(url), | ||||
|                     'title': title_filter(to_string(title)), | ||||
|                     'content': content_filter(to_string(content)), | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|     if not suggestion_query: | ||||
|         return results | ||||
|  |  | |||
|  | @ -34,8 +34,7 @@ content_xpath = './/span[@class="font11px lightgrey block"]' | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     params['url'] = search_url.format(search_term=quote(query), | ||||
|                                       pageno=params['pageno']) | ||||
|     params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno']) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -79,7 +78,9 @@ def response(resp): | |||
|         torrentfileurl = quote(torrentfile, safe="%/:=&?~#+!$,;'@()*") | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': href, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': href, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'seed': seed, | ||||
|  | @ -88,7 +89,9 @@ def response(resp): | |||
|                 'files': files, | ||||
|                 'magnetlink': magnetlink, | ||||
|                 'torrentfile': torrentfileurl, | ||||
|                         'template': 'torrent.html'}) | ||||
|                 'template': 'torrent.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results sorted by seeder | ||||
|     return sorted(results, key=itemgetter('seed'), reverse=True) | ||||
|  |  | |||
|  | @ -34,9 +34,7 @@ IMG_SRC_FIXES = { | |||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     search_path = search_string.format( | ||||
|         query=urlencode({'q': query}), | ||||
|         page=params['pageno']) | ||||
|     search_path = search_string.format(query=urlencode({'q': query}), page=params['pageno']) | ||||
| 
 | ||||
|     params['url'] = base_url + search_path | ||||
| 
 | ||||
|  | @ -56,13 +54,15 @@ def response(resp): | |||
|                 break | ||||
|         else: | ||||
|             img_src = result['image']['thumb'] | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': result['links']['item'], | ||||
|                 'title': result['title'], | ||||
|                 'img_src': img_src, | ||||
|                 'thumbnail_src': result['image']['thumb'], | ||||
|                 'author': result['creator'], | ||||
|             'template': 'images.html' | ||||
|         }) | ||||
|                 'template': 'images.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -22,29 +22,33 @@ paging = True | |||
| time_range_support = False | ||||
| safesearch = False | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     params['url'] = 'https://mediathekviewweb.de/api/query' | ||||
|     params['method'] = 'POST' | ||||
|     params['headers']['Content-type'] = 'text/plain' | ||||
|     params['data'] = dumps({ | ||||
|     params['data'] = dumps( | ||||
|         { | ||||
|             'queries': [ | ||||
|                 { | ||||
|                     'fields': [ | ||||
|                         'title', | ||||
|                         'topic', | ||||
|                     ], | ||||
| 	    'query' : query | ||||
|                     'query': query, | ||||
|                 }, | ||||
|             ], | ||||
|             'sortBy': 'timestamp', | ||||
|             'sortOrder': 'desc', | ||||
|             'future': True, | ||||
|             'offset': (params['pageno'] - 1) * 10, | ||||
|         'size' : 10 | ||||
|     }) | ||||
|             'size': 10, | ||||
|         } | ||||
|     ) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     resp = loads(resp.text) | ||||
|  | @ -58,11 +62,13 @@ def response(resp): | |||
| 
 | ||||
|         item['hms'] = str(datetime.timedelta(seconds=item['duration'])) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': item['url_video_hd'], | ||||
|                 'title': "%(channel)s: %(title)s (%(hms)s)" % item, | ||||
|                 'length': item['hms'], | ||||
|                 'content': "%(description)s" % item, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -25,23 +25,24 @@ search_type = 'nearmatch'  # possible values: title, text, nearmatch | |||
| 
 | ||||
| # search-url | ||||
| base_url = 'https://{language}.wikipedia.org/' | ||||
| search_postfix = 'w/api.php?action=query'\ | ||||
|     '&list=search'\ | ||||
|     '&{query}'\ | ||||
|     '&format=json'\ | ||||
|     '&sroffset={offset}'\ | ||||
|     '&srlimit={limit}'\ | ||||
| search_postfix = ( | ||||
|     'w/api.php?action=query' | ||||
|     '&list=search' | ||||
|     '&{query}' | ||||
|     '&format=json' | ||||
|     '&sroffset={offset}' | ||||
|     '&srlimit={limit}' | ||||
|     '&srwhat={searchtype}' | ||||
| ) | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     offset = (params['pageno'] - 1) * number_of_results | ||||
| 
 | ||||
|     string_args = dict(query=urlencode({'srsearch': query}), | ||||
|                        offset=offset, | ||||
|                        limit=number_of_results, | ||||
|                        searchtype=search_type) | ||||
|     string_args = dict( | ||||
|         query=urlencode({'srsearch': query}), offset=offset, limit=number_of_results, searchtype=search_type | ||||
|     ) | ||||
| 
 | ||||
|     format_strings = list(Formatter().parse(base_url)) | ||||
| 
 | ||||
|  | @ -78,13 +79,14 @@ def response(resp): | |||
|     for result in search_results['query']['search']: | ||||
|         if result.get('snippet', '').startswith('#REDIRECT'): | ||||
|             continue | ||||
|         url = base_url.format(language=resp.search_params['language']) +\ | ||||
|             'wiki/' + quote(result['title'].replace(' ', '_').encode()) | ||||
|         url = ( | ||||
|             base_url.format(language=resp.search_params['language']) | ||||
|             + 'wiki/' | ||||
|             + quote(result['title'].replace(' ', '_').encode()) | ||||
|         ) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|                         'title': result['title'], | ||||
|                         'content': ''}) | ||||
|         results.append({'url': url, 'title': result['title'], 'content': ''}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -26,7 +26,8 @@ def request(query, params): | |||
|     params['url'] = search_url | ||||
|     params['method'] = 'POST' | ||||
|     params['headers']['content-type'] = 'application/json; charset=utf-8' | ||||
|     params['data'] = dumps({ | ||||
|     params['data'] = dumps( | ||||
|         { | ||||
|             'query': query, | ||||
|             'queryExpression': '', | ||||
|             'filters': [], | ||||
|  | @ -36,7 +37,8 @@ def request(query, params): | |||
|             'take': 10, | ||||
|             'includeCitationContexts': False, | ||||
|             'profileId': '', | ||||
|     }) | ||||
|         } | ||||
|     ) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -54,11 +56,13 @@ def response(resp): | |||
|         title = result['paper']['dn'] | ||||
|         content = _get_content(result['paper']) | ||||
|         url = _paper_url.format(id=result['paper']['id']) | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': html_to_text(title), | ||||
|                 'content': html_to_text(content), | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
|  |  | |||
|  | @ -25,16 +25,17 @@ paging = True | |||
| url = 'https://api.mixcloud.com/' | ||||
| search_url = url + 'search/?{query}&type=cloudcast&limit=10&offset={offset}' | ||||
| 
 | ||||
| embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' +\ | ||||
|     'data-src="https://www.mixcloud.com/widget/iframe/?feed={url}" width="300" height="300"></iframe>' | ||||
| embedded_url = ( | ||||
|     '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' | ||||
|     + 'data-src="https://www.mixcloud.com/widget/iframe/?feed={url}" width="300" height="300"></iframe>' | ||||
| ) | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     offset = (params['pageno'] - 1) * 10 | ||||
| 
 | ||||
|     params['url'] = search_url.format(query=urlencode({'q': query}), | ||||
|                                       offset=offset) | ||||
|     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -54,11 +55,9 @@ def response(resp): | |||
|         publishedDate = parser.parse(result['created_time']) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|                         'title': title, | ||||
|                         'embedded': embedded, | ||||
|                         'publishedDate': publishedDate, | ||||
|                         'content': content}) | ||||
|         results.append( | ||||
|             {'url': url, 'title': title, 'embedded': embedded, 'publishedDate': publishedDate, 'content': content} | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -26,9 +26,11 @@ result_template = 'key-value.html' | |||
| 
 | ||||
| _client = None | ||||
| 
 | ||||
| 
 | ||||
| def init(_): | ||||
|     connect() | ||||
| 
 | ||||
| 
 | ||||
| def connect(): | ||||
|     global _client  # pylint: disable=global-statement | ||||
|     kwargs = {'port': port} | ||||
|  | @ -38,6 +40,7 @@ def connect(): | |||
|         kwargs['password'] = password | ||||
|     _client = MongoClient(host, **kwargs)[database][collection] | ||||
| 
 | ||||
| 
 | ||||
| def search(query, params): | ||||
|     results = [] | ||||
|     if exact_match_only: | ||||
|  | @ -46,13 +49,7 @@ def search(query, params): | |||
|         _re = re.compile('.*{0}.*'.format(re.escape(query)), re.I | re.M) | ||||
|         q = {'$regex': _re} | ||||
| 
 | ||||
|     query = _client.find( | ||||
|         {key: q} | ||||
|     ).skip( | ||||
|         ( params['pageno'] -1 ) * results_per_page | ||||
|     ).limit( | ||||
|         results_per_page | ||||
|     ) | ||||
|     query = _client.find({key: q}).skip((params['pageno'] - 1) * results_per_page).limit(results_per_page) | ||||
| 
 | ||||
|     results.append({'number_of_results': query.count()}) | ||||
|     for r in query: | ||||
|  |  | |||
|  | @ -20,6 +20,7 @@ paging = True | |||
| result_template = 'key-value.html' | ||||
| _connection = None | ||||
| 
 | ||||
| 
 | ||||
| def init(engine_settings): | ||||
|     global _connection  # pylint: disable=global-statement | ||||
| 
 | ||||
|  | @ -37,6 +38,7 @@ def init(engine_settings): | |||
|         auth_plugin=auth_plugin, | ||||
|     ) | ||||
| 
 | ||||
| 
 | ||||
| def search(query, params): | ||||
|     query_params = {'query': query} | ||||
|     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) | ||||
|  | @ -46,6 +48,7 @@ def search(query, params): | |||
| 
 | ||||
|         return _fetch_results(cur) | ||||
| 
 | ||||
| 
 | ||||
| def _fetch_results(cur): | ||||
|     results = [] | ||||
|     for res in cur: | ||||
|  |  | |||
|  | @ -98,7 +98,9 @@ def response(resp): | |||
|         content = 'Category: "{category}". Downloaded {downloads} times.' | ||||
|         content = content.format(category=category, downloads=downloads) | ||||
| 
 | ||||
|         results.append({'url': href, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': href, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'seed': seed, | ||||
|  | @ -106,6 +108,8 @@ def response(resp): | |||
|                 'filesize': filesize, | ||||
|                 'torrentfile': torrent_link, | ||||
|                 'magnetlink': magnet_link, | ||||
|                         'template': 'torrent.html'}) | ||||
|                 'template': 'torrent.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -151,10 +151,12 @@ def response(resp): | |||
|     user_language = resp.search_params['language'] | ||||
| 
 | ||||
|     if resp.search_params['route']: | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'answer': gettext('Get directions'), | ||||
|                 'url': route_url.format(*resp.search_params['route'].groups()), | ||||
|             }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     fetch_wikidata(nominatim_json, user_language) | ||||
| 
 | ||||
|  | @ -170,7 +172,8 @@ def response(resp): | |||
|         links, link_keys = get_links(result, user_language) | ||||
|         data = get_data(result, user_language, link_keys) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'template': 'map.html', | ||||
|                 'title': title, | ||||
|                 'address': address, | ||||
|  | @ -181,15 +184,14 @@ def response(resp): | |||
|                 'img_src': img_src, | ||||
|                 'links': links, | ||||
|                 'data': data, | ||||
|             'type': get_tag_label( | ||||
|                 result.get('category'), result.get('type', ''), user_language | ||||
|             ), | ||||
|                 'type': get_tag_label(result.get('category'), result.get('type', ''), user_language), | ||||
|                 'type_icon': result.get('icon'), | ||||
|                 'content': '', | ||||
|                 'longitude': result['lon'], | ||||
|                 'latitude': result['lat'], | ||||
|                 'boundingbox': result['boundingbox'], | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
|  | @ -297,8 +299,7 @@ def get_title_address(result): | |||
| 
 | ||||
| 
 | ||||
| def get_url_osm_geojson(result): | ||||
|     """Get url, osm and geojson | ||||
|     """ | ||||
|     """Get url, osm and geojson""" | ||||
|     osm_type = result.get('osm_type', result.get('type')) | ||||
|     if 'osm_id' not in result: | ||||
|         # see https://github.com/osm-search/Nominatim/issues/1521 | ||||
|  | @ -349,11 +350,13 @@ def get_links(result, user_language): | |||
|             url, url_label = mapping_function(raw_value) | ||||
|             if url.startswith('https://wikidata.org'): | ||||
|                 url_label = result.get('wikidata', {}).get('itemLabel') or url_label | ||||
|             links.append({ | ||||
|             links.append( | ||||
|                 { | ||||
|                     'label': get_key_label(k, user_language), | ||||
|                     'url': url, | ||||
|                     'url_label': url_label, | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
|             link_keys.add(k) | ||||
|     return links, link_keys | ||||
| 
 | ||||
|  | @ -373,11 +376,13 @@ def get_data(result, user_language, ignore_keys): | |||
|             continue | ||||
|         k_label = get_key_label(k, user_language) | ||||
|         if k_label: | ||||
|             data.append({ | ||||
|             data.append( | ||||
|                 { | ||||
|                     'label': k_label, | ||||
|                     'key': k, | ||||
|                     'value': v, | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
|     data.sort(key=lambda entry: (get_key_rank(entry['key']), entry['label'])) | ||||
|     return data | ||||
| 
 | ||||
|  |  | |||
|  | @ -34,10 +34,7 @@ def request(query, params): | |||
| 
 | ||||
|     params['url'] = pdbe_solr_url | ||||
|     params['method'] = 'POST' | ||||
|     params['data'] = { | ||||
|         'q': query, | ||||
|         'wt': "json"  # request response in parsable format | ||||
|     } | ||||
|     params['data'] = {'q': query, 'wt': "json"}  # request response in parsable format | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -53,12 +50,21 @@ def construct_body(result): | |||
|         if result['journal']: | ||||
|             content = content.format( | ||||
|                 title=result['citation_title'], | ||||
|                 authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'], | ||||
|                 page=result['journal_page'], year=result['citation_year']) | ||||
|                 authors=result['entry_author_list'][0], | ||||
|                 journal=result['journal'], | ||||
|                 volume=result['journal_volume'], | ||||
|                 page=result['journal_page'], | ||||
|                 year=result['citation_year'], | ||||
|             ) | ||||
|         else: | ||||
|             content = content.format( | ||||
|                 title=result['citation_title'], | ||||
|                 authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year']) | ||||
|                 authors=result['entry_author_list'][0], | ||||
|                 journal='', | ||||
|                 volume='', | ||||
|                 page='', | ||||
|                 year=result['release_year'], | ||||
|             ) | ||||
|         img_src = pdbe_preview_url.format(pdb_id=result['pdb_id']) | ||||
|     except (KeyError): | ||||
|         content = None | ||||
|  | @ -96,20 +102,21 @@ def response(resp): | |||
|             # since we can't construct a proper body from the response, we'll make up our own | ||||
|             msg_superseded = gettext("This entry has been superseded by") | ||||
|             content = '{msg_superseded}: {url} ({pdb_id})'.format( | ||||
|                 msg_superseded=msg_superseded, | ||||
|                 url=superseded_url, | ||||
|                 pdb_id=result['superseded_by']) | ||||
|                 msg_superseded=msg_superseded, url=superseded_url, pdb_id=result['superseded_by'] | ||||
|             ) | ||||
| 
 | ||||
|             # obsoleted entries don't have preview images | ||||
|             img_src = None | ||||
|         else: | ||||
|             title, content, img_src = construct_body(result) | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': pdbe_entry_url.format(pdb_id=result['pdb_id']), | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|             'img_src': img_src | ||||
|         }) | ||||
|                 'img_src': img_src, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -36,9 +36,7 @@ def request(query, params): | |||
|     language = params["language"].split("-")[0] | ||||
|     if "all" != language and language in supported_languages: | ||||
|         query_dict["languageOneOf"] = language | ||||
|     params["url"] = search_url.format( | ||||
|         query=urlencode(query_dict), pageno=pageno | ||||
|     ) | ||||
|     params["url"] = search_url.format(query=urlencode(query_dict), pageno=pageno) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  |  | |||
|  | @ -33,9 +33,7 @@ supported_languages = ['de', 'en', 'fr', 'it'] | |||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     params['url'] = base_url +\ | ||||
|         search_string.format(query=urlencode({'q': query}), | ||||
|                              limit=number_of_results) | ||||
|     params['url'] = base_url + search_string.format(query=urlencode({'q': query}), limit=number_of_results) | ||||
| 
 | ||||
|     if params['language'] != 'all': | ||||
|         language = params['language'].split('_')[0] | ||||
|  | @ -75,50 +73,60 @@ def response(resp): | |||
|             # continue if invalide osm-type | ||||
|             continue | ||||
| 
 | ||||
|         url = result_base_url.format(osm_type=osm_type, | ||||
|                                      osm_id=properties.get('osm_id')) | ||||
|         url = result_base_url.format(osm_type=osm_type, osm_id=properties.get('osm_id')) | ||||
| 
 | ||||
|         osm = {'type': osm_type, | ||||
|                'id': properties.get('osm_id')} | ||||
|         osm = {'type': osm_type, 'id': properties.get('osm_id')} | ||||
| 
 | ||||
|         geojson = r.get('geometry') | ||||
| 
 | ||||
|         if properties.get('extent'): | ||||
|             boundingbox = [properties.get('extent')[3], | ||||
|             boundingbox = [ | ||||
|                 properties.get('extent')[3], | ||||
|                 properties.get('extent')[1], | ||||
|                 properties.get('extent')[0], | ||||
|                            properties.get('extent')[2]] | ||||
|                 properties.get('extent')[2], | ||||
|             ] | ||||
|         else: | ||||
|             # TODO: better boundingbox calculation | ||||
|             boundingbox = [geojson['coordinates'][1], | ||||
|             boundingbox = [ | ||||
|                 geojson['coordinates'][1], | ||||
|                 geojson['coordinates'][1], | ||||
|                 geojson['coordinates'][0], | ||||
|                            geojson['coordinates'][0]] | ||||
|                 geojson['coordinates'][0], | ||||
|             ] | ||||
| 
 | ||||
|         # address calculation | ||||
|         address = {} | ||||
| 
 | ||||
|         # get name | ||||
|         if properties.get('osm_key') == 'amenity' or\ | ||||
|            properties.get('osm_key') == 'shop' or\ | ||||
|            properties.get('osm_key') == 'tourism' or\ | ||||
|            properties.get('osm_key') == 'leisure': | ||||
|         if ( | ||||
|             properties.get('osm_key') == 'amenity' | ||||
|             or properties.get('osm_key') == 'shop' | ||||
|             or properties.get('osm_key') == 'tourism' | ||||
|             or properties.get('osm_key') == 'leisure' | ||||
|         ): | ||||
|             address = {'name': properties.get('name')} | ||||
| 
 | ||||
|         # add rest of adressdata, if something is already found | ||||
|         if address.get('name'): | ||||
|             address.update({'house_number': properties.get('housenumber'), | ||||
|             address.update( | ||||
|                 { | ||||
|                     'house_number': properties.get('housenumber'), | ||||
|                     'road': properties.get('street'), | ||||
|                            'locality': properties.get('city', | ||||
|                                        properties.get('town',           # noqa | ||||
|                                        properties.get('village'))),     # noqa | ||||
|                     'locality': properties.get( | ||||
|                         'city', properties.get('town', properties.get('village'))  # noqa | ||||
|                     ),  # noqa | ||||
|                     'postcode': properties.get('postcode'), | ||||
|                            'country': properties.get('country')}) | ||||
|                     'country': properties.get('country'), | ||||
|                 } | ||||
|             ) | ||||
|         else: | ||||
|             address = None | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'template': 'map.html', | ||||
|         results.append( | ||||
|             { | ||||
|                 'template': 'map.html', | ||||
|                 'title': title, | ||||
|                 'content': '', | ||||
|                 'longitude': geojson['coordinates'][0], | ||||
|  | @ -127,7 +135,9 @@ def response(resp): | |||
|                 'geojson': geojson, | ||||
|                 'address': address, | ||||
|                 'osm': osm, | ||||
|                         'url': url}) | ||||
|                 'url': url, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -40,17 +40,14 @@ trackers = [ | |||
| ] | ||||
| 
 | ||||
| # piratebay specific type-definitions | ||||
| search_types = {"files": "0", | ||||
|                 "music": "100", | ||||
|                 "videos": "200"} | ||||
| search_types = {"files": "0", "music": "100", "videos": "200"} | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     search_type = search_types.get(params["category"], "0") | ||||
| 
 | ||||
|     params["url"] = search_url.format(search_term=quote(query), | ||||
|                                       search_type=search_type) | ||||
|     params["url"] = search_url.format(search_term=quote(query), search_type=search_type) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -68,8 +65,9 @@ def response(resp): | |||
|     # parse results | ||||
|     for result in search_res: | ||||
|         link = url + "description.php?id=" + result["id"] | ||||
|         magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"]\ | ||||
|                      + "&tr=" + "&tr=".join(trackers) | ||||
|         magnetlink = ( | ||||
|             "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"] + "&tr=" + "&tr=".join(trackers) | ||||
|         ) | ||||
| 
 | ||||
|         params = { | ||||
|             "url": link, | ||||
|  | @ -77,7 +75,7 @@ def response(resp): | |||
|             "seed": result["seeders"], | ||||
|             "leech": result["leechers"], | ||||
|             "magnetlink": magnetlink, | ||||
|             "template": "torrent.html" | ||||
|             "template": "torrent.html", | ||||
|         } | ||||
| 
 | ||||
|         # extract and convert creation date | ||||
|  |  | |||
|  | @ -20,6 +20,7 @@ paging = True | |||
| result_template = 'key-value.html' | ||||
| _connection = None | ||||
| 
 | ||||
| 
 | ||||
| def init(engine_settings): | ||||
|     global _connection  # pylint: disable=global-statement | ||||
| 
 | ||||
|  | @ -37,18 +38,17 @@ def init(engine_settings): | |||
|         port=port, | ||||
|     ) | ||||
| 
 | ||||
| 
 | ||||
| def search(query, params): | ||||
|     query_params = {'query': query} | ||||
|     query_to_run = ( | ||||
|         query_str | ||||
|         + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) | ||||
|     ) | ||||
|     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) | ||||
| 
 | ||||
|     with _connection: | ||||
|         with _connection.cursor() as cur: | ||||
|             cur.execute(query_to_run, query_params) | ||||
|             return _fetch_results(cur) | ||||
| 
 | ||||
| 
 | ||||
| def _fetch_results(cur): | ||||
|     results = [] | ||||
|     titles = [] | ||||
|  |  | |||
|  | @ -15,7 +15,7 @@ about = { | |||
|     "wikidata_id": 'Q1540899', | ||||
|     "official_api_documentation": { | ||||
|         'url': 'https://www.ncbi.nlm.nih.gov/home/develop/api/', | ||||
|         'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/' | ||||
|         'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/', | ||||
|     }, | ||||
|     "use_official_api": True, | ||||
|     "require_api_key": False, | ||||
|  | @ -24,8 +24,9 @@ about = { | |||
| 
 | ||||
| categories = ['science'] | ||||
| 
 | ||||
| base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'\ | ||||
|            + '?db=pubmed&{query}&retstart={offset}&retmax={hits}' | ||||
| base_url = ( | ||||
|     'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi' + '?db=pubmed&{query}&retstart={offset}&retmax={hits}' | ||||
| ) | ||||
| 
 | ||||
| # engine dependent config | ||||
| number_of_results = 10 | ||||
|  | @ -36,9 +37,7 @@ def request(query, params): | |||
|     # basic search | ||||
|     offset = (params['pageno'] - 1) * number_of_results | ||||
| 
 | ||||
|     string_args = dict(query=urlencode({'term': query}), | ||||
|                        offset=offset, | ||||
|                        hits=number_of_results) | ||||
|     string_args = dict(query=urlencode({'term': query}), offset=offset, hits=number_of_results) | ||||
| 
 | ||||
|     params['url'] = base_url.format(**string_args) | ||||
| 
 | ||||
|  | @ -49,8 +48,9 @@ def response(resp): | |||
|     results = [] | ||||
| 
 | ||||
|     # First retrieve notice of each result | ||||
|     pubmed_retrieve_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'\ | ||||
|                               + 'db=pubmed&retmode=xml&id={pmids_string}' | ||||
|     pubmed_retrieve_api_url = ( | ||||
|         'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' + 'db=pubmed&retmode=xml&id={pmids_string}' | ||||
|     ) | ||||
| 
 | ||||
|     pmids_results = etree.XML(resp.content) | ||||
|     pmids = pmids_results.xpath('//eSearchResult/IdList/Id') | ||||
|  | @ -88,14 +88,17 @@ def response(resp): | |||
|             content = content[0:300] + "..." | ||||
|         # TODO: center snippet on query term | ||||
| 
 | ||||
|         res_dict = {'url': url, | ||||
|                     'title': title, | ||||
|                     'content': content} | ||||
|         res_dict = {'url': url, 'title': title, 'content': content} | ||||
| 
 | ||||
|         try: | ||||
|             publishedDate = datetime.strptime(entry.xpath('.//DateCreated/Year')[0].text | ||||
|                                               + '-' + entry.xpath('.//DateCreated/Month')[0].text | ||||
|                                               + '-' + entry.xpath('.//DateCreated/Day')[0].text, '%Y-%m-%d') | ||||
|             publishedDate = datetime.strptime( | ||||
|                 entry.xpath('.//DateCreated/Year')[0].text | ||||
|                 + '-' | ||||
|                 + entry.xpath('.//DateCreated/Month')[0].text | ||||
|                 + '-' | ||||
|                 + entry.xpath('.//DateCreated/Day')[0].text, | ||||
|                 '%Y-%m-%d', | ||||
|             ) | ||||
|             res_dict['publishedDate'] = publishedDate | ||||
|         except: | ||||
|             pass | ||||
|  |  | |||
|  | @ -61,6 +61,7 @@ category_to_keyword = { | |||
| # search-url | ||||
| url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     """Qwant search request""" | ||||
|     keyword = category_to_keyword[categories[0]] | ||||
|  | @ -111,7 +112,14 @@ def response(resp): | |||
| 
 | ||||
|     # check for an API error | ||||
|     if search_results.get('status') != 'success': | ||||
|         msg = ",".join(data.get('message', ['unknown', ])) | ||||
|         msg = ",".join( | ||||
|             data.get( | ||||
|                 'message', | ||||
|                 [ | ||||
|                     'unknown', | ||||
|                 ], | ||||
|             ) | ||||
|         ) | ||||
|         raise SearxEngineAPIException('API error::' + msg) | ||||
| 
 | ||||
|     # raise for other errors | ||||
|  | @ -153,11 +161,13 @@ def response(resp): | |||
| 
 | ||||
|             if mainline_type == 'web': | ||||
|                 content = item['desc'] | ||||
|                 results.append({ | ||||
|                 results.append( | ||||
|                     { | ||||
|                         'title': title, | ||||
|                         'url': res_url, | ||||
|                         'content': content, | ||||
|                 }) | ||||
|                     } | ||||
|                 ) | ||||
| 
 | ||||
|             elif mainline_type == 'news': | ||||
| 
 | ||||
|  | @ -168,23 +178,27 @@ def response(resp): | |||
|                 img_src = None | ||||
|                 if news_media: | ||||
|                     img_src = news_media[0].get('pict', {}).get('url', None) | ||||
|                 results.append({ | ||||
|                 results.append( | ||||
|                     { | ||||
|                         'title': title, | ||||
|                         'url': res_url, | ||||
|                         'publishedDate': pub_date, | ||||
|                         'img_src': img_src, | ||||
|                 }) | ||||
|                     } | ||||
|                 ) | ||||
| 
 | ||||
|             elif mainline_type == 'images': | ||||
|                 thumbnail = item['thumbnail'] | ||||
|                 img_src = item['media'] | ||||
|                 results.append({ | ||||
|                 results.append( | ||||
|                     { | ||||
|                         'title': title, | ||||
|                         'url': res_url, | ||||
|                         'template': 'images.html', | ||||
|                         'thumbnail_src': thumbnail, | ||||
|                         'img_src': img_src, | ||||
|                 }) | ||||
|                     } | ||||
|                 ) | ||||
| 
 | ||||
|             elif mainline_type == 'videos': | ||||
|                 # some videos do not have a description: while qwant-video | ||||
|  | @ -208,11 +222,9 @@ def response(resp): | |||
|                 thumbnail = item['thumbnail'] | ||||
|                 # from some locations (DE and others?) the s2 link do | ||||
|                 # response a 'Please wait ..' but does not deliver the thumbnail | ||||
|                 thumbnail = thumbnail.replace( | ||||
|                     'https://s2.qwant.com', | ||||
|                     'https://s1.qwant.com', 1 | ||||
|                 ) | ||||
|                 results.append({ | ||||
|                 thumbnail = thumbnail.replace('https://s2.qwant.com', 'https://s1.qwant.com', 1) | ||||
|                 results.append( | ||||
|                     { | ||||
|                         'title': title, | ||||
|                         'url': res_url, | ||||
|                         'content': content, | ||||
|  | @ -220,7 +232,8 @@ def response(resp): | |||
|                         'thumbnail': thumbnail, | ||||
|                         'template': 'videos.html', | ||||
|                         'length': length, | ||||
|             }) | ||||
|                     } | ||||
|                 ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
|  |  | |||
|  | @ -28,18 +28,12 @@ mount_prefix = None | |||
| dl_prefix = None | ||||
| 
 | ||||
| # embedded | ||||
| embedded_url = '<{ttype} controls height="166px" ' +\ | ||||
|     'src="{url}" type="{mtype}"></{ttype}>' | ||||
| embedded_url = '<{ttype} controls height="166px" ' + 'src="{url}" type="{mtype}"></{ttype}>' | ||||
| 
 | ||||
| 
 | ||||
| # helper functions | ||||
| def get_time_range(time_range): | ||||
|     sw = { | ||||
|         'day': 1, | ||||
|         'week': 7, | ||||
|         'month': 30, | ||||
|         'year': 365 | ||||
|     } | ||||
|     sw = {'day': 1, 'week': 7, 'month': 30, 'year': 365} | ||||
| 
 | ||||
|     offset = sw.get(time_range, 0) | ||||
|     if not offset: | ||||
|  | @ -52,11 +46,9 @@ def get_time_range(time_range): | |||
| def request(query, params): | ||||
|     search_after = get_time_range(params['time_range']) | ||||
|     search_url = base_url + 'json?{query}&highlight=0' | ||||
|     params['url'] = search_url.format(query=urlencode({ | ||||
|         'query': query, | ||||
|         'page': params['pageno'], | ||||
|         'after': search_after, | ||||
|         'dir': search_dir})) | ||||
|     params['url'] = search_url.format( | ||||
|         query=urlencode({'query': query, 'page': params['pageno'], 'after': search_after, 'dir': search_dir}) | ||||
|     ) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -76,10 +68,7 @@ def response(resp): | |||
|         content = '{}'.format(result['snippet']) | ||||
| 
 | ||||
|         # append result | ||||
|         item = {'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'template': 'files.html'} | ||||
|         item = {'url': url, 'title': title, 'content': content, 'template': 'files.html'} | ||||
| 
 | ||||
|         if result['size']: | ||||
|             item['size'] = int(result['size']) | ||||
|  | @ -96,9 +85,8 @@ def response(resp): | |||
| 
 | ||||
|             if mtype in ['audio', 'video']: | ||||
|                 item['embedded'] = embedded_url.format( | ||||
|                     ttype=mtype, | ||||
|                     url=quote(url.encode('utf8'), '/:'), | ||||
|                     mtype=result['mtype']) | ||||
|                     ttype=mtype, url=quote(url.encode('utf8'), '/:'), mtype=result['mtype'] | ||||
|                 ) | ||||
| 
 | ||||
|             if mtype in ['image'] and subtype in ['bmp', 'gif', 'jpeg', 'png']: | ||||
|                 item['img_src'] = url | ||||
|  |  | |||
|  | @ -52,10 +52,7 @@ def response(resp): | |||
|         data = post['data'] | ||||
| 
 | ||||
|         # extract post information | ||||
|         params = { | ||||
|             'url': urljoin(base_url, data['permalink']), | ||||
|             'title': data['title'] | ||||
|         } | ||||
|         params = {'url': urljoin(base_url, data['permalink']), 'title': data['title']} | ||||
| 
 | ||||
|         # if thumbnail field contains a valid URL, we need to change template | ||||
|         thumbnail = data['thumbnail'] | ||||
|  |  | |||
|  | @ -20,6 +20,8 @@ result_template = 'key-value.html' | |||
| exact_match_only = True | ||||
| 
 | ||||
| _redis_client = None | ||||
| 
 | ||||
| 
 | ||||
| def init(_engine_settings): | ||||
|     global _redis_client  # pylint: disable=global-statement | ||||
|     _redis_client = redis.StrictRedis( | ||||
|  | @ -30,6 +32,7 @@ def init(_engine_settings): | |||
|         decode_responses=True, | ||||
|     ) | ||||
| 
 | ||||
| 
 | ||||
| def search(query, _params): | ||||
|     if not exact_match_only: | ||||
|         return search_keys(query) | ||||
|  | @ -42,21 +45,20 @@ def search(query, _params): | |||
|     if ' ' in query: | ||||
|         qset, rest = query.split(' ', 1) | ||||
|         ret = [] | ||||
|         for res in _redis_client.hscan_iter( | ||||
|                 qset, match='*{}*'.format(rest) | ||||
|         ): | ||||
|             ret.append({ | ||||
|         for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)): | ||||
|             ret.append( | ||||
|                 { | ||||
|                     res[0]: res[1], | ||||
|                     'template': result_template, | ||||
|             }) | ||||
|                 } | ||||
|             ) | ||||
|         return ret | ||||
|     return [] | ||||
| 
 | ||||
| 
 | ||||
| def search_keys(query): | ||||
|     ret = [] | ||||
|     for key in _redis_client.scan_iter( | ||||
|             match='*{}*'.format(query) | ||||
|     ): | ||||
|     for key in _redis_client.scan_iter(match='*{}*'.format(query)): | ||||
|         key_type = _redis_client.type(key) | ||||
|         res = None | ||||
| 
 | ||||
|  |  | |||
|  | @ -68,7 +68,8 @@ def response(resp): | |||
|         else: | ||||
|             content = f"{views} views - {rumbles} rumbles" | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|  | @ -77,5 +78,6 @@ def response(resp): | |||
|                 'template': 'videos.html', | ||||
|                 'publishedDate': fixed_date, | ||||
|                 'thumbnail': thumbnail, | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
|     return results | ||||
|  |  | |||
|  | @ -32,12 +32,16 @@ def request(query, params): | |||
|     params['url'] = search_url | ||||
|     params['method'] = 'POST' | ||||
|     params['headers']['Content-type'] = "application/json" | ||||
|     params['data'] = dumps({"query": query, | ||||
|     params['data'] = dumps( | ||||
|         { | ||||
|             "query": query, | ||||
|             "searchField": "ALL", | ||||
|             "sortDirection": "ASC", | ||||
|             "sortOrder": "RELEVANCY", | ||||
|             "page": params['pageno'], | ||||
|                             "pageSize": page_size}) | ||||
|             "pageSize": page_size, | ||||
|         } | ||||
|     ) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -69,11 +73,15 @@ def response(resp): | |||
|             content = result['highlights'][0]['value'] | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url + 'structure/' + result['id'], | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url + 'structure/' + result['id'], | ||||
|                 'title': result['label'], | ||||
|                 # 'thumbnail': thumbnail, | ||||
|                 'img_src': thumbnail, | ||||
|                         'content': html_to_text(content)}) | ||||
|                 'content': html_to_text(content), | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -25,10 +25,7 @@ url = 'https://searchcode.com/' | |||
| search_url = url + 'api/codesearch_I/?{query}&p={pageno}' | ||||
| 
 | ||||
| # special code-endings which are not recognised by the file ending | ||||
| code_endings = {'cs': 'c#', | ||||
|                 'h': 'c', | ||||
|                 'hpp': 'cpp', | ||||
|                 'cxx': 'cpp'} | ||||
| code_endings = {'cs': 'c#', 'h': 'c', 'hpp': 'cpp', 'cxx': 'cpp'} | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
|  | @ -55,17 +52,21 @@ def response(resp): | |||
|             lines[int(line)] = code | ||||
| 
 | ||||
|         code_language = code_endings.get( | ||||
|             result['filename'].split('.')[-1].lower(), | ||||
|             result['filename'].split('.')[-1].lower()) | ||||
|             result['filename'].split('.')[-1].lower(), result['filename'].split('.')[-1].lower() | ||||
|         ) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': href, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': href, | ||||
|                 'title': title, | ||||
|                 'content': '', | ||||
|                 'repository': repo, | ||||
|                 'codelines': sorted(lines.items()), | ||||
|                 'code_language': code_language, | ||||
|                         'template': 'code.html'}) | ||||
|                 'template': 'code.html', | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -37,7 +37,7 @@ def request(query, params): | |||
|         'language': params['language'], | ||||
|         'time_range': params['time_range'], | ||||
|         'category': params['category'], | ||||
|         'format': 'json' | ||||
|         'format': 'json', | ||||
|     } | ||||
| 
 | ||||
|     return params | ||||
|  |  | |||
|  | @ -13,7 +13,8 @@ def request(query, params): | |||
|     params['url'] = search_url | ||||
|     params['method'] = 'POST' | ||||
|     params['headers']['content-type'] = 'application/json' | ||||
|     params['data'] = dumps({ | ||||
|     params['data'] = dumps( | ||||
|         { | ||||
|             "queryString": query, | ||||
|             "page": params['pageno'], | ||||
|             "pageSize": 10, | ||||
|  | @ -25,7 +26,8 @@ def request(query, params): | |||
|             "coAuthors": [], | ||||
|             "venues": [], | ||||
|             "performTitleMatch": True, | ||||
|     }) | ||||
|         } | ||||
|     ) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
|  | @ -33,10 +35,12 @@ def response(resp): | |||
|     res = loads(resp.text) | ||||
|     results = [] | ||||
|     for result in res['results']: | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': result['primaryPaperLink']['url'], | ||||
|                 'title': result['title']['text'], | ||||
|             'content': result['paperAbstractTruncated'] | ||||
|         }) | ||||
|                 'content': result['paperAbstractTruncated'], | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -31,17 +31,13 @@ supported_languages = [ | |||
| ] | ||||
| base_url = 'https://sepiasearch.org/api/v1/search/videos' | ||||
| 
 | ||||
| safesearch_table = { | ||||
|     0: 'both', | ||||
|     1: 'false', | ||||
|     2: 'false' | ||||
| } | ||||
| safesearch_table = {0: 'both', 1: 'false', 2: 'false'} | ||||
| 
 | ||||
| time_range_table = { | ||||
|     'day': relativedelta.relativedelta(), | ||||
|     'week': relativedelta.relativedelta(weeks=-1), | ||||
|     'month': relativedelta.relativedelta(months=-1), | ||||
|     'year': relativedelta.relativedelta(years=-1) | ||||
|     'year': relativedelta.relativedelta(years=-1), | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
|  | @ -55,13 +51,19 @@ def minute_to_hm(minute): | |||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = base_url + '?' + urlencode({ | ||||
|     params['url'] = ( | ||||
|         base_url | ||||
|         + '?' | ||||
|         + urlencode( | ||||
|             { | ||||
|                 'search': query, | ||||
|                 'start': (params['pageno'] - 1) * 10, | ||||
|                 'count': 10, | ||||
|                 'sort': '-match', | ||||
|         'nsfw': safesearch_table[params['safesearch']] | ||||
|     }) | ||||
|                 'nsfw': safesearch_table[params['safesearch']], | ||||
|             } | ||||
|         ) | ||||
|     ) | ||||
| 
 | ||||
|     language = params['language'].split('-')[0] | ||||
|     if language in supported_languages: | ||||
|  | @ -91,7 +93,9 @@ def response(resp): | |||
|         length = minute_to_hm(result.get('duration')) | ||||
|         url = result['url'] | ||||
| 
 | ||||
|         results.append({'url': url, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': content, | ||||
|                 'author': author, | ||||
|  | @ -99,6 +103,8 @@ def response(resp): | |||
|                 'template': 'videos.html', | ||||
|                 'publishedDate': publishedDate, | ||||
|                 'embedded': embedded, | ||||
|                         'thumbnail': thumbnail}) | ||||
|                 'thumbnail': thumbnail, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -58,10 +58,12 @@ def response(resp): | |||
|         if result_data is None: | ||||
|             continue | ||||
|         title_element = eval_xpath_getindex(result_element, './/h3/a', 0) | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': title_element.get('href'), | ||||
|                 'title': extract_text(title_element), | ||||
|                 'content': extract_text(eval_xpath(result_data, './/div[@class="_3eded7"]')), | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -28,9 +28,11 @@ URL = 'https://sjp.pwn.pl' | |||
| SEARCH_URL = URL + '/szukaj/{query}.html' | ||||
| 
 | ||||
| word_xpath = '//div[@class="query"]' | ||||
| dict_xpath = ['//div[@class="wyniki sjp-so-wyniki sjp-so-anchor"]', | ||||
| dict_xpath = [ | ||||
|     '//div[@class="wyniki sjp-so-wyniki sjp-so-anchor"]', | ||||
|     '//div[@class="wyniki sjp-wyniki sjp-anchor"]', | ||||
|               '//div[@class="wyniki sjp-doroszewski-wyniki sjp-doroszewski-anchor"]'] | ||||
|     '//div[@class="wyniki sjp-doroszewski-wyniki sjp-doroszewski-anchor"]', | ||||
| ] | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|  | @ -85,9 +87,11 @@ def response(resp): | |||
|                 infobox += "</ol>" | ||||
|         infobox += "</ul></div>" | ||||
| 
 | ||||
|     results.append({ | ||||
|     results.append( | ||||
|         { | ||||
|             'infobox': word, | ||||
|             'content': infobox, | ||||
|     }) | ||||
|         } | ||||
|     ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -36,7 +36,8 @@ def response(resp): | |||
|     search_results = loads(resp.text) | ||||
| 
 | ||||
|     for result in search_results["results"]: | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'infohash': result["infohash"], | ||||
|                 'seed': result["swarm"]["seeders"], | ||||
|                 'leech': result["swarm"]["leechers"], | ||||
|  | @ -45,5 +46,6 @@ def response(resp): | |||
|                 'filesize': result["size"], | ||||
|                 'magnetlink': result["magnet"], | ||||
|                 'template': "torrent.html", | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
|     return results | ||||
|  |  | |||
|  | @ -27,17 +27,21 @@ paging = True | |||
| # search-url | ||||
| # missing attribute: user_id, app_version, app_locale | ||||
| url = 'https://api-v2.soundcloud.com/' | ||||
| search_url = url + 'search?{query}'\ | ||||
|                          '&variant_ids='\ | ||||
|                          '&facet=model'\ | ||||
|                          '&limit=20'\ | ||||
|                          '&offset={offset}'\ | ||||
|                          '&linked_partitioning=1'\ | ||||
|                          '&client_id={client_id}'   # noqa | ||||
| search_url = ( | ||||
|     url + 'search?{query}' | ||||
|     '&variant_ids=' | ||||
|     '&facet=model' | ||||
|     '&limit=20' | ||||
|     '&offset={offset}' | ||||
|     '&linked_partitioning=1' | ||||
|     '&client_id={client_id}' | ||||
| )  # noqa | ||||
| 
 | ||||
| embedded_url = '<iframe width="100%" height="166" ' +\ | ||||
|     'scrolling="no" frameborder="no" ' +\ | ||||
|     'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>' | ||||
| embedded_url = ( | ||||
|     '<iframe width="100%" height="166" ' | ||||
|     + 'scrolling="no" frameborder="no" ' | ||||
|     + 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>' | ||||
| ) | ||||
| 
 | ||||
| cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U) | ||||
| guest_client_id = '' | ||||
|  | @ -75,9 +79,7 @@ def init(engine_settings=None): | |||
| def request(query, params): | ||||
|     offset = (params['pageno'] - 1) * 20 | ||||
| 
 | ||||
|     params['url'] = search_url.format(query=urlencode({'q': query}), | ||||
|                                       offset=offset, | ||||
|                                       client_id=guest_client_id) | ||||
|     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset, client_id=guest_client_id) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -98,11 +100,15 @@ def response(resp): | |||
|             embedded = embedded_url.format(uri=uri) | ||||
| 
 | ||||
|             # append result | ||||
|             results.append({'url': result['permalink_url'], | ||||
|             results.append( | ||||
|                 { | ||||
|                     'url': result['permalink_url'], | ||||
|                     'title': title, | ||||
|                     'publishedDate': publishedDate, | ||||
|                     'embedded': embedded, | ||||
|                             'content': content}) | ||||
|                     'content': content, | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -42,9 +42,10 @@ def request(query, params): | |||
|     r = http_post( | ||||
|         'https://accounts.spotify.com/api/token', | ||||
|         data={'grant_type': 'client_credentials'}, | ||||
|         headers={'Authorization': 'Basic ' + base64.b64encode( | ||||
|             "{}:{}".format(api_client_id, api_client_secret).encode() | ||||
|         ).decode()} | ||||
|         headers={ | ||||
|             'Authorization': 'Basic ' | ||||
|             + base64.b64encode("{}:{}".format(api_client_id, api_client_secret).encode()).decode() | ||||
|         }, | ||||
|     ) | ||||
|     j = loads(r.text) | ||||
|     params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))} | ||||
|  | @ -63,18 +64,12 @@ def response(resp): | |||
|         if result['type'] == 'track': | ||||
|             title = result['name'] | ||||
|             url = result['external_urls']['spotify'] | ||||
|             content = '{} - {} - {}'.format( | ||||
|                 result['artists'][0]['name'], | ||||
|                 result['album']['name'], | ||||
|                 result['name']) | ||||
|             content = '{} - {} - {}'.format(result['artists'][0]['name'], result['album']['name'], result['name']) | ||||
| 
 | ||||
|             embedded = embedded_url.format(audioid=result['id']) | ||||
| 
 | ||||
|             # append result | ||||
|             results.append({'url': url, | ||||
|                             'title': title, | ||||
|                             'embedded': embedded, | ||||
|                             'content': content}) | ||||
|             results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -26,15 +26,11 @@ api_key = 'unset' | |||
| 
 | ||||
| base_url = 'https://api.springernature.com/metadata/json?' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     if api_key == 'unset': | ||||
|         raise SearxEngineAPIException('missing Springer-Nature API key') | ||||
|     args = urlencode({ | ||||
|         'q' : query, | ||||
|         's' : nb_per_page * (params['pageno'] - 1), | ||||
|         'p' : nb_per_page, | ||||
|         'api_key' : api_key | ||||
|         }) | ||||
|     args = urlencode({'q': query, 's': nb_per_page * (params['pageno'] - 1), 'p': nb_per_page, 'api_key': api_key}) | ||||
|     params['url'] = base_url + args | ||||
|     logger.debug("query_url --> %s", params['url']) | ||||
|     return params | ||||
|  | @ -50,21 +46,27 @@ def response(resp): | |||
|             content += "..." | ||||
|         published = datetime.strptime(record['publicationDate'], '%Y-%m-%d') | ||||
| 
 | ||||
|         metadata = [record[x] for x in [ | ||||
|         metadata = [ | ||||
|             record[x] | ||||
|             for x in [ | ||||
|                 'publicationName', | ||||
|                 'identifier', | ||||
|                 'contentType', | ||||
|         ] if record.get(x) is not None] | ||||
|             ] | ||||
|             if record.get(x) is not None | ||||
|         ] | ||||
| 
 | ||||
|         metadata = ' / '.join(metadata) | ||||
|         if record.get('startingPage') and record.get('endingPage') is not None: | ||||
|             metadata += " (%(startingPage)s-%(endingPage)s)" % record | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'title': record['title'], | ||||
|                 'url': record['url'][0]['value'].replace('http://', 'https://', 1), | ||||
|                 'content': content, | ||||
|                 'publishedDate': published, | ||||
|             'metadata' : metadata | ||||
|         }) | ||||
|                 'metadata': metadata, | ||||
|             } | ||||
|         ) | ||||
|     return results | ||||
|  |  | |||
|  | @ -49,7 +49,7 @@ def search(query, params): | |||
|         'query': query, | ||||
|         'wildcard': r'%' + query.replace(' ', r'%') + r'%', | ||||
|         'limit': limit, | ||||
|         'offset': (params['pageno'] - 1) * limit | ||||
|         'offset': (params['pageno'] - 1) * limit, | ||||
|     } | ||||
|     query_to_run = query_str + ' LIMIT :limit OFFSET :offset' | ||||
| 
 | ||||
|  |  | |||
|  | @ -29,20 +29,24 @@ api_order = 'desc' | |||
| # https://api.stackexchange.com/docs/advanced-search | ||||
| search_api = 'https://api.stackexchange.com/2.3/search/advanced?' | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     args = urlencode({ | ||||
|     args = urlencode( | ||||
|         { | ||||
|             'q': query, | ||||
|             'page': params['pageno'], | ||||
|             'pagesize': pagesize, | ||||
|             'site': api_site, | ||||
|             'sort': api_sort, | ||||
|             'order': 'desc', | ||||
|         }) | ||||
|         } | ||||
|     ) | ||||
|     params['url'] = search_api + args | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
| 
 | ||||
|     results = [] | ||||
|  | @ -56,10 +60,12 @@ def response(resp): | |||
|             content += ' // is answered' | ||||
|         content += " // score: %s" % result['score'] | ||||
| 
 | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': "https://%s.com/q/%s" % (api_site, result['question_id']), | ||||
|                 'title': html.unescape(result['title']), | ||||
|                 'content': html.unescape(content), | ||||
|         }) | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -123,15 +123,10 @@ def response(resp): | |||
| 
 | ||||
|         if published_date: | ||||
|             # append result | ||||
|             results.append({'url': url, | ||||
|                             'title': title, | ||||
|                             'content': content, | ||||
|                             'publishedDate': published_date}) | ||||
|             results.append({'url': url, 'title': title, 'content': content, 'publishedDate': published_date}) | ||||
|         else: | ||||
|             # append result | ||||
|             results.append({'url': url, | ||||
|                             'title': title, | ||||
|                             'content': content}) | ||||
|             results.append({'url': url, 'title': title, 'content': content}) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  | @ -152,7 +147,7 @@ def _fetch_supported_languages(resp): | |||
|         'malayam': 'ml', | ||||
|         'norsk': 'nb', | ||||
|         'sinhalese': 'si', | ||||
|         'sudanese': 'su' | ||||
|         'sudanese': 'su', | ||||
|     } | ||||
| 
 | ||||
|     # get the English name of every language known by babel | ||||
|  |  | |||
|  | @ -56,11 +56,7 @@ def response(resp): | |||
|         name_row = rows[i] | ||||
| 
 | ||||
|         links = name_row.xpath('./td[@class="desc-top"]/a') | ||||
|         params = { | ||||
|             'template': 'torrent.html', | ||||
|             'url': links[-1].attrib.get('href'), | ||||
|             'title': extract_text(links[-1]) | ||||
|         } | ||||
|         params = {'template': 'torrent.html', 'url': links[-1].attrib.get('href'), 'title': extract_text(links[-1])} | ||||
|         # I have not yet seen any torrents without magnet links, but | ||||
|         # it's better to be prepared to stumble upon one some day | ||||
|         if len(links) == 2: | ||||
|  |  | |||
|  | @ -35,10 +35,12 @@ api_key = '' | |||
| # https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories | ||||
| torznab_categories = [] | ||||
| 
 | ||||
| 
 | ||||
| def init(engine_settings=None):  # pylint: disable=unused-argument | ||||
|     if len(base_url) < 1: | ||||
|         raise ValueError('missing torznab base_url') | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
| 
 | ||||
|     search_url = base_url + '?t=search&q={search_query}' | ||||
|  | @ -48,13 +50,12 @@ def request(query, params): | |||
|         search_url += '&cat={torznab_categories}' | ||||
| 
 | ||||
|     params['url'] = search_url.format( | ||||
|         search_query = quote(query), | ||||
|         api_key = api_key, | ||||
|         torznab_categories = ",".join([str(x) for x in torznab_categories]) | ||||
|         search_query=quote(query), api_key=api_key, torznab_categories=",".join([str(x) for x in torznab_categories]) | ||||
|     ) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     results = [] | ||||
| 
 | ||||
|  | @ -103,8 +104,7 @@ def response(resp): | |||
| 
 | ||||
|         result["publishedDate"] = None | ||||
|         try: | ||||
|             result["publishedDate"] = datetime.strptime( | ||||
|                 get_property(item, 'pubDate'), '%a, %d %b %Y %H:%M:%S %z') | ||||
|             result["publishedDate"] = datetime.strptime(get_property(item, 'pubDate'), '%a, %d %b %Y %H:%M:%S %z') | ||||
|         except (ValueError, TypeError) as e: | ||||
|             logger.debug("ignore exception (publishedDate): %s", e) | ||||
| 
 | ||||
|  | @ -134,9 +134,7 @@ def get_property(item, property_name): | |||
| def get_torznab_attr(item, attr_name): | ||||
|     element = item.find( | ||||
|         './/torznab:attr[@name="{attr_name}"]'.format(attr_name=attr_name), | ||||
|         { | ||||
|             'torznab': 'http://torznab.com/schemas/2015/feed' | ||||
|         } | ||||
|         {'torznab': 'http://torznab.com/schemas/2015/feed'}, | ||||
|     ) | ||||
| 
 | ||||
|     if element is not None: | ||||
|  |  | |||
|  | @ -28,24 +28,25 @@ def request(query, params): | |||
|         key_form = '&key=' + api_key | ||||
|     else: | ||||
|         key_form = '' | ||||
|     params['url'] = url.format(from_lang=params['from_lang'][1], | ||||
|                                to_lang=params['to_lang'][1], | ||||
|                                query=params['query'], | ||||
|                                key=key_form) | ||||
|     params['url'] = url.format( | ||||
|         from_lang=params['from_lang'][1], to_lang=params['to_lang'][1], query=params['query'], key=key_form | ||||
|     ) | ||||
|     return params | ||||
| 
 | ||||
| 
 | ||||
| def response(resp): | ||||
|     results = [] | ||||
|     results.append({ | ||||
|     results.append( | ||||
|         { | ||||
|             'url': web_url.format( | ||||
|                 from_lang=resp.search_params['from_lang'][2], | ||||
|                 to_lang=resp.search_params['to_lang'][2], | ||||
|             query=resp.search_params['query']), | ||||
|                 query=resp.search_params['query'], | ||||
|             ), | ||||
|             'title': '[{0}-{1}] {2}'.format( | ||||
|             resp.search_params['from_lang'][1], | ||||
|             resp.search_params['to_lang'][1], | ||||
|             resp.search_params['query']), | ||||
|         'content': resp.json()['responseData']['translatedText'] | ||||
|     }) | ||||
|                 resp.search_params['from_lang'][1], resp.search_params['to_lang'][1], resp.search_params['query'] | ||||
|             ), | ||||
|             'content': resp.json()['responseData']['translatedText'], | ||||
|         } | ||||
|     ) | ||||
|     return results | ||||
|  |  | |||
|  | @ -26,23 +26,13 @@ paging = True | |||
| 
 | ||||
| def clean_url(url): | ||||
|     parsed = urlparse(url) | ||||
|     query = [(k, v) for (k, v) | ||||
|              in parse_qsl(parsed.query) if k not in ['ixid', 's']] | ||||
|     query = [(k, v) for (k, v) in parse_qsl(parsed.query) if k not in ['ixid', 's']] | ||||
| 
 | ||||
|     return urlunparse(( | ||||
|         parsed.scheme, | ||||
|         parsed.netloc, | ||||
|         parsed.path, | ||||
|         parsed.params, | ||||
|         urlencode(query), | ||||
|         parsed.fragment | ||||
|     )) | ||||
|     return urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlencode(query), parsed.fragment)) | ||||
| 
 | ||||
| 
 | ||||
| def request(query, params): | ||||
|     params['url'] = search_url + urlencode({ | ||||
|         'query': query, 'page': params['pageno'], 'per_page': page_size | ||||
|     }) | ||||
|     params['url'] = search_url + urlencode({'query': query, 'page': params['pageno'], 'per_page': page_size}) | ||||
|     logger.debug("query_url --> %s", params['url']) | ||||
|     return params | ||||
| 
 | ||||
|  | @ -53,13 +43,15 @@ def response(resp): | |||
| 
 | ||||
|     if 'results' in json_data: | ||||
|         for result in json_data['results']: | ||||
|             results.append({ | ||||
|             results.append( | ||||
|                 { | ||||
|                     'template': 'images.html', | ||||
|                     'url': clean_url(result['links']['html']), | ||||
|                     'thumbnail_src': clean_url(result['urls']['thumb']), | ||||
|                     'img_src': clean_url(result['urls']['raw']), | ||||
|                     'title': result.get('alt_description') or 'unknown', | ||||
|                 'content': result.get('description') or '' | ||||
|             }) | ||||
|                     'content': result.get('description') or '', | ||||
|                 } | ||||
|             ) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
|  | @ -25,15 +25,16 @@ paging = True | |||
| base_url = 'https://vimeo.com/' | ||||
| search_url = base_url + '/search/page:{pageno}?{query}' | ||||
| 
 | ||||
| embedded_url = '<iframe data-src="https://player.vimeo.com/video/{videoid}" ' +\ | ||||
|     'width="540" height="304" frameborder="0" ' +\ | ||||
|     'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>' | ||||
| embedded_url = ( | ||||
|     '<iframe data-src="https://player.vimeo.com/video/{videoid}" ' | ||||
|     + 'width="540" height="304" frameborder="0" ' | ||||
|     + 'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>' | ||||
| ) | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
| def request(query, params): | ||||
|     params['url'] = search_url.format(pageno=params['pageno'], | ||||
|                                       query=urlencode({'q': query})) | ||||
|     params['url'] = search_url.format(pageno=params['pageno'], query=urlencode({'q': query})) | ||||
| 
 | ||||
|     return params | ||||
| 
 | ||||
|  | @ -56,13 +57,17 @@ def response(resp): | |||
|         embedded = embedded_url.format(videoid=videoid) | ||||
| 
 | ||||
|         # append result | ||||
|         results.append({'url': url, | ||||
|         results.append( | ||||
|             { | ||||
|                 'url': url, | ||||
|                 'title': title, | ||||
|                 'content': '', | ||||
|                 'template': 'videos.html', | ||||
|                 'publishedDate': publishedDate, | ||||
|                 'embedded': embedded, | ||||
|                         'thumbnail': thumbnail}) | ||||
|                 'thumbnail': thumbnail, | ||||
|             } | ||||
|         ) | ||||
| 
 | ||||
|     # return results | ||||
|     return results | ||||
|  |  | |||
|  | @ -14,7 +14,10 @@ from searx.data import WIKIDATA_UNITS | |||
| from searx.network import post, get | ||||
| from searx.utils import match_language, searx_useragent, get_string_replaces_function | ||||
| from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom | ||||
| from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import | ||||
| from searx.engines.wikipedia import ( | ||||
|     _fetch_supported_languages, | ||||
|     supported_languages_url, | ||||
| )  # NOQA # pylint: disable=unused-import | ||||
| 
 | ||||
| # about | ||||
| about = { | ||||
|  | @ -112,10 +115,7 @@ replace_http_by_https = get_string_replaces_function({'http:': 'https:'}) | |||
| 
 | ||||
| def get_headers(): | ||||
|     # user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits | ||||
|     return { | ||||
|         'Accept': 'application/sparql-results+json', | ||||
|         'User-Agent': searx_useragent() | ||||
|     } | ||||
|     return {'Accept': 'application/sparql-results+json', 'User-Agent': searx_useragent()} | ||||
| 
 | ||||
| 
 | ||||
| def get_label_for_entity(entity_id, language): | ||||
|  | @ -211,9 +211,9 @@ def get_results(attribute_result, attributes, language): | |||
|                         results.append({'title': infobox_title, 'url': url}) | ||||
|                     # update the infobox_id with the wikipedia URL | ||||
|                     # first the local wikipedia URL, and as fallback the english wikipedia URL | ||||
|                     if attribute_type == WDArticle\ | ||||
|                        and ((attribute.language == 'en' and infobox_id_lang is None) | ||||
|                             or attribute.language != 'en'): | ||||
|                     if attribute_type == WDArticle and ( | ||||
|                         (attribute.language == 'en' and infobox_id_lang is None) or attribute.language != 'en' | ||||
|                     ): | ||||
|                         infobox_id_lang = attribute.language | ||||
|                         infobox_id = url | ||||
|             elif attribute_type == WDImageAttribute: | ||||
|  | @ -232,13 +232,11 @@ def get_results(attribute_result, attributes, language): | |||
|                 osm_zoom = area_to_osm_zoom(area) if area else 19 | ||||
|                 url = attribute.get_geo_url(attribute_result, osm_zoom=osm_zoom) | ||||
|                 if url: | ||||
|                     infobox_urls.append({'title': attribute.get_label(language), | ||||
|                                          'url': url, | ||||
|                                          'entity': attribute.name}) | ||||
|                     infobox_urls.append({'title': attribute.get_label(language), 'url': url, 'entity': attribute.name}) | ||||
|             else: | ||||
|                 infobox_attributes.append({'label': attribute.get_label(language), | ||||
|                                            'value': value, | ||||
|                                            'entity': attribute.name}) | ||||
|                 infobox_attributes.append( | ||||
|                     {'label': attribute.get_label(language), 'value': value, 'entity': attribute.name} | ||||
|                 ) | ||||
| 
 | ||||
|     if infobox_id: | ||||
|         infobox_id = replace_http_by_https(infobox_id) | ||||
|  | @ -246,22 +244,19 @@ def get_results(attribute_result, attributes, language): | |||
|     # add the wikidata URL at the end | ||||
|     infobox_urls.append({'title': 'Wikidata', 'url': attribute_result['item']}) | ||||
| 
 | ||||
|     if img_src is None and len(infobox_attributes) == 0 and len(infobox_urls) == 1 and\ | ||||
|        len(infobox_content) == 0: | ||||
|         results.append({ | ||||
|             'url': infobox_urls[0]['url'], | ||||
|             'title': infobox_title, | ||||
|             'content': infobox_content | ||||
|         }) | ||||
|     if img_src is None and len(infobox_attributes) == 0 and len(infobox_urls) == 1 and len(infobox_content) == 0: | ||||
|         results.append({'url': infobox_urls[0]['url'], 'title': infobox_title, 'content': infobox_content}) | ||||
|     else: | ||||
|         results.append({ | ||||
|         results.append( | ||||
|             { | ||||
|                 'infobox': infobox_title, | ||||
|                 'id': infobox_id, | ||||
|                 'content': infobox_content, | ||||
|                 'img_src': img_src, | ||||
|                 'urls': infobox_urls, | ||||
|             'attributes': infobox_attributes | ||||
|         }) | ||||
|                 'attributes': infobox_attributes, | ||||
|             } | ||||
|         ) | ||||
|     return results | ||||
| 
 | ||||
| 
 | ||||
|  | @ -271,13 +266,14 @@ def get_query(query, language): | |||
|     where = list(filter(lambda s: len(s) > 0, [a.get_where() for a in attributes])) | ||||
|     wikibase_label = list(filter(lambda s: len(s) > 0, [a.get_wikibase_label() for a in attributes])) | ||||
|     group_by = list(filter(lambda s: len(s) > 0, [a.get_group_by() for a in attributes])) | ||||
|     query = QUERY_TEMPLATE\ | ||||
|         .replace('%QUERY%', sparql_string_escape(query))\ | ||||
|         .replace('%SELECT%', ' '.join(select))\ | ||||
|         .replace('%WHERE%', '\n  '.join(where))\ | ||||
|         .replace('%WIKIBASE_LABELS%', '\n      '.join(wikibase_label))\ | ||||
|         .replace('%GROUP_BY%', ' '.join(group_by))\ | ||||
|     query = ( | ||||
|         QUERY_TEMPLATE.replace('%QUERY%', sparql_string_escape(query)) | ||||
|         .replace('%SELECT%', ' '.join(select)) | ||||
|         .replace('%WHERE%', '\n  '.join(where)) | ||||
|         .replace('%WIKIBASE_LABELS%', '\n      '.join(wikibase_label)) | ||||
|         .replace('%GROUP_BY%', ' '.join(group_by)) | ||||
|         .replace('%LANGUAGE%', language) | ||||
|     ) | ||||
|     return query, attributes | ||||
| 
 | ||||
| 
 | ||||
|  | @ -303,28 +299,34 @@ def get_attributes(language): | |||
|         attributes.append(WDDateAttribute(name)) | ||||
| 
 | ||||
|     # Dates | ||||
|     for p in ['P571',    # inception date | ||||
|     for p in [ | ||||
|         'P571',  # inception date | ||||
|         'P576',  # dissolution date | ||||
|         'P580',  # start date | ||||
|         'P582',  # end date | ||||
|         'P569',  # date of birth | ||||
|         'P570',  # date of death | ||||
|         'P619',  # date of spacecraft launch | ||||
|               'P620']:   # date of spacecraft landing | ||||
|         'P620', | ||||
|     ]:  # date of spacecraft landing | ||||
|         add_date(p) | ||||
| 
 | ||||
|     for p in ['P27',     # country of citizenship | ||||
|     for p in [ | ||||
|         'P27',  # country of citizenship | ||||
|         'P495',  # country of origin | ||||
|         'P17',  # country | ||||
|               'P159']:   # headquarters location | ||||
|         'P159', | ||||
|     ]:  # headquarters location | ||||
|         add_label(p) | ||||
| 
 | ||||
|     # Places | ||||
|     for p in ['P36',     # capital | ||||
|     for p in [ | ||||
|         'P36',  # capital | ||||
|         'P35',  # head of state | ||||
|         'P6',  # head of government | ||||
|         'P122',  # basic form of government | ||||
|               'P37']:    # official language | ||||
|         'P37', | ||||
|     ]:  # official language | ||||
|         add_label(p) | ||||
| 
 | ||||
|     add_value('P1082')  # population | ||||
|  | @ -334,7 +336,8 @@ def get_attributes(language): | |||
|     add_amount('P2048')  # heigth (building) | ||||
| 
 | ||||
|     # Media | ||||
|     for p in ['P400',    # platform (videogames, computing) | ||||
|     for p in [ | ||||
|         'P400',  # platform (videogames, computing) | ||||
|         'P50',  # author | ||||
|         'P170',  # creator | ||||
|         'P57',  # director | ||||
|  | @ -348,7 +351,8 @@ def get_attributes(language): | |||
|         'P123',  # publisher | ||||
|         'P449',  # original network | ||||
|         'P750',  # distributed by | ||||
|               'P86']:    # composer | ||||
|         'P86', | ||||
|     ]:  # composer | ||||
|         add_label(p) | ||||
| 
 | ||||
|     add_date('P577')  # publication date | ||||
|  | @ -415,7 +419,7 @@ def get_attributes(language): | |||
| 
 | ||||
| class WDAttribute: | ||||
| 
 | ||||
|     __slots__ = 'name', | ||||
|     __slots__ = ('name',) | ||||
| 
 | ||||
|     def __init__(self, name): | ||||
|         self.name = name | ||||
|  | @ -443,14 +447,15 @@ class WDAttribute: | |||
| 
 | ||||
| 
 | ||||
| class WDAmountAttribute(WDAttribute): | ||||
| 
 | ||||
|     def get_select(self): | ||||
|         return '?{name} ?{name}Unit'.replace('{name}', self.name) | ||||
| 
 | ||||
|     def get_where(self): | ||||
|         return """  OPTIONAL { ?item p:{name} ?{name}Node . | ||||
|     ?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} . | ||||
|     OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace('{name}', self.name) | ||||
|     OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace( | ||||
|             '{name}', self.name | ||||
|         ) | ||||
| 
 | ||||
|     def get_group_by(self): | ||||
|         return self.get_select() | ||||
|  | @ -484,7 +489,9 @@ class WDArticle(WDAttribute): | |||
|         return """OPTIONAL { ?article{language} schema:about ?item ; | ||||
|              schema:inLanguage "{language}" ; | ||||
|              schema:isPartOf <https://{language}.wikipedia.org/> ; | ||||
|              schema:name ?articleName{language} . }""".replace('{language}', self.language) | ||||
|              schema:name ?articleName{language} . }""".replace( | ||||
|             '{language}', self.language | ||||
|         ) | ||||
| 
 | ||||
|     def get_group_by(self): | ||||
|         return self.get_select() | ||||
|  | @ -495,7 +502,6 @@ class WDArticle(WDAttribute): | |||
| 
 | ||||
| 
 | ||||
| class WDLabelAttribute(WDAttribute): | ||||
| 
 | ||||
|     def get_select(self): | ||||
|         return '(group_concat(distinct ?{name}Label;separator=", ") as ?{name}Labels)'.replace('{name}', self.name) | ||||
| 
 | ||||
|  | @ -533,7 +539,6 @@ class WDURLAttribute(WDAttribute): | |||
| 
 | ||||
| 
 | ||||
| class WDGeoAttribute(WDAttribute): | ||||
| 
 | ||||
|     def get_label(self, language): | ||||
|         return "OpenStreetMap" | ||||
| 
 | ||||
|  | @ -543,7 +548,9 @@ class WDGeoAttribute(WDAttribute): | |||
|     def get_where(self): | ||||
|         return """OPTIONAL { ?item p:{name}/psv:{name} [ | ||||
|     wikibase:geoLatitude ?{name}Lat ; | ||||
|     wikibase:geoLongitude ?{name}Long ] }""".replace('{name}', self.name) | ||||
|     wikibase:geoLongitude ?{name}Long ] }""".replace( | ||||
|             '{name}', self.name | ||||
|         ) | ||||
| 
 | ||||
|     def get_group_by(self): | ||||
|         return self.get_select() | ||||
|  | @ -565,7 +572,7 @@ class WDGeoAttribute(WDAttribute): | |||
| 
 | ||||
| class WDImageAttribute(WDURLAttribute): | ||||
| 
 | ||||
|     __slots__ = 'priority', | ||||
|     __slots__ = ('priority',) | ||||
| 
 | ||||
|     def __init__(self, name, url_id=None, priority=100): | ||||
|         super().__init__(name, url_id) | ||||
|  | @ -573,7 +580,6 @@ class WDImageAttribute(WDURLAttribute): | |||
| 
 | ||||
| 
 | ||||
| class WDDateAttribute(WDAttribute): | ||||
| 
 | ||||
|     def get_select(self): | ||||
|         return '?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar'.replace('{name}', self.name) | ||||
| 
 | ||||
|  | @ -587,7 +593,9 @@ class WDDateAttribute(WDAttribute): | |||
|     wikibase:timePrecision ?{name}timePrecision ; | ||||
|     wikibase:timeTimezone ?{name}timeZone ; | ||||
|     wikibase:timeCalendarModel ?{name}timeCalendar ] . } | ||||
|     hint:Prior hint:rangeSafe true;""".replace('{name}', self.name) | ||||
|     hint:Prior hint:rangeSafe true;""".replace( | ||||
|             '{name}', self.name | ||||
|         ) | ||||
| 
 | ||||
|     def get_group_by(self): | ||||
|         return self.get_select() | ||||
|  | @ -619,11 +627,12 @@ class WDDateAttribute(WDAttribute): | |||
|     def format_13(self, value, locale): | ||||
|         timestamp = isoparse(value) | ||||
|         # precision: minute | ||||
|         return get_datetime_format(format, locale=locale) \ | ||||
|             .replace("'", "") \ | ||||
|             .replace('{0}', format_time(timestamp, 'full', tzinfo=None, | ||||
|                                         locale=locale)) \ | ||||
|         return ( | ||||
|             get_datetime_format(format, locale=locale) | ||||
|             .replace("'", "") | ||||
|             .replace('{0}', format_time(timestamp, 'full', tzinfo=None, locale=locale)) | ||||
|             .replace('{1}', format_date(timestamp, 'short', locale=locale)) | ||||
|         ) | ||||
| 
 | ||||
|     def format_14(self, value, locale): | ||||
|         # precision: second. | ||||
|  | @ -644,7 +653,7 @@ class WDDateAttribute(WDAttribute): | |||
|         '11': ('format_11', 0),  # day | ||||
|         '12': ('format_13', 0),  # hour (not supported by babel, display minute) | ||||
|         '13': ('format_13', 0),  # minute | ||||
|         '14': ('format_14', 0)  # second | ||||
|         '14': ('format_14', 0),  # second | ||||
|     } | ||||
| 
 | ||||
|     def get_str(self, result, language): | ||||
|  |  | |||
|  | @ -39,8 +39,7 @@ def request(query, params): | |||
|         query = query.title() | ||||
| 
 | ||||
|     language = url_lang(params['language']) | ||||
|     params['url'] = search_url.format(title=quote(query), | ||||
|                                       language=language) | ||||
|     params['url'] = search_url.format(title=quote(query), language=language) | ||||
| 
 | ||||
|     if params['language'].lower() in language_variants.get(language, []): | ||||
|         params['headers']['Accept-Language'] = params['language'].lower() | ||||
|  | @ -63,8 +62,10 @@ def response(resp): | |||
|         except: | ||||
|             pass | ||||
|         else: | ||||
|             if api_result['type'] == 'https://mediawiki.org/wiki/HyperSwitch/errors/bad_request' \ | ||||
|                and api_result['detail'] == 'title-invalid-characters': | ||||
|             if ( | ||||
|                 api_result['type'] == 'https://mediawiki.org/wiki/HyperSwitch/errors/bad_request' | ||||
|                 and api_result['detail'] == 'title-invalid-characters' | ||||
|             ): | ||||
|                 return [] | ||||
| 
 | ||||
|     raise_for_httperror(resp) | ||||
|  | @ -81,11 +82,15 @@ def response(resp): | |||
| 
 | ||||
|     results.append({'url': wikipedia_link, 'title': title}) | ||||
| 
 | ||||
|     results.append({'infobox': title, | ||||
|     results.append( | ||||
|         { | ||||
|             'infobox': title, | ||||
|             'id': wikipedia_link, | ||||
|             'content': api_result.get('extract', ''), | ||||
|             'img_src': api_result.get('thumbnail', {}).get('source'), | ||||
|                     'urls': [{'title': 'Wikipedia', 'url': wikipedia_link}]}) | ||||
|             'urls': [{'title': 'Wikipedia', 'url': wikipedia_link}], | ||||
|         } | ||||
|     ) | ||||
| 
 | ||||
|     return results | ||||
| 
 | ||||
|  |  | |||
|  | @ -36,8 +36,7 @@ img_alt_xpath = './@alt' | |||
| 
 | ||||
| # pods to display as image in infobox | ||||
| # this pods do return a plaintext, but they look better and are more useful as images | ||||
| image_pods = {'VisualRepresentation', | ||||
|               'Illustration'} | ||||
| image_pods = {'VisualRepresentation', 'Illustration'} | ||||
| 
 | ||||
| 
 | ||||
| # do search-request | ||||
|  | @ -50,7 +49,8 @@ def request(query, params): | |||
| 
 | ||||
| # replace private user area characters to make text legible | ||||
| def replace_pua_chars(text): | ||||
|     pua_chars = {'\uf522': '\u2192',  # rigth arrow | ||||
|     pua_chars = { | ||||
|         '\uf522': '\u2192',  # rigth arrow | ||||
|         '\uf7b1': '\u2115',  # set of natural numbers | ||||
|         '\uf7b4': '\u211a',  # set of rational numbers | ||||
|         '\uf7b5': '\u211d',  # set of real numbers | ||||
|  | @ -58,7 +58,8 @@ def replace_pua_chars(text): | |||
|         '\uf74c': 'd',  # differential | ||||
|         '\uf74d': '\u212f',  # euler's number | ||||
|         '\uf74e': 'i',  # imaginary number | ||||
|                  '\uf7d9': '='}       # equals sign | ||||
|         '\uf7d9': '=', | ||||
|     }  # equals sign | ||||
| 
 | ||||
|     for k, v in pua_chars.items(): | ||||
|         text = text.replace(k, v) | ||||
|  | @ -112,9 +113,12 @@ def response(resp): | |||
|                 result_chunks.append({'label': pod_title, 'value': content}) | ||||
| 
 | ||||
|             elif image: | ||||
|                 result_chunks.append({'label': pod_title, | ||||
|                                       'image': {'src': image[0].xpath(img_src_xpath)[0], | ||||
|                                                 'alt': image[0].xpath(img_alt_xpath)[0]}}) | ||||
|                 result_chunks.append( | ||||
|                     { | ||||
|                         'label': pod_title, | ||||
|                         'image': {'src': image[0].xpath(img_src_xpath)[0], 'alt': image[0].xpath(img_alt_xpath)[0]}, | ||||
|                     } | ||||
|                 ) | ||||
| 
 | ||||
|     if not result_chunks: | ||||
|         return [] | ||||
|  | @ -122,13 +126,15 @@ def response(resp): | |||
|     title = "Wolfram|Alpha (%s)" % infobox_title | ||||
| 
 | ||||
|     # append infobox | ||||
|     results.append({'infobox': infobox_title, | ||||
|     results.append( | ||||
|         { | ||||
|             'infobox': infobox_title, | ||||
|             'attributes': result_chunks, | ||||
|                     'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) | ||||
|             'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}], | ||||
|         } | ||||
|     ) | ||||
| 
 | ||||
|     # append link to site | ||||
|     results.append({'url': resp.request.headers['Referer'], | ||||
|                     'title': title, | ||||
|                     'content': result_content}) | ||||
|     results.append({'url': resp.request.headers['Referer'], 'title': title, 'content': result_content}) | ||||
| 
 | ||||
|     return results | ||||
|  |  | |||
Some files were not shown because too many files have changed in this diff Show more
		Loading…
	
	Add table
		
		Reference in a new issue
	
	 Markus Heiser
						Markus Heiser