From b7886bb15c35691e2db9089c4111c2fa2b685bed Mon Sep 17 00:00:00 2001 From: Alice Ferrazzi Date: Wed, 1 Aug 2018 17:48:38 +0900 Subject: [PATCH 1/4] gentoo engine test --- searx/engines/gentoo.py | 122 ++++++++++++++++++++++++++++++++++++++++ searx/settings.yml | 4 ++ 2 files changed, 126 insertions(+) create mode 100644 searx/engines/gentoo.py diff --git a/searx/engines/gentoo.py b/searx/engines/gentoo.py new file mode 100644 index 000000000..4501b87ee --- /dev/null +++ b/searx/engines/gentoo.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +""" + Gentoo Wiki + + @website https://wiki.gentoo.org + @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it + @using-api no + @results HTML + @stable no (HTML can change) + @parse url, title +""" + +from lxml import html +from searx.engines.xpath import extract_text +from searx.url_utils import urlencode, urljoin + +# engine dependent config +categories = ['it'] +language_support = True +paging = True +base_url = 'https://wiki.gentoo.org' + +# xpath queries +xpath_results = '//ul[@class="mw-search-results"]/li' +xpath_link = './/div[@class="mw-search-result-heading"]/a' + + +# cut 'en' from 'en-US', 'de' from 'de-CH', and so on +def locale_to_lang_code(locale): + if locale.find('-') >= 0: + locale = locale.split('-')[0] + return locale + + +# wikis for some languages were moved off from the main site, we need to make +# requests to correct URLs to be able to get results in those languages +lang_urls = { + 'en': { + 'base': 'https://wiki.gentoo.org', + 'search': '/index.php?title=Special:Search&offset={offset}&{query}' + } +} + + +# get base & search URLs for selected language +def get_lang_urls(language): + if language in lang_urls: + return lang_urls[language] + return lang_urls['en'] + + +# Language names to build search requests for +# those languages which are hosted on the main site. +main_langs = { + 'ar': 'العربية', + 'bg': 'Български', + 'cs': 'Česky', + 'da': 'Dansk', + 'el': 'Ελληνικά', + 'es': 'Español', + 'he': 'עברית', + 'hr': 'Hrvatski', + 'hu': 'Magyar', + 'it': 'Italiano', + 'ko': '한국어', + 'lt': 'Lietuviškai', + 'nl': 'Nederlands', + 'pl': 'Polski', + 'pt': 'Português', + 'ru': 'Русский', + 'sl': 'Slovenský', + 'th': 'ไทย', + 'uk': 'Українська', + 'zh': '简体中文' +} +supported_languages = dict(lang_urls, **main_langs) + + +# do search-request +def request(query, params): + # translate the locale (e.g. 'en-US') to language code ('en') + language = locale_to_lang_code(params['language']) + + # if our language is hosted on the main site, we need to add its name + # to the query in order to narrow the results to that language + if language in main_langs: + query += b' (' + main_langs[language] + b')' + + # prepare the request parameters + query = urlencode({'search': query}) + offset = (params['pageno'] - 1) * 20 + + # get request URLs for our language of choice + urls = get_lang_urls(language) + search_url = urls['base'] + urls['search'] + + params['url'] = search_url.format(query=query, offset=offset) + + return params + + +# get response from search-request +def response(resp): + # get the base URL for the language in which request was made + language = locale_to_lang_code(resp.search_params['language']) + base_url = get_lang_urls(language)['base'] + + results = [] + + dom = html.fromstring(resp.text) + + # parse results + for result in dom.xpath(xpath_results): + link = result.xpath(xpath_link)[0] + href = urljoin(base_url, link.attrib.get('href')) + title = extract_text(link) + + results.append({'url': href, + 'title': title}) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index 0bd16ca2a..bd5457341 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -273,6 +273,10 @@ engines: timeout : 3.0 disabled: True + - name : gentoo + engine : gentoo + shortcut : ge + - name : gitlab engine : json_engine paging : True From 5f0888b8f4f7a46a7418e6faf34d47d45adca1c7 Mon Sep 17 00:00:00 2001 From: Alice Ferrazzi Date: Sun, 12 Aug 2018 22:22:25 +0900 Subject: [PATCH 2/4] added languages to Gentoo engine --- searx/engines/gentoo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/engines/gentoo.py b/searx/engines/gentoo.py index 4501b87ee..3611068d3 100644 --- a/searx/engines/gentoo.py +++ b/searx/engines/gentoo.py @@ -38,7 +38,7 @@ def locale_to_lang_code(locale): lang_urls = { 'en': { 'base': 'https://wiki.gentoo.org', - 'search': '/index.php?title=Special:Search&offset={offset}&{query}' + 'search': '/index.php?title=Special:Search&offset={offset}&{query}&profile=translation&languagefilter={language}' } } @@ -85,7 +85,7 @@ def request(query, params): # if our language is hosted on the main site, we need to add its name # to the query in order to narrow the results to that language if language in main_langs: - query += b' (' + main_langs[language] + b')' + query += b' (' + (main_langs[language]).encode('utf-8') + b')' # prepare the request parameters query = urlencode({'search': query}) @@ -95,7 +95,7 @@ def request(query, params): urls = get_lang_urls(language) search_url = urls['base'] + urls['search'] - params['url'] = search_url.format(query=query, offset=offset) + params['url'] = search_url.format(query=query, offset=offset, language=language) return params From ff2eac9f6b382e8ad30eed2b733740ce4a50f6e6 Mon Sep 17 00:00:00 2001 From: Alice Ferrazzi Date: Sun, 19 Aug 2018 10:22:04 +0900 Subject: [PATCH 3/4] Fix choosing languges function Searching for english is now giving all pages results. Instead searching for a specific language different from english is correctly returning only translated pages for the selected language. --- searx/engines/gentoo.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/searx/engines/gentoo.py b/searx/engines/gentoo.py index 3611068d3..39568a148 100644 --- a/searx/engines/gentoo.py +++ b/searx/engines/gentoo.py @@ -4,7 +4,7 @@ Gentoo Wiki @website https://wiki.gentoo.org - @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it + @provide-api yes @using-api no @results HTML @stable no (HTML can change) @@ -35,18 +35,23 @@ def locale_to_lang_code(locale): # wikis for some languages were moved off from the main site, we need to make # requests to correct URLs to be able to get results in those languages -lang_urls = { +lang_urls= { 'en': { 'base': 'https://wiki.gentoo.org', - 'search': '/index.php?title=Special:Search&offset={offset}&{query}&profile=translation&languagefilter={language}' + 'search': '/index.php?title=Special:Search&offset={offset}&{query}' + }, + 'others': { + 'base': 'https://wiki.gentoo.org', + 'search': '/index.php?title=Special:Search&offset={offset}&{query}\ + &profile=translation&languagefilter={language}' } } # get base & search URLs for selected language def get_lang_urls(language): - if language in lang_urls: - return lang_urls[language] + if language != 'en': + return lang_urls['others'] return lang_urls['en'] From d24c58cf4eafaf574cbc5b19243da922de2416f6 Mon Sep 17 00:00:00 2001 From: Alice Ferrazzi Date: Sun, 19 Aug 2018 10:31:09 +0900 Subject: [PATCH 4/4] Fix pep8 errors --- searx/engines/gentoo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/searx/engines/gentoo.py b/searx/engines/gentoo.py index 39568a148..a7a966cc9 100644 --- a/searx/engines/gentoo.py +++ b/searx/engines/gentoo.py @@ -35,7 +35,7 @@ def locale_to_lang_code(locale): # wikis for some languages were moved off from the main site, we need to make # requests to correct URLs to be able to get results in those languages -lang_urls= { +lang_urls = { 'en': { 'base': 'https://wiki.gentoo.org', 'search': '/index.php?title=Special:Search&offset={offset}&{query}' @@ -100,7 +100,8 @@ def request(query, params): urls = get_lang_urls(language) search_url = urls['base'] + urls['search'] - params['url'] = search_url.format(query=query, offset=offset, language=language) + params['url'] = search_url.format(query=query, offset=offset, + language=language) return params