Merge pull request #525 from ukwt/master

Add Arch Linux Wiki search engine

AUTHORS.rst

@@ -41,3 +41,4 @@ generally made searx better:
 - @GreenLunar
 - Noemi Vanyi
 - Kang-min Liu
+- Kirill Isakov

searx/engines/archlinux.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+"""
+ Arch Linux Wiki
+
+ @website      https://wiki.archlinux.org
+ @provide-api  no (Mediawiki provides API, but Arch Wiki blocks access to it
+ @using-api    no
+ @results      HTML
+ @stable       no (HTML can change)
+ @parse        url, title
+"""
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import urlencode
+from lxml import html
+from searx.engines.xpath import extract_text
+
+# engine dependent config
+categories = ['it']
+language_support = True
+paging = True
+base_url = 'https://wiki.archlinux.org'
+
+# xpath queries
+xpath_results = '//ul[@class="mw-search-results"]/li'
+xpath_link = './/div[@class="mw-search-result-heading"]/a'
+
+
+# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
+def locale_to_lang_code(locale):
+    if locale.find('_') >= 0:
+        locale = locale.split('_')[0]
+    return locale
+
+# wikis for some languages were moved off from the main site, we need to make
+# requests to correct URLs to be able to get results in those languages
+lang_urls = {
+    'all': {
+        'base': 'https://wiki.archlinux.org',
+        'search': '/index.php?title=Special:Search&offset={offset}&{query}'
+    },
+    'de': {
+        'base': 'https://wiki.archlinux.de',
+        'search': '/index.php?title=Spezial:Suche&offset={offset}&{query}'
+    },
+    'fr': {
+        'base': 'https://wiki.archlinux.fr',
+        'search': '/index.php?title=Spécial:Recherche&offset={offset}&{query}'
+    },
+    'ja': {
+        'base': 'https://wiki.archlinuxjp.org',
+        'search': '/index.php?title=特別:検索&offset={offset}&{query}'
+    },
+    'ro': {
+        'base': 'http://wiki.archlinux.ro',
+        'search': '/index.php?title=Special:Căutare&offset={offset}&{query}'
+    },
+    'tr': {
+        'base': 'http://archtr.org/wiki',
+        'search': '/index.php?title=Özel:Ara&offset={offset}&{query}'
+    }
+}
+
+
+# get base & search URLs for selected language
+def get_lang_urls(language):
+    if language in lang_urls:
+        return lang_urls[language]
+    return lang_urls['all']
+
+# Language names to build search requests for
+# those languages which are hosted on the main site.
+main_langs = {
+    'ar': 'العربية',
+    'bg': 'Български',
+    'cs': 'Česky',
+    'da': 'Dansk',
+    'el': 'Ελληνικά',
+    'es': 'Español',
+    'he': 'עברית',
+    'hr': 'Hrvatski',
+    'hu': 'Magyar',
+    'it': 'Italiano',
+    'ko': '한국어',
+    'lt': 'Lietuviškai',
+    'nl': 'Nederlands',
+    'pl': 'Polski',
+    'pt': 'Português',
+    'ru': 'Русский',
+    'sl': 'Slovenský',
+    'th': 'ไทย',
+    'uk': 'Українська',
+    'zh': '简体中文'
+}
+
+
+# do search-request
+def request(query, params):
+    # translate the locale (e.g. 'en_US') to language code ('en')
+    language = locale_to_lang_code(params['language'])
+
+    # if our language is hosted on the main site, we need to add its name
+    # to the query in order to narrow the results to that language
+    if language in main_langs:
+        query += '(' + main_langs[language] + ')'
+
+    # prepare the request parameters
+    query = urlencode({'search': query})
+    offset = (params['pageno'] - 1) * 20
+
+    # get request URLs for our language of choice
+    urls = get_lang_urls(language)
+    search_url = urls['base'] + urls['search']
+
+    params['url'] = search_url.format(query=query, offset=offset)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    # get the base URL for the language in which request was made
+    language = locale_to_lang_code(resp.search_params['language'])
+    base_url = get_lang_urls(language)['base']
+
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    # parse results
+    for result in dom.xpath(xpath_results):
+        link = result.xpath(xpath_link)[0]
+        href = urljoin(base_url, link.attrib.get('href'))
+        title = escape(extract_text(link))
+
+        results.append({'url': href,
+                        'title': title})
+
+    return results

searx/settings.yml

@@ -34,6 +34,10 @@ outgoing: # communication with search engines
 #        - 1.1.1.2
 
 engines:
+  - name : arch linux wiki
+    engine : archlinux
+    shortcut : al
+
   - name : wikipedia
     engine : mediawiki
     shortcut : wp

tests/unit/engines/test_archlinux.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+import mock
+from searx.engines import archlinux
+from searx.testing import SearxTestCase
+
+domains = {
+    'all': 'https://wiki.archlinux.org',
+    'de': 'https://wiki.archlinux.de',
+    'fr': 'https://wiki.archlinux.fr',
+    'ja': 'https://wiki.archlinuxjp.org',
+    'ro': 'http://wiki.archlinux.ro',
+    'tr': 'http://archtr.org/wiki'
+}
+
+
+class TestArchLinuxEngine(SearxTestCase):
+
+    def test_request(self):
+        query = 'test_query'
+        dic = defaultdict(dict)
+        dic['pageno'] = 1
+        dic['language'] = 'en_US'
+        params = archlinux.request(query, dic)
+        self.assertTrue('url' in params)
+        self.assertTrue(query in params['url'])
+        self.assertTrue('wiki.archlinux.org' in params['url'])
+
+        for lang, domain in domains.iteritems():
+            dic['language'] = lang
+            params = archlinux.request(query, dic)
+            self.assertTrue(domain in params['url'])
+
+    def test_response(self):
+        response = mock.Mock(text='<html></html>',
+                             search_params={'language': 'en_US'})
+        self.assertEqual(archlinux.response(response), [])
+
+        html = """
+        <ul class="mw-search-results">
+          <li>
+          <div class="mw-search-result-heading">
+            <a href="/index.php/ATI" title="ATI">ATI</a>
+          </div>
+          <div class="searchresult">
+            Lorem ipsum dolor sit amet
+          </div>
+          <div class="mw-search-result-data">
+            30 KB (4,630 words) - 19:04, 17 March 2016</div>
+          </li>
+          <li>
+          <div class="mw-search-result-heading">
+            <a href="/index.php/Frequently_asked_questions" title="Frequently asked questions">
+              Frequently asked questions
+            </a>
+          </div>
+          <div class="searchresult">
+            CPUs with AMDs instruction set "AMD64"
+          </div>
+          <div class="mw-search-result-data">
+            17 KB (2,722 words) - 20:13, 21 March 2016
+          </div>
+          </li>
+          <li>
+          <div class="mw-search-result-heading">
+            <a href="/index.php/CPU_frequency_scaling" title="CPU frequency scaling">CPU frequency scaling</a>
+          </div>
+          <div class="searchresult">
+            ondemand for AMD and older Intel CPU
+          </div>
+          <div class="mw-search-result-data">
+            15 KB (2,319 words) - 23:46, 16 March 2016
+          </div>
+          </li>
+        </ul>
+        """
+
+        expected = [
+            {
+                'title': 'ATI',
+                'url': 'https://wiki.archlinux.org/index.php/ATI'
+            },
+            {
+                'title': 'Frequently asked questions',
+                'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions'
+            },
+            {
+                'title': 'CPU frequency scaling',
+                'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling'
+            }
+        ]
+
+        response = mock.Mock(text=html)
+        response.search_params = {
+            'language': 'en_US'
+        }
+        results = archlinux.response(response)
+
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), len(expected))
+
+        i = 0
+        for exp in expected:
+            res = results[i]
+            i += 1
+            for key, value in exp.iteritems():
+                self.assertEqual(res[key], value)