Merge https://github.com/asciimoo/searx into template_oscar

Conflicts:
	searx/translations/de/LC_MESSAGES/messages.po
	searx/translations/en/LC_MESSAGES/messages.po
	searx/translations/es/LC_MESSAGES/messages.po
	searx/translations/fr/LC_MESSAGES/messages.po
	searx/translations/hu/LC_MESSAGES/messages.po
	searx/translations/it/LC_MESSAGES/messages.po
	searx/translations/nl/LC_MESSAGES/messages.po
	searx/webapp.py

AUTHORS.rst

@@ -26,3 +26,4 @@ generally made searx better:
 - dp
 - Martin Zimmermann
 - @courgette
+- @kernc

searx/__init__.py

@@ -17,6 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 from os import environ
 from os.path import realpath, dirname, join, abspath
+from searx.https_rewrite import load_https_rules
 try:
     from yaml import load
 except:
@@ -27,14 +28,24 @@ except:
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 
-# if possible set path to settings using the enviroment variable SEARX_SETTINGS_PATH
+# if possible set path to settings using the
+# enviroment variable SEARX_SETTINGS_PATH
 if 'SEARX_SETTINGS_PATH' in environ:
     settings_path = environ['SEARX_SETTINGS_PATH']
 # otherwise using default path
 else:
     settings_path = join(searx_dir, 'settings.yml')
 
+if 'SEARX_HTTPS_REWRITE_PATH' in environ:
+    https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH']
+else:
+    https_rewrite_path = join(searx_dir, 'https_rules')
 
 # load settings
 with open(settings_path) as settings_yaml:
     settings = load(settings_yaml)
+
+# load https rules only if https rewrite is enabled
+if settings.get('server', {}).get('https_rewrite'):
+    # loade https rules
+    load_https_rules(https_rewrite_path)

searx/engines/__init__.py

@@ -41,7 +41,7 @@ def load_module(filename):
     module.name = modname
     return module
 
-if not 'engines' in settings or not settings['engines']:
+if 'engines' not in settings or not settings['engines']:
     print '[E] Error no engines found. Edit your settings.yml'
     exit(2)
 
@@ -68,15 +68,15 @@ for engine_data in settings['engines']:
         engine.categories = ['general']
 
     if not hasattr(engine, 'language_support'):
-        #engine.language_support = False
+        # engine.language_support = False
         engine.language_support = True
 
     if not hasattr(engine, 'timeout'):
-        #engine.language_support = False
+        # engine.language_support = False
         engine.timeout = settings['server']['request_timeout']
 
     if not hasattr(engine, 'shortcut'):
-        #engine.shortcut = '''
+        # engine.shortcut = '''
         engine.shortcut = ''
 
     # checking required variables
@@ -161,7 +161,8 @@ def get_engines_stats():
 
     for engine in scores_per_result:
         if max_score_per_result:
-            engine['percentage'] = int(engine['avg'] / max_score_per_result * 100)
+            engine['percentage'] = int(engine['avg']
+                                       / max_score_per_result * 100)
         else:
             engine['percentage'] = 0
 

searx/engines/duckduckgo_definitions.py

@@ -116,6 +116,13 @@ def response(resp):
 
     if len(heading)>0:
         # TODO get infobox.meta.value where .label='article_title'
+        if image==None and len(attributes)==0 and len(urls)==1 and len(relatedTopics)==0 and len(content)==0:
+            results.append({
+                    'url': urls[0]['url'],
+                    'title': heading,
+                    'content': content
+                    })
+        else:
             results.append({
                     'infobox': heading,
                     'id': infobox_id,

searx/engines/faroo.py

@@ -0,0 +1,108 @@
+## Faroo (Web, News)
+# 
+# @website     http://www.faroo.com
+# @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title, content, publishedDate, img_src
+
+from urllib import urlencode
+from json import loads
+import datetime
+from searx.utils import searx_useragent
+
+# engine dependent config
+categories = ['general', 'news']
+paging = True
+language_support = True
+number_of_results = 10
+api_key = None
+
+# search-url
+url = 'http://www.faroo.com/'
+search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'
+
+search_category = {'general': 'web',                
+                'news': 'news'}
+
+# do search-request
+def request(query, params):
+    offset = (params['pageno']-1) * number_of_results + 1
+    categorie = search_category.get(params['category'], 'web')
+
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('_')[0]
+
+    # skip, if language is not supported
+    if language != 'en' and\
+       language != 'de' and\
+       language != 'zh':
+        return params
+
+    params['url'] = search_url.format(offset=offset,
+                                      number_of_results=number_of_results,
+                                      query=urlencode({'q': query}),
+                                      language=language,
+                                      categorie=categorie,
+                                      api_key=api_key )
+
+    # using searx User-Agent
+    params['headers']['User-Agent'] = searx_useragent()
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    # HTTP-Code 401: api-key is not valide
+    if resp.status_code == 401:
+        raise Exception("API key is not valide")
+        return []
+
+    # HTTP-Code 429: rate limit exceeded
+    if resp.status_code == 429:
+        raise Exception("rate limit has been exceeded!")
+        return []
+
+    results = []
+
+    search_res = loads(resp.text)
+
+    # return empty array if there are no results
+    if not search_res.get('results', {}):
+        return []
+
+    # parse results
+    for result in search_res['results']:
+        if result['news']:
+            # timestamp (how many milliseconds have passed between now and the beginning of 1970)
+            publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0)
+
+            # append news result
+            results.append({'url': result['url'],
+                            'title': result['title'],
+                            'publishedDate': publishedDate,
+                            'content': result['kwic']})
+
+        else:
+            # append general result
+            # TODO, publishedDate correct?
+            results.append({'url': result['url'],
+                            'title': result['title'],
+                            'content': result['kwic']})
+
+        # append image result if image url is set
+        # TODO, show results with an image like in faroo
+        if result['iurl']:
+            results.append({'template': 'images.html',
+                            'url': result['url'],
+                            'title': result['title'],
+                            'content': result['kwic'],  
+                            'img_src': result['iurl']})
+
+    # return results
+    return results

searx/engines/wikidata.py

@@ -2,7 +2,7 @@ import json
 from requests import get
 from urllib import urlencode
 
-resultCount=2
+resultCount=1
 urlSearch = 'https://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
 urlDetail = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=labels%7Cinfo%7Csitelinks%7Csitelinks%2Furls%7Cdescriptions%7Cclaims&{query}'
 urlMap = 'https://www.openstreetmap.org/?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
@@ -33,17 +33,20 @@ def response(resp):
     return results
 
 def getDetail(jsonresponse, wikidata_id, language):
-    result = jsonresponse.get('entities', {}).get(wikidata_id, {})
-
-    title = result.get('labels', {}).get(language, {}).get('value', None)
-    if title == None:
-        title = result.get('labels', {}).get('en', {}).get('value', wikidata_id)
     results = []
     urls = []
     attributes = []
 
-    description = result.get('descriptions', {}).get(language, {}).get('value', '')
+    result = jsonresponse.get('entities', {}).get(wikidata_id, {})
-    if description == '':
+
+    title = result.get('labels', {}).get(language, {}).get('value', None)
+    if title == None:
+        title = result.get('labels', {}).get('en', {}).get('value', None)
+    if title == None:
+        return results
+
+    description = result.get('descriptions', {}).get(language, {}).get('value', None)
+    if description == None:
         description = result.get('descriptions', {}).get('en', {}).get('value', '')
 
     claims = result.get('claims', {})
@@ -52,10 +55,15 @@ def getDetail(jsonresponse, wikidata_id, language):
         urls.append({ 'title' : 'Official site', 'url': official_website })
         results.append({ 'title': title, 'url' : official_website })
 
+    wikipedia_link_count = 0
     if language != 'en':
-        add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
+        wikipedia_link_count += add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
     wikipedia_en_link = get_wikilink(result, 'enwiki')
-    add_url(urls, 'Wikipedia (en)', wikipedia_en_link)
+    wikipedia_link_count += add_url(urls, 'Wikipedia (en)', wikipedia_en_link)
+    if wikipedia_link_count == 0:
+        misc_language = get_wiki_firstlanguage(result, 'wiki')
+        if misc_language != None:
+            add_url(urls, 'Wikipedia (' + misc_language + ')', get_wikilink(result, misc_language + 'wiki'))
 
     if language != 'en':
         add_url(urls, 'Wiki voyage (' + language + ')', get_wikilink(result, language + 'wikivoyage'))
@@ -105,7 +113,13 @@ def getDetail(jsonresponse, wikidata_id, language):
     if date_of_death != None:
         attributes.append({'label' : 'Date of death', 'value' : date_of_death})
 
-
+    if len(attributes)==0 and len(urls)==2 and len(description)==0:
+        results.append({
+                'url': urls[0]['url'],
+                'title': title,
+                'content': description
+                })
+    else:
         results.append({
                 'infobox' : title,
                 'id' : wikipedia_en_link,
@@ -120,7 +134,9 @@ def getDetail(jsonresponse, wikidata_id, language):
 def add_url(urls, title, url):
     if url != None:
         urls.append({'title' : title, 'url' : url})
-
+        return 1
+    else:
+        return 0
 
 def get_mainsnak(claims, propertyName):
     propValue = claims.get(propertyName, {})
@@ -147,7 +163,8 @@ def get_string(claims, propertyName, defaultValue=None):
     if len(result) == 0:
         return defaultValue
     else:
-        return ', '.join(result)
+        #TODO handle multiple urls
+        return result[0]
 
 
 def get_time(claims, propertyName, defaultValue=None):
@@ -213,3 +230,9 @@ def get_wikilink(result, wikiid):
     elif url.startswith('//'):
         url = 'https:' + url
     return url
+
+def get_wiki_firstlanguage(result, wikipatternid):
+    for k in result.get('sitelinks', {}).keys():
+        if k.endswith(wikipatternid) and len(k)==(2+len(wikipatternid)):
+            return k[0:2]
+    return None

searx/engines/yahoo_news.py

@@ -1,7 +1,8 @@
-## Yahoo (News)
+# Yahoo (News)
 #
 # @website     https://news.yahoo.com
-# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries
+# @provide-api yes (https://developer.yahoo.com/boss/search/)
+#              $0.80/1000 queries
 #
 # @using-api   no (because pricing)
 # @results     HTML (using search portal)
@@ -22,7 +23,7 @@ paging = True
 language_support = True
 
 # search-url
-search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}'
+search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}'  # noqa
 
 # specific xpath variables
 results_xpath = '//div[@class="res"]'

searx/engines/youtube.py

@@ -13,7 +13,7 @@ from urllib import urlencode
 from dateutil import parser
 
 # engine dependent config
-categories = ['videos']
+categories = ['videos', 'music']
 paging = True
 language_support = True
 

searx/https_rewrite.py

@@ -1,14 +1,145 @@
+'''
+searx is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+searx is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with searx. If not, see < http://www.gnu.org/licenses/ >.
+
+(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
+'''
+
 import re
+from lxml import etree
+from os import listdir
+from os.path import isfile, isdir, join
+
 
 # https://gitweb.torproject.org/\
 # pde/https-everywhere.git/tree/4.0:/src/chrome/content/rules
 
 # HTTPS rewrite rules
-https_rules = (
+https_rules = []
-    # from
+
-    (re.compile(r'^http://(www\.|m\.|)?xkcd\.(?:com|org)/', re.I | re.U),
+
-     # to
+# load single ruleset from a xml file
-     r'https://\1xkcd.com/'),
+def load_single_https_ruleset(filepath):
-    (re.compile(r'^https?://(?:ssl)?imgs\.xkcd\.com/', re.I | re.U),
+    ruleset = ()
-     r'https://sslimgs.xkcd.com/'),
+
-)
+    # init parser
+    parser = etree.XMLParser()
+
+    # load and parse xml-file
+    try:
+        tree = etree.parse(filepath, parser)
+    except:
+        # TODO, error message
+        return ()
+
+    # get root node
+    root = tree.getroot()
+
+    # check if root is a node with the name ruleset
+    # TODO improve parsing
+    if root.tag != 'ruleset':
+        return ()
+
+    # check if rule is deactivated by default
+    if root.attrib.get('default_off'):
+        return ()
+
+    # check if rule does only work for specific platforms
+    if root.attrib.get('platform'):
+        return ()
+
+    hosts = []
+    rules = []
+    exclusions = []
+
+    # parse childs from ruleset
+    for ruleset in root:
+        # this child define a target
+        if ruleset.tag == 'target':
+            # check if required tags available
+            if not ruleset.attrib.get('host'):
+                continue
+
+            # convert host-rule to valid regex
+            host = ruleset.attrib.get('host')\
+                .replace('.', '\.').replace('*', '.*')
+
+            # append to host list
+            hosts.append(host)
+
+        # this child define a rule
+        elif ruleset.tag == 'rule':
+            # check if required tags available
+            if not ruleset.attrib.get('from')\
+               or not ruleset.attrib.get('to'):
+                continue
+
+            # TODO hack, which convert a javascript regex group
+            # into a valid python regex group
+            rule_from = ruleset.attrib.get('from').replace('$', '\\')
+            rule_to = ruleset.attrib.get('to').replace('$', '\\')
+
+            # TODO, not working yet because of the hack above,
+            # currently doing that in webapp.py
+            # rule_from_rgx = re.compile(rule_from, re.I)
+
+            # append rule
+            rules.append((rule_from, rule_to))
+
+        # this child define an exclusion
+        elif ruleset.tag == 'exclusion':
+            # check if required tags available
+            if not ruleset.attrib.get('pattern'):
+                continue
+
+            exclusion_rgx = re.compile(ruleset.attrib.get('pattern'))
+
+            # append exclusion
+            exclusions.append(exclusion_rgx)
+
+    # convert list of possible hosts to a simple regex
+    # TODO compress regex to improve performance
+    try:
+        target_hosts = re.compile('^(' + '|'.join(hosts) + ')', re.I | re.U)
+    except:
+        return ()
+
+    # return ruleset
+    return (target_hosts, rules, exclusions)
+
+
+# load all https rewrite rules
+def load_https_rules(rules_path):
+    # check if directory exists
+    if not isdir(rules_path):
+        print("[E] directory not found: '" + rules_path + "'")
+        return
+
+    # search all xml files which are stored in the https rule directory
+    xml_files = [join(rules_path, f)
+                 for f in listdir(rules_path)
+                 if isfile(join(rules_path, f)) and f[-4:] == '.xml']
+
+    # load xml-files
+    for ruleset_file in xml_files:
+        # calculate rewrite-rules
+        ruleset = load_single_https_ruleset(ruleset_file)
+
+        # skip if no ruleset returned
+        if not ruleset:
+            continue
+
+        # append ruleset
+        https_rules.append(ruleset)
+
+    print(' * {n} https-rules loaded'.format(n=len(https_rules)))

searx/https_rules/00README

@@ -0,0 +1,17 @@
+<!-- 
+This directory contains web site rewriting rules for the
+HTTPS Everywhere software, available from
+https://www.eff.org/https-everywhere
+
+These rules were contributed to the project by users and aim to
+enable routine secure access to as many different web sites as
+possible.  They are automatically installed together with the
+HTTPS Everywhere software.  The presence of these rules does not
+mean that an HTTPS Everywhere user accessed, or intended to
+access, any particular web site.
+
+For information about how to create additional HTTPS Everywhere
+rewriting rules to add support for new sites, please see
+
+https://www.eff.org/https-everywhere/rulesets
+-->

searx/https_rules/Bing.xml

@@ -0,0 +1,56 @@
+<!--
+	For other Microsoft coverage, see Microsoft.xml.
+
+
+	CDN buckets:
+
+		- a134.lm.akamai.net
+
+			- akam.bing.com
+			- *.mm.bing.net
+
+
+	Nonfunctional domains:
+
+		- m2.cn.bing.com
+		- origin.bj1.bing.com
+		- blogs.bing.com
+
+
+	Fully covered domains:
+
+		- bing.com subdomains:
+
+			- (www.)
+			- c.bing		(tracking beacons)
+			- cn.bing
+			- h.bing
+			- ssl
+			- testfamilysafety.bing
+			- udc.bing
+			- (www.)bing
+
+		- *.mm.bing.net
+		- api.bing.com
+
+-->
+<ruleset name="Bing">
+
+	<target host="bing.com" />
+	<target host="*.bing.com" />
+	<target host="*.mm.bing.net" />
+
+
+	<securecookie host=".*\.bing\.com$" name=".+" />
+
+
+	<rule from="^http://((?:c|cn|h|ssl|testfamilysafety|udc|www)\.)?bing\.com/"
+		to="https://$1bing.com/" />
+
+	<rule from="^http://([^/:@]*)\.mm\.bing\.net/"
+		to="https://$1.mm.bing.com/"/>
+
+	<rule from="^http://([^/:@]*)\.api\.bing\.net/"
+		to="https://$1.api.bing.com/"/>
+
+</ruleset>

searx/https_rules/Dailymotion.xml

@@ -0,0 +1,69 @@
+<!--
+	Nonfunctional domains:
+
+		- blog.dailymotion.com
+		- press.dailymotion.com		(shows steaw.com, CN: www.steaw.com)
+		- proxy-46.dailymotion.com
+		- publicite.dailymotion.com
+		- publisher.dailymotion.com	(reset)
+		- vid.ak.dmcdn.net		(403, Akamai)
+		- vid2.ak.dmcdn.net		(504, akamai)
+
+
+	Problematic domains:
+
+		- ak2.static.dailymotion.com	(mismatched, CN: *.dmcdn.net)
+		- support.dmcloud.net		(mismatched, CN: *.zendesk.com)
+
+
+	Partially covered domains:
+
+		- (www.)dailymotion.com
+
+			- cdn/manifest/video/\w+.mnft 403s
+			- crossdomain.xml breaks videos
+
+-->
+<ruleset name="Dailymotion (default off)" default_off="breaks some embedded videos">
+
+	<target host="dailymotion.com" />
+	<!--
+		* for cross-domain cookie.
+					-->
+	<target host="*.dailymotion.com" />
+		<!--
+			https://mail1.eff.org/pipermail/https-everywhere-rules/2012-July/001241.html
+													-->
+		<exclusion pattern="^http://(?:www\.)?dailymotion\.com/(?:cdn/[\w-]+/video/|crossdomain\.xml$)" />
+	<target host="ak2.static.dailymotion.com" />
+	<target host="*.dmcdn.net" />
+	<target host="dmcloud.net" />
+	<target host="*.dmcloud.net" />
+
+
+	<!--	Testing wrt embedded breakage.
+
+		securecookie host="^.*\.dailymotion\.com$" name=".+" /-->
+	<!--
+		Omniture tracking cookies:
+						-->
+	<securecookie host="^\.dailymotion\.com$" name="^s_\w+$" />
+	<securecookie host="^www\.dailymotion\.com$" name=".+" />
+
+
+	<rule from="^http://(erroracct\.|www\.)?dailymotion\.com/"
+		to="https://$1dailymotion.com/" />
+
+	<rule from="^http://(s\d|static(?:\d|s\d-ssl))\.dmcdn\.net/"
+		to="https://$1.dmcdn.net/" />
+
+	<rule from="^https?://ak2\.static\.dailymotion\.com/"
+		to="https://static1-ssl.dmcdn.net/" />
+
+	<rule from="^http://(s\.|www\.)?dmcloud\.net/"
+		to="https://$1dmcloud.net/" />
+
+	<rule from="^https?://support\.dmcloud\.net/"
+		to="https://dmcloud.zendesk.com/" />
+
+</ruleset>

searx/https_rules/Deviantart.xml

@@ -0,0 +1,53 @@
+<!--
+	For problematic rules, see Deviantart-mismatches.xml.
+
+
+	Other deviantArt rulesets:
+
+		- Sta.sh.xml
+
+
+	ToDo: Find edgecast URL for /(fc|th)\d+.
+
+
+	Mixed content:
+
+		- Images on *.....com from e.deviantart.net *
+
+	* Secured by us
+
+-->
+<ruleset name="DeviantArt (pending)" default_off="site operator says not ready yet">
+
+	<target host="deviantart.com" />
+	<target host="*.deviantart.com" />
+	<target host="deviantart.net" />
+	<target host="*.deviantart.net" />
+
+
+	<!--	Not secured by server:
+					-->
+	<!--securecookie host="^\.deviantart\.com$" name="^userinfo$" /-->
+
+	<securecookie host="^\.deviantart\.com$" name=".*" />
+
+
+	<!--	Redirects from com to net, but does so successfully by itself.
+										-->
+	<rule from="^http://([aei]|fc\d\d|s[ht]|th\d\d)\.deviantart\.(com|net)/"
+		to="https://$1.deviantart.$2/" />
+
+	<!--	This handles everything that isn't in the first rule.
+		Namely, usernames, backend, fc, th, and (www.).
+			These domains present a cert that is only
+		valid for .com.
+			Note that .net isn't used on DA, but.net does
+		redirect to .com, and we shouldn't break what would
+		otherwise work.
+			Mustn't rewrite from https here, as doing so
+		would conflict with the first rule.
+								-->
+	<rule from="^http://([^/:@\.]+\.)?deviantart\.(?:com|net)/"
+		to="https://$1deviantart.com/" />
+
+</ruleset>

searx/https_rules/DuckDuckGo.xml

@@ -0,0 +1,38 @@
+<!--
+	Problematic domains:
+
+		- www.dukgo.com		(mismatched, CN: dukgo.com)
+
+
+	Fully covered domains:
+
+		- (www.)dukgo.com	(www → ^)
+
+-->
+<ruleset name="DuckDuckGo">
+  <target host="duckduckgo.com" />
+  <target host="*.duckduckgo.com" />
+  <target host="ddg.gg" />
+  <target host="duck.co" />
+  <target host="i.duck.co" />
+	<target host="dukgo.com" />
+	<target host="www.dukgo.com" />
+
+  <exclusion pattern="^http://(help|meme)\.duckduckgo\.com/" />
+
+	<securecookie host="^duck\.co$" name=".*"/>
+
+  <rule from="^http://duckduckgo\.com/" to="https://duckduckgo.com/"/>
+  <rule from="^http://([^/:@\.]+)\.duckduckgo\.com/" to="https://$1.duckduckgo.com/"/>
+	<!-- TODO: What does ddg.gg/foo do? Runs query foo, redirects to homepage, or error? -->
+    <rule from="^http://ddg\.gg/$" to="https://duckduckgo.com/" />
+	
+    <rule from="^http://duck\.co/" to="https://duck.co/" />
+
+	<rule from="^http://i\.duck\.co/"
+		to="https://duckduckgo.com/"/>
+
+	<rule from="^http://(?:www\.)?dukgo\.com/"
+		to="https://dukgo.com/" />
+
+</ruleset>

searx/https_rules/Flickr.xml

@@ -0,0 +1,44 @@
+<!--
+	For other Yahoo coverage, see Yahoo.xml.
+
+
+	These altnames don't exist:
+
+		- www.blog.flickr.net
+		- www.code.flickr.net
+
+-->
+<ruleset name="Flickr">
+
+	<target host="flic.kr" />
+	<target host="*.flic.kr" />
+	<target host="flickr.com" />
+	<target host="*.flickr.com" />
+	<target host="*.flickr.net" />
+	<target host="*.staticflickr.com" />
+
+
+	<!--	Not secured by server:
+					-->
+	<!--securecookie host="^\.flic\.kr$" name="^BX$" /-->
+
+	<securecookie host="^\.flic\.kr$" name=".+" />
+	<securecookie host=".*\.flickr\.com$" name=".+" />
+
+
+	<rule from="^http://flic\.kr/"
+		to="https://flic.kr/" />
+
+	<rule from="^http://(api\.|www\.)?flickr\.com/"
+		to="https://$1flickr.com/" />
+
+	<rule from="^http://s(ecure|tatic)\.flickr\.com/"
+		to="https://s$1.flickr.com/" />
+
+	<rule from="^http://(c2|farm\d+)\.static(\.)?flickr\.com/"
+		to="https://$1.static$2flickr.com/" />
+
+	<rule from="^http://(blog|code)\.flickr\.net/"
+		to="https://$1.flickr.net/" />
+
+</ruleset>

searx/https_rules/Github-Pages.xml

@@ -0,0 +1,11 @@
+<!--
+	For other GitHub coverage, see Github.xml.
+-->
+<ruleset name="GitHub Pages">
+
+	<target host="*.github.io" />
+
+	<rule from="^http://([^/@:\.]+)\.github\.io/"
+		to="https://$1.github.io/" />
+
+</ruleset>

searx/https_rules/Github.xml

@@ -0,0 +1,94 @@
+<!--
+	Other GitHub rulesets:
+
+		- Github-Pages.xml
+		- Guag.es.xml
+		- Speaker_Deck.com.xml
+
+
+	CDN buckets:
+
+		- github-images.s3.amazonaws.com
+		- github.global.ssl.fastly.net
+		- a248.e.akamai.net/assets.github.com/
+		- a248.e.akamai.net/camo.github.com/
+		- s3.amazonaws.com/github/ | d24z2fz21y4fag.cloudfront.net
+		- github.myshopify.com
+
+
+	Fully covered domains:
+
+		- github.com subdomains:
+
+			- (www.)
+			- assets\d+
+			- assets-cdn
+			- bounty
+			- cloud
+			- f.cloud
+			- codeload
+			- developer
+			- eclipse
+			- enterprise
+			- gist
+			- gist-assets
+			- help
+			- identicons
+			- jobs
+			- mac
+			- mobile
+			- nodeload
+			- octodex
+			- pages
+			- raw
+			- rg3
+			- shop
+			- status
+			- support
+			- training
+			- try
+			- wiki
+			- windows
+
+		- collector.githubapp.com
+
+		- githubusercontent.com
+
+-->
+<ruleset name="GitHub">
+
+	<target host="github.com" />
+	<target host="*.github.com" />
+	<target host="github.io" />
+	<target host="*.githubusercontent.com" />
+	<target host="collector.githubapp.com" />
+
+
+	<!--	Secured by server:
+					-->
+	<!--securecookie host="^github\.com$" name="^(_gh_sess|tz|user_session)$" /-->
+	<!--securecookie host="^\.github\.com$" name="^(dotcom_user|logged_in)$" /-->
+	<!--securecookie host="^enterprise\.github\.com$" name="^(_enterprise_web|request_method)$" /-->
+	<!--securecookie host="^gist\.github\.com$" name="^_gist_session$" /-->
+	<!--securecookie host="^help\.github\.com$" name="^_help_session$" /-->
+	<!--
+		Not secured by server:
+					-->
+	<!--securecookie host="^status\.github\.com$" name="^rack\.session$" /-->
+
+	<securecookie host="^(?:.*\.)?github\.com$" name=".+" />
+
+
+	<rule from="^http://((?:assets\d+|assets-cdn|bounty|cloud|f\.cloud|codeload|developer|eclipse|enterprise|gist|gist-assets|help|identicons|jobs|mac|mobile|nodeload|octodex|pages|raw|rg3|shop|status|support|training|try|wiki|windows|www)\.)?github\.com/"
+		to="https://$1github.com/" />
+
+	<rule from="^http://collector\.githubapp\.com/"
+		to="https://collector.githubapp.com/" />
+
+	<rule from="^https?://github\.io/"
+		to="https://pages.github.com/" />
+
+	<rule from="^http://([^/@:\.]+)\.githubusercontent\.com/"
+		to="https://$1.githubusercontent.com/" />
+
+</ruleset>

searx/https_rules/Google-mismatches.xml

@@ -0,0 +1,26 @@
+<!--
+
+	Problematic domains:
+
+		- (www.)apture.com	(works, mismatched, CN: *.google.com)
+
+-->
+<ruleset name="Google (mismatches)" default_off="mismatches">
+
+	<!--	Akamai	-->
+	<target host="js.admeld.com"/>
+	<target host="apture.com" />
+	<target host="www.apture.com" />
+	<target host="googleartproject.com"/>
+	<target host="www.googleartproject.com"/>
+
+	<rule from="^http://js\.admeld\.com/"
+		to="https://js.admeld.com/"/>
+
+	<rule from="^https?://(?:www\.)?apture\.com/"
+		to="https://apture.com/" />
+
+	<rule from="^http://(?:www\.)?googleartproject\.com/"
+		to="https://www.googleartproject.com/"/>
+
+</ruleset>

searx/https_rules/Google.org.xml

@@ -0,0 +1,14 @@
+<!--
+	For other Google coverage, see GoogleServices.xml.
+
+-->
+<ruleset name="Google.org">
+
+	<target host="google.org" />
+	<target host="www.google.org" />
+
+
+	<rule from="^http://(www\.)?google\.org/"
+		to="https://$1google.org/" />
+
+</ruleset>

searx/https_rules/GoogleAPIs.xml

@@ -0,0 +1,143 @@
+<!--
+	For other Google coverage, see GoogleServices.xml.
+
+
+	Nonfunctional domains:
+
+		- hosted.gmodules.com *
+		- img0.gmodules.com *
+		- p.gmodules.com *
+
+	* 404; mismatched, CN: *.googleusercontent.com
+
+
+	Problematic domains:
+
+		- gmodules.com			(503, CN: www.google.com)
+		- www.gmodules.com		(503, CN: *.googleusercontent.com)
+		- gstatic.com			(404, valid cert)
+		- api.recaptcha.net		(works; mismatched, CN: google.com)
+
+
+	Partially covered domains:
+
+		- (www.)gmodules.com		(→ www.google.com)
+		- (www.)google.com
+		- chart.apis.google.com		(→ chart.googleapis.com)
+
+
+	Fully covered domains:
+
+		- api.google.com
+
+		- *.clients.google.com:
+
+			- linkhelp
+
+		- ssl.google-analytics.com
+		- www.google-analytics.com
+
+		- googleapis.com subdomains:
+
+			- ajax
+			- chart
+			- *.commondatastorage
+			- fonts
+			- *.storage
+			- www
+
+		- gstatic.com subdomains:
+
+			- (www.)	(^ → www)
+			- csi
+			- encrypted-tbn\d
+			- g0
+			- *.metric
+			- ssl
+			- t\d
+
+		- api.recaptcha.net	(→ www.google.com)
+		- api-secure.recaptcha.net
+		- gdata.youtube.com
+
+
+	ssl.google-analytics.com/ga.js sets __utm\w wildcard
+	cookies on whichever domain it is loaded from.
+
+-->
+<ruleset name="Google APIs">
+
+	<target host="gmodules.com" />
+	<target host="www.gmodules.com" />
+	<target host="google.com" />
+	<target host="apis.google.com" />
+	<target host="*.apis.google.com" />
+	<target host="*.clients.google.com" />
+	<target host="www.google.com" />
+	<target host="*.google-analytics.com" />
+	<target host="*.googleapis.com" />
+	<target host="gstatic.com" />
+	<target host="*.gstatic.com" />
+	<!--	Captive portal detection redirects to this URL, and many captive
+		portals break TLS, so exempt this redirect URL.
+		See GitHub bug #368
+							-->
+		<exclusion pattern="^http://www\.gstatic\.com/generate_204" />
+	<target host="*.recaptcha.net" />
+	<target host="gdata.youtube.com" />
+		<exclusion pattern="^http://gdata\.youtube\.com/crossdomain\.xml" />
+
+
+	<securecookie host="^ssl\.google-analytics\.com$" name=".+" />
+
+
+	<rule from="^http://(?:www\.)?gmodules\.com/ig/images/"
+		to="https://www.google.com/ig/images/" />
+
+	<!--	jsapi was causing problems on some sites that embed google maps:
+		https://trac.torproject.org/projects/tor/ticket/2335
+		Apparently now fixed; thanks, Google!
+							-->
+	<rule from="^http://(?:www\.)?google\.com/(afsonline/|chart|jsapi|recaptcha/|uds)"
+		to="https://www.google.com/$1" />
+
+	<rule from="^http://(api|[\w-]+\.client)s\.google\.com/"
+		to="https://$1s.google.com/" />
+
+	<rule from="^http://chart\.apis\.google\.com/chart"
+		to="https://chart.googleapis.com/chart" />
+
+	<rule from="^http://(ssl|www)\.google-analytics\.com/"
+		to="https://$1.google-analytics.com/" />
+
+	<rule from="^http://(ajax|chart|fonts|www)\.googleapis\.com/"
+		to="https://$1.googleapis.com/" />
+
+	<rule from="^http://([^@:\./]+\.)?(commondata)?storage\.googleapis\.com/"
+		to="https://$1$2storage.googleapis.com/" />
+
+	<!--	There is an interesting question about whether we should
+		append &strip=1 to all cache URLs.  This causes them to load
+		without images and styles, which is more secure but can look
+		worse.
+			Without &strip=1, the images and styles from the cached
+		pages still load from the original, typically unencrypted, page.
+			With &strip=1, the cached page will be text-only and
+		will come exclusively from Google's HTTPS server.
+									-->
+	<rule from="^http://(?:www\.)?gstatic\.com/"
+		to="https://www.gstatic.com/" />
+
+	<rule from="^http://(csi|encrypted-tbn\d|g0|[\w-]+\.metric|ssl|t\d)\.gstatic\.com/"
+		to="https://$1.gstatic.com/" />
+
+	<rule from="^http://api\.recaptcha\.net/"
+		to="https://www.google.com/recaptcha/api/" />
+
+	<rule from="^http://api-secure\.recaptcha\.net/"
+		to="https://api-secure.recaptcha.net/" />
+
+	<rule from="^http://gdata\.youtube\.com/"
+		to="https://gdata.youtube.com/" />
+
+</ruleset>

searx/https_rules/GoogleCanada.xml

@@ -0,0 +1,6 @@
+<ruleset name="GoogleCanada">
+	<target host="google.ca" />
+	<target host="*.google.ca" />
+	<rule from="^http://([^/:@\.]+)\.google\.ca/finance" to="https://$1.google.ca/finance"/>
+</ruleset>
+

searx/https_rules/GoogleImages.xml

@@ -0,0 +1,65 @@
+<!--
+	For other Google coverage, see GoogleServices.xml.
+
+
+	Problematic domains:
+
+		- www.google.bo *
+		- www.google.co *
+		- www.google.ec *
+		- www.google.in *
+		- www.google.kr *
+		- www.google.com.kz **
+		- www.google.com.lk *
+		- www.google.mx **
+		- www.google.sg *
+		- www.google.sl *
+		- www.google.ug *
+		- www.google.vn *
+
+	* 404; mismatched, CN: google.com
+	** Works; mismatched, CN: google.com
+
+-->
+<ruleset name="Google Images">
+
+	<target host="google.*" />
+	<target host="www.google.*" />
+	<target host="google.co.*" />
+	<target host="www.google.co.*" />
+	<target host="google.com" />
+	<target host="images.google.com" />
+	<target host="google.com.*" />
+	<target host="www.google.com.*" />
+		<!--
+			Only handle image-related paths in this ruleset:
+										-->
+		<exclusion pattern="^http://(?:www\.)?google(?:\.com?)?\.\w{2,3}/(?!(?:advanced_image_search|imghp|.*tb(?:m=isch|s=sbi)))" />
+
+
+	<rule from="^http://(?:www\.)?google\.com/"
+		to="https://www.google.com/" />
+
+	<rule from="^http://images\.google\.com/"
+		to="https://images.google.com/" />
+
+	<!--	First handle problematic domains:
+							-->
+	<rule from="^http://(?:www\.)?google\.co/"
+		to="https://www.google.com/" />
+
+	<rule from="^http://(?:www\.)?google\.(?:co\.)?(in|kr|ug)/"
+		to="https://www.google.co.$1/" />
+
+	<rule from="^http://(?:www\.)?google\.(?:com\.)?(kz|lk)/"
+		to="https://www.google.$1/" />
+
+	<rule from="^http://(?:www\.)?google\.(?:com\.)?(bo|ec|mx|sg|sl|vn)/"
+		to="https://www.google.com.$1/" />
+
+	<!--	And then the rest:
+					-->
+	<rule from="^http://(?:www\.)?google\.(com?\.)?(ae|ar|at|au|bg|bh|br|ca|ch|cl|co|cr|cu|de|eg|es|fi|fr|gh|gt|hr|id|ie|il|it|jo|jp|jm|ke|kw|lb|ly|my|na|ng|nl|no|nz|om|pa|pe|pk|pl|pt|py|qa|ro|ru|rw|sa|se|sv|th|tr|uk|uy|ve|za|zw)/"
+		to="https://www.google.$1$2/" />
+
+</ruleset>

searx/https_rules/GoogleMainSearch.xml

@@ -0,0 +1,78 @@
+<ruleset name="Search www.google.com">
+
+<!-- 
+Enabling this ruleset should cause searches to go to
+https://www.google.com rather than https://encrypted.google.com.  Note that
+the filename is important; it must be before GoogleSearch.xml in a bash
+expansion of src/chrome/content/rules/*.xml in order to take precedence. 
+-->
+
+  <target host="*.google.com" />
+  <target host="google.com" />
+  <target host="www.google.com.*" />
+  <target host="google.com.*" />
+  <target host="www.google.co.*" />
+  <target host="google.co.*" />
+  <target host="www.google.*" />
+  <target host="google.*" />
+  <!-- beyond clients1 these do not currently exist in the ccTLDs,
+       but just in case... -->
+  <target host="clients1.google.com.*" />
+  <target host="clients2.google.com.*" />
+  <target host="clients3.google.com.*" />
+  <target host="clients4.google.com.*" />
+  <target host="clients5.google.com.*" />
+  <target host="clients6.google.com.*" />
+  <target host="clients1.google.co.*" />
+  <target host="clients2.google.co.*" />
+  <target host="clients3.google.co.*" />
+  <target host="clients4.google.co.*" />
+  <target host="clients5.google.co.*" />
+  <target host="clients6.google.co.*" />
+  <target host="clients1.google.*" />
+  <target host="clients2.google.*" />
+  <target host="clients3.google.*" />
+  <target host="clients4.google.*" />
+  <target host="clients5.google.*" />
+  <target host="clients6.google.*" />
+
+  <rule from="^http://www\.google\.com/$"
+          to="https://www.google.com/"/>
+
+  <!-- The most basic case. -->
+
+  <rule from="^http://(?:www\.)?google\.com/search"
+          to="https://www.google.com/search"/>
+
+  <!-- A very annoying exception that we seem to need for the basic case -->
+
+  <exclusion pattern="^http://(?:www\.)?google\.com/search.*tbs=shop" />
+  <exclusion pattern="^http://clients[0-9]\.google\.com/.*client=products.*" />
+  <exclusion pattern="^http://suggestqueries\.google\.com/.*client=.*" />
+
+  <!-- https://trac.torproject.org/projects/tor/ticket/9713 -->
+
+  <exclusion pattern="^http://clients[0-9]\.google\.com/ocsp" />
+
+  <!-- This is necessary for image results links from web search results -->
+
+  <exclusion pattern="^http://(?:www\.)?google\.com/search.*tbm=isch.*" />
+
+  <rule from="^http://(?:www\.)?google\.com/webhp"
+          to="https://www.google.com/webhp"/>
+
+  <rule from="^http://(?:www\.)?google\.com/#"
+          to="https://www.google.com/#"/>
+
+  <rule from="^http://(?:www\.)?google\.com/$"
+          to="https://www.google.com/"/>
+
+   <!-- Completion urls look like this:
+
+http://clients2.google.co.jp/complete/search?hl=ja&client=hp&expIds=17259,24660,24729,24745&q=m&cp=1 HTTP/1.1\r\n
+
+   -->
+  <rule from="^http://clients[0-9]\.google\.com/complete/search"
+          to="https://clients1.google.com/complete/search"/>
+
+</ruleset>

searx/https_rules/GoogleMaps.xml

@@ -0,0 +1,67 @@
+<!--
+	Problematic domains:
+
+		- khms *
+		- khms[0-3] *
+
+	* $ 404s
+
+
+	Fully covered domains:
+
+		- google.com subdomains:
+
+			- khms
+			- khms[0-3]
+
+-->
+<ruleset name="Google Maps">
+
+	<target host="maps.google.*" />
+		<!--
+			https://trac.torproject.org/projects/tor/ticket/8627
+										-->
+		<exclusion pattern="^http://maps\.google\.com/local_url" />
+		<exclusion pattern="^http://maps\.google\.gr/transitathens" />
+	<target host="maps.google.co.*" />
+	<target host="khms.google.com" />
+	<target host="khms0.google.com" />
+	<target host="khms1.google.com" />
+	<target host="khms2.google.com" />
+	<target host="khms3.google.com" />
+	<target host="maps-api-ssl.google.com" />
+	<target host="mw2.google.com" />
+	<target host="maps.google.com.*" />
+	<target host="maps.googleapis.com" />
+		<!--
+			https://mail1.eff.org/pipermail/https-everywhere-rules/2012-September/001317.html
+														-->
+		<!--exclusion pattern="^http://maps\.googleapis\.com/map(files/lib/map_1_20\.swf|sapi/publicapi\?file=flashapi)" /-->
+		<exclusion pattern="^http://maps\.googleapis\.com/map(?:files/lib/map_\d+_\d+\.swf|sapi/publicapi\?file=flashapi)" />
+	<target host="maps.gstatic.com" />
+
+
+	<!--securecookie host="^maps\.google\.(com?\.)?(au|ca|gh|ie|in|jm|ke|lk|my|n[agz]|pk|rw|sl|sg|ug|uk|za|zw)$" name=".+" /-->
+	<securecookie host="^maps\.google\.[\w.]{2,6}$" name=".+" />
+	<securecookie host="^maps\.g(?:oogle|oogleapis|static)\.com$" name=".+" />
+	<securecookie host="^maps-api-ssl\.google\.com$" name=".+" />
+
+
+	<rule from="^http://maps\.google\.([^/]+)/"
+		to="https://maps.google.$1/" />
+
+	<!--	http://khms.../$ 404s:
+					-->
+	<rule from="^http://khms\d?\.google\.com/+\??$"
+		to="https://www.google.com/" />
+
+	<rule from="^http://(khms\d?|maps-api-ssl|mw2)\.google\.com/"
+		to="https://$1.google.com/" />
+
+	<rule from="^http://maps\.g(oogleapis|static)\.com/"
+		to="https://maps.g$1.com/" />
+
+	<rule from="^https://maps\.googleapis\.com/map(?=files/lib/map_\d+_\d+\.swf|sapi/publicapi\?file=flashapi)"
+		to="http://maps.googleapis.com/map" downgrade="1" />
+
+</ruleset>

searx/https_rules/GoogleMelange.xml

@@ -0,0 +1,6 @@
+<ruleset name="GoogleMelange">
+  <target host="www.google-melange.com" />
+  <target host="google-melange.com" />
+
+  <rule from="^http://(www\.)?google-melange\.com/" to="https://www.google-melange.com/" />
+</ruleset>

searx/https_rules/GoogleSearch.xml

@@ -0,0 +1,135 @@
+<ruleset name="Google Search">
+
+	<target host="google.com" />
+	<target host="*.google.com" />
+	<target host="google.com.*" />
+	<target host="www.google.com.*" />
+	<target host="google.co.*" />
+	<target host="www.google.co.*" />
+	<target host="google.*" />
+	<target host="www.google.*" />
+	<!--
+		Beyond clients1 these do not currently
+		exist in the ccTLDs, but just in case...
+							-->
+	<target host="clients1.google.com.*" />
+	<target host="clients2.google.com.*" />
+	<target host="clients3.google.com.*" />
+	<target host="clients4.google.com.*" />
+	<target host="clients5.google.com.*" />
+	<target host="clients6.google.com.*" />
+	<target host="clients1.google.co.*" />
+	<target host="clients2.google.co.*" />
+	<target host="clients3.google.co.*" />
+	<target host="clients4.google.co.*" />
+	<target host="clients5.google.co.*" />
+	<target host="clients6.google.co.*" />
+	<target host="clients1.google.*" />
+	<target host="clients2.google.*" />
+	<target host="clients3.google.*" />
+	<target host="clients4.google.*" />
+	<target host="clients5.google.*" />
+	<target host="clients6.google.*" />
+
+
+	<!--	Some Google pages can generate naive links back to the
+		unencrypted version of encrypted.google.com, which is
+		a 301 but theoretically vulnerable to SSL stripping.
+									-->
+	<rule from="^http://encrypted\.google\.com/"
+		to="https://encrypted.google.com/" />
+
+	<!--	The most basic case.
+					-->
+	<rule from="^http://(?:www\.)?google\.com/search"
+		to="https://encrypted.google.com/search" />
+
+	<!--	A very annoying exception that we
+		seem to need for the basic case
+						-->
+	<exclusion pattern="^http://(?:www\.)?google\.com/search.*tbs=shop" />
+	<exclusion pattern="^http://clients\d\.google\.com/.*client=products.*" />
+	<exclusion pattern="^http://suggestqueries\.google\.com/.*client=.*" />
+
+  <!-- https://trac.torproject.org/projects/tor/ticket/9713 
+         -->
+
+  <exclusion pattern="^http://clients[0-9]\.google\.com/ocsp" />
+
+
+	<!--	This is necessary for image results
+		links from web search results
+						-->
+	<exclusion pattern="^http://(?:www\.)?google\.com/search.*tbm=isch.*" />
+
+	<rule from="^http://(?:www\.)?google\.com/about"
+		to="https://www.google.com/about" />
+
+	<!--	There are two distinct cases for these firefox searches	-->
+
+	<rule from="^http://(?:www\.)?google(?:\.com?)?\.[a-z]{2}/firefox/?$"
+		to="https://encrypted.google.com/" />
+
+	<rule from="^http://(?:www\.)?google(?:\.com?)?\.[a-z]{2}/firefox"
+		to="https://encrypted.google.com/webhp" />
+
+	<rule from="^http://(?:www\.)?google\.com/webhp"
+		to="https://encrypted.google.com/webhp" />
+
+	<rule from="^http://codesearch\.google\.com/"
+		to="https://codesearch.google.com/" />
+
+	<rule from="^http://(?:www\.)?google\.com/codesearch"
+		to="https://www.google.com/codesearch" />
+
+	<rule from="^http://(?:www\.)?google\.com/#"
+		to="https://encrypted.google.com/#" />
+
+	<rule from="^http://(?:www\.)?google\.com/$"
+		to="https://encrypted.google.com/" />
+
+	<!--	Google supports IPv6 search, including
+		HTTPS with a valid certificate!	-->
+	<rule from="^http://ipv6\.google\.com/"
+		to="https://ipv6.google.com/" />
+
+	<!--	most google international sites look like
+		"google.fr", some look like "google.co.jp",
+		and some crazy ones like "google.com.au"	-->
+
+	<rule from="^http://(www\.)?google(\.com?)?\.([a-z]{2})/(search\?|#)"
+		to="https://$1google$2.$3/$4" />
+
+	<!--	Language preference setting	-->
+	<rule from="^http://(www\.)?google(\.com?)?\.([a-z]{2})/setprefs"
+	to="https://$1google$2.$3/setprefs" />
+
+	<!--	Completion urls look like this:
+
+http://clients2.google.co.jp/complete/search?hl=ja&client=hp&expIds=17259,24660,24729,24745&q=m&cp=1 HTTP/1.1\r\n
+
+		-->
+	<rule from="^http://clients\d\.google\.com/complete/search"
+		to="https://clients1.google.com/complete/search" />
+
+	<rule from="^http://clients\d\.google(\.com?\.[a-z]{2})/complete/search"
+		to="https://clients1.google.$1/complete/search" />
+
+	<rule from="^http://clients\d\.google\.([a-z]{2})/complete/search"
+		to="https://clients1.google.$1/complete/search" />
+
+	<rule from="^http://suggestqueries\.google\.com/complete/search"
+		to="https://clients1.google.com/complete/search" />
+
+	<rule from="^http://(www\.)?google\.(com?\.)?([a-z]{2})/(?:webhp)?$"
+		to="https://$1google.$2$3/" />
+
+	<!--	If there are URL parameters, keep them.	-->
+	<rule from="^http://(www\.)?google\.(com?\.)?([a-z]{2})/(?:webhp)?\?"
+		to="https://$1google.$2$3/webhp?" />
+
+	<!-- teapot -->
+	<rule from="^http://(www\.)?google(\.com?)?\.([a-z]{2})/teapot"
+		to="https://$1google$2.$3/teapot" />
+
+</ruleset>

searx/https_rules/GoogleServices.xml

@@ -0,0 +1,345 @@
+<!--
+	Other Google rulesets:
+
+		- 2mdn.net.xml
+		- Admeld.xml
+		- ChannelIntelligence.com.xml
+		- Doubleclick.net.xml
+		- FeedBurner.xml
+		- Google.org.xml
+		- GoogleAPIs.xml
+		- Google_App_Engine.xml
+		- GoogleImages.xml
+		- GoogleShopping.xml
+		- Ingress.xml
+		- Meebo.xml
+		- Orkut.xml
+		- Postini.xml
+		- WebM_Project.org.xml
+
+
+	Nonfunctional domains:
+
+		- feedproxy.google.com			(404, valid cert)
+		- partnerpage.google.com *
+		- safebrowsing.clients.google.com	(404, mismatched)
+		- (www.)googlesyndicatedsearch.com	(404; mismatched, CN: google.com)
+		- buttons.googlesyndication.com *
+
+	* 404, valid cert
+
+
+	Nonfunctional google.com paths:
+
+		- analytics	(redirects to http)
+		- imgres
+		- gadgets *
+		- hangouts	(404)
+		- u/		(404)
+
+	* Redirects to http
+
+
+	Problematic domains:
+
+		- www.goo.gl		(404; mismatched, CN: *.google.com)
+
+		- google.com subdomains:
+
+			- books		(googlebooks/, images/, & intl/ 404, but works when rewritten to www)
+			- cbks0 ****
+			- earth *
+			- gg		($ 404s)
+			- knoll *
+			- scholar **
+			- trends *
+
+		- news.google.cctld **
+		- scholar.google.cctld **
+		- *-opensocial.googleusercontent.com ***
+
+	**** $ 404s
+	* 404, valid cert
+	** Redirects to http, valid cert
+	*** Breaks followers widget - https://trac.torproject.org/projects/tor/ticket/7294
+
+
+	Partially covered domains:
+
+		- google.cctld subdomains:
+
+			- scholar	(→ www)
+
+		- google.com subdomains:
+
+			- (www.)
+			- cbks0		($ 404s)
+			- gg		($ 404s)
+			- news		(→ www)
+			- scholar	(→ www)
+
+		- *.googleusercontent.com	(*-opensocial excluded)
+
+
+	Fully covered domains:
+
+		- lh[3-6].ggpht.com
+		- (www.)goo.gl		(www → ^)
+
+		- google.com subdomains:
+
+			- accounts
+			- adwords
+			- apis
+			- appengine
+			- books		(→ encrypted)
+			- calendar
+			- checkout
+			- chrome
+			- clients[12]
+			- code
+			- *.corp
+			- developers
+			- dl
+			- docs
+			- docs\d
+			- \d.docs
+			- drive
+			- earth		(→ www)
+			- encrypted
+			- encrypted-tbn[123]
+			- feedburner
+			- fiber
+			- finance
+			- glass
+			- groups
+			- health
+			- helpouts
+			- history
+			- hostedtalkgadget
+			- id
+			- investor
+			- knol
+			- knoll		(→ knol)
+			- lh\d
+			- mail
+			- chatenabled.mail
+			- pack
+			- picasaweb
+			- pki
+			- play
+			- plus
+			- plusone
+			- productforums
+			- profiles
+			- safebrowsing-cache
+			- cert-test.sandbox
+			- plus.sandbox
+			- sb-ssl
+			- script
+			- security
+			- services
+			- servicessites
+			- sites
+			- spreadsheets
+			- spreadsheets\d
+			- support
+			- talk
+			- talkgadget
+			- tbn2			(→ encrypted-tbn2)
+			- tools
+			- trends		(→ www)
+
+		- partner.googleadservices.com
+		- (www.)googlecode.com
+		- *.googlecode.com	(per-project subdomains)
+		- googlesource.com
+		- *.googlesource.com
+		- pagead2.googlesyndication.com
+		- tpc.googlesyndication.com
+		- mail-attachment.googleusercontent.com
+		- webcache.googleusercontent.com
+
+
+	XXX: Needs more testing
+
+-->
+<ruleset name="Google Services">
+
+	<target host="*.ggpht.com" />
+	<target host="gmail.com" />
+	<target host="www.gmail.com" />
+	<target host="goo.gl" />
+	<target host="www.goo.gl" />
+	<target host="google.*" />
+	<target host="accounts.google.*" />
+	<target host="adwords.google.*" />
+	<target host="finance.google.*" />
+	<target host="groups.google.*" />
+	<target host="it.google.*" />
+	<target host="news.google.*" />
+		<exclusion pattern="^http://(?:news\.)?google\.com/(?:archivesearch|newspapers)" />
+	<target host="picasaweb.google.*" />
+	<target host="scholar.google.*" />
+	<target host="www.google.*" />
+	<target host="*.google.ca" />
+	<target host="google.co.*" />
+	<target host="accounts.google.co.*" />
+	<target host="adwords.google.co.*" />
+	<target host="finance.google.co.*" />
+	<target host="groups.google.co.*" />
+	<target host="id.google.co.*" />
+	<target host="news.google.co.*" />
+	<target host="picasaweb.google.co.*" />
+	<target host="scholar.google.co.*" />
+	<target host="www.google.co.*" />
+	<target host="google.com" />
+	<target host="*.google.com" />
+		<exclusion pattern="^http://(?:www\.)?google\.com/analytics/*(?:/[^/]+)?(?:\?.*)?$" />
+		<!--exclusion pattern="^http://books\.google\.com/(?!books/(\w+\.js|css/|javascript/)|favicon\.ico|googlebooks/|images/|intl/)" /-->
+		<exclusion pattern="^http://cbks0\.google\.com/(?:$|\?)" />
+		<exclusion pattern="^http://gg\.google\.com/(?!csi(?:$|\?))" />
+	<target host="google.com.*" />
+	<target host="accounts.google.com.*" />
+	<target host="adwords.google.com.*" />
+	<target host="groups.google.com.*" />
+	<target host="id.google.com.*" />
+	<target host="news.google.com.*" />
+	<target host="picasaweb.google.com.*" />
+	<target host="scholar.google.com.*" />
+	<target host="www.google.com.*" />
+	<target host="partner.googleadservices.com" />
+	<target host="googlecode.com" />
+	<target host="*.googlecode.com" />
+	<target host="googlemail.com" />
+	<target host="www.googlemail.com" />
+	<target host="googlesource.com" />
+	<target host="*.googlesource.com" />
+	<target host="*.googlesyndication.com" />
+	<target host="www.googletagservices.com" />
+	<target host="googleusercontent.com" />
+	<target host="*.googleusercontent.com" />
+		<!--
+			Necessary for the Followers widget:
+
+				 https://trac.torproject.org/projects/tor/ticket/7294
+											-->
+		<exclusion pattern="http://[^@:\./]+-opensocial\.googleusercontent\.com" />
+
+
+	<!--	Can we secure any of these wildcard cookies safely?
+									-->
+	<!--securecookie host="^\.google\.com$" name="^(hl|I4SUserLocale|NID|PREF|S)$" /-->
+	<!--securecookie host="^\.google\.[\w.]{2,6}$" name="^(hl|I4SUserLocale|NID|PREF|S|S_awfe)$" /-->
+	<securecookie host="^(?:accounts|adwords|\.code|login\.corp|developers|docs|\d\.docs|fiber|mail|picasaweb|plus|\.?productforums|support)\.google\.[\w.]{2,6}$" name=".+" />
+	<securecookie host="^www\.google\.com$" name="^GoogleAccountsLocale_session$" />
+	<securecookie host="^mail-attachment\.googleusercontent\.com$" name=".+" />
+	<securecookie host="^gmail\.com$" name=".+" />
+	<securecookie host="^www\.gmail\.com$" name=".+" />
+	<securecookie host="^googlemail\.com$" name=".+" />
+	<securecookie host="^www\.googlemail\.com$" name=".+" />
+
+
+	<!--    - lh 3-6 exist
+		- All appear identical
+		- Identical to lh\d.googleusercontent.com
+					-->
+	<rule from="^http://lh(\d)\.ggpht\.com/"
+		to="https://lh$1.ggpht.com/" />
+
+	<rule from="^http://lh(\d)\.google\.ca/"
+		to="https://lh$1.google.ca/" />
+
+
+	<rule from="^http://(www\.)?g(oogle)?mail\.com/"
+		to="https://$1g$2mail.com/" />
+
+	<rule from="^http://(?:www\.)?goo\.gl/"
+		to="https://goo.gl/" />
+
+
+	<!--	Redirects to http when rewritten to www:
+							-->
+	<rule from="^http://books\.google\.com/"
+		to="https://encrypted.google.com/" />
+
+	<!--	tisp$ 404s:
+				-->
+	<rule from="^http://(?:www\.)?google\.((?:com?\.)?\w{2,3})/tisp(?=$|\?)"
+		to="https://www.google.$1/tisp/" />
+
+	<!--	Paths that work on all in google.*
+							-->
+	<rule from="^http://(?:www\.)?google\.((?:com?\.)?\w{2,3})/(accounts|adplanner|ads|adsense|adwords|analytics|bookmarks|chrome|contacts|coop|cse|css|culturalinstitute|doodles|earth|favicon\.ico|finance|get|goodtoknow|googleblogs|grants|green|hostednews|images|intl|js|landing|logos|mapmaker|newproducts|news|nexus|patents|policies|prdhp|profiles|products|reader|s2|settings|shopping|support|tisp|tools|transparencyreport|trends|urchin|webmasters)(?=$|[?/])"
+		 to="https://www.google.$1/$2" />
+
+	<!--	Paths that 404 on .ccltd, but work on .com:
+								-->
+	<rule from="^http://(?:www\.)?google\.(?:com?\.)?\w{2,3}/(?=calendar|dictionary|doubleclick|help|ideas|pacman|postini|powermeter|url)"
+		 to="https://www.google.com/" />
+
+	<rule from="^http://(?:www\.)?google\.(?:com?\.)?\w{2,3}/custom"
+		 to="https://www.google.com/cse" />
+
+	<!--	Paths that only exist/work on .com
+							-->
+	<rule from="^http://(?:www\.)?google\.com/(\+|appsstatus|books|buzz|extern_js|glass|googlebooks|ig|insights|moderator|phone|safebrowsing|videotargetting|webfonts)(?=$|[?/])"
+		to="https://www.google.com/$1" />
+
+	<!--	Subdomains that work on all in google.*
+							-->
+	<rule from="^http://(accounts|adwords|finance|groups|id|picasaweb|)\.google\.((?:com?\.)?\w{2,3})/"
+		to="https://$1.google.$2/" />
+
+	<!--	Subdomains that only exist/work on .com
+							-->
+	<rule from="^http://(apis|appengine|books|calendar|cbks0|chat|checkout|chrome|clients[12]|code|[\w-]+\.corp|developers|dl|docs\d?|\d\.docs|drive|encrypted|encrypted-tbn[123]|feedburner|fiber|fonts|gg|glass||health|helpouts|history|(?:hosted)?talkgadget|investor|lh\d|(?:chatenabled\.)?mail|pack|pki|play|plus(?:\.sandbox)?|plusone|productforums|profiles|safebrowsing-cache|cert-test\.sandbox|sb-ssl|script|security|services|servicessites|sites|spreadsheets\d?|support|talk|tools)\.google\.com/"
+		to="https://$1.google.com/" />
+
+	<exclusion pattern="^http://clients[0-9]\.google\.com/ocsp"/>
+
+	<rule from="^http://earth\.google\.com/"
+		to="https://www.google.com/earth/" />
+
+	<rule from="^http://scholar\.google\.((?:com?\.)?\w{2,3})/intl/"
+		to="https://www.google.$1/intl/" />
+
+	<rule from="^http://(?:encrypted-)?tbn2\.google\.com/"
+		to="https://encrypted-tbn2.google.com/" />
+
+
+	<rule from="^http://knoll?\.google\.com/"
+		to="https://knol.google.com/" />
+
+
+	<rule from="^http://news\.google\.(?:com?\.)?\w{2,3}/(?:$|news|newshp)"
+		to="https://www.google.com/news" />
+
+	<rule from="^http://trends\.google\.com/"
+		 to="https://www.google.com/trends" />
+
+
+	<rule from="^http://([^/:@\.]+\.)?googlecode\.com/"
+		 to="https://$1googlecode.com/" />
+
+	<rule from="^http://([^\./]\.)?googlesource\.com/"
+		to="https://$1googlesource.com/" />
+
+
+	<rule from="^http://partner\.googleadservices\.com/"
+		 to="https://partner.googleadservices.com/" />
+
+	<rule from="^http://(pagead2|tpc)\.googlesyndication\.com/"
+		 to="https://$1.googlesyndication.com/" />
+
+	<!--	!www doesn't exist.
+					-->
+	<rule from="^http://www\.googletagservices\.com/tag/js/"
+		to="https://www.googletagservices.com/tag/js/" />
+
+
+	<rule from="^http://([^@:\./]+)\.googleusercontent\.com/"
+		to="https://$1.googleusercontent.com/" />
+	
+
+</ruleset>

searx/https_rules/GoogleShopping.xml

@@ -0,0 +1,28 @@
+<!--
+	For other Google coverage, see GoogleServices.xml.
+
+-->
+<ruleset name="Google Shopping">
+
+	<target host="google.*" />
+	<target host="www.google.*" />
+	<target host="google.co.*" />
+	<target host="www.google.co.*" />
+	<target host="*.google.com" />
+	<target host="google.com.*" />
+	<target host="www.google.com.*" />
+
+
+	<rule from="^http://encrypted\.google\.com/(prdhp|shopping)" 
+		to="https://www.google.com/$1" />
+
+	<rule from="^http://shopping\.google\.com/"
+		to="https://shopping.google.com/" />
+
+	<rule from="^http://(?:encrypted|www)\.google\.com/(.*tbm=shop)"
+		to="https://www.google.com/$1" />
+
+	<rule from="^http://(?:www\.)?google\.((?:com?\.)?(?:ae|ar|at|au|bg|bh|bo|br|ca|ch|cl|cr|co|cu|de|ec|eg|es|fi|fr|gh|gt|hr|id|ie|il|in|it|jm|jo|jp|ke|kr|kw|kz|lb|lk|ly|mx|my|na|ng|nl|no|nz|om|pa|pe|pk|pl|pt|py|qa|ro|ru|rw|sa|sg|sl|se|sv|th|tr|ug|uk|uy|ve|vn|za|zw))/(?=prdhp|shopping)"
+		to="https://www.google.com/$1" />
+
+</ruleset>

searx/https_rules/GoogleSorry.xml

@@ -0,0 +1,7 @@
+<ruleset name="GoogleSorry">
+  <target host="sorry.google.com" />
+  <target host="www.google.com" />
+  <target host="google.com" />
+
+  <rule from="^http://((sorry|www)\.)?google\.com/sorry/" to="https://sorry.google.com/sorry/" />
+</ruleset>

searx/https_rules/GoogleTranslate.xml

@@ -0,0 +1,8 @@
+<ruleset name="Google Translate (broken)" default_off="redirect loops">
+  <target host="translate.googleapis.com" />
+  <target host="translate.google.com" />
+
+  <rule from="^http://translate\.googleapis\.com/" to="https://translate.googleapis.com/"/>
+  <rule from="^http://translate\.google\.com/"
+      to="https://translate.google.com/" />
+</ruleset>

searx/https_rules/GoogleVideos.xml

@@ -0,0 +1,83 @@
+<ruleset name="Google Videos">
+  <target host="*.google.com" />
+  <target host="google.com" />
+  <target host="www.google.com.*" />
+  <target host="google.com.*" />
+  <target host="www.google.co.*" />
+  <target host="google.co.*" />
+  <target host="www.google.*" />
+  <target host="google.*" />
+
+  <rule from="^http://encrypted\.google\.com/videohp" 
+          to="https://encrypted.google.com/videohp" />
+
+  <!-- https://videos.google.com is currently broken; work around that... -->
+  <rule from="^https?://videos?\.google\.com/$"
+          to="https://encrypted.google.com/videohp" />
+  <rule from="^http://(?:www\.)?google\.com/videohp"
+	  to="https://encrypted.google.com/videohp" />
+  <rule from="^http://(?:images|www|encrypted)\.google\.com/(.*tbm=isch)"
+          to="https://encrypted.google.com/$1" />
+
+  <rule
+   from="^http://(?:www\.)?google\.(?:com?\.)?(?:au|ca|gh|ie|in|jm|ke|lk|my|na|ng|nz|pk|rw|sl|sg|ug|uk|za|zw)/videohp"
+     to="https://encrypted.google.com/videohp" />
+  <rule
+   from="^http://(?:www\.)?google\.(?:com?\.)?(?:ar|bo|cl|co|cu|cr|ec|es|gt|mx|pa|pe|py|sv|uy|ve)/videohp$"
+    to="https://encrypted.google.com/videohp?hl=es" />
+  <rule
+   from="^http://(?:www\.)?google\.(?:com\.)?(?:ae|bh|eg|jo|kw|lb|ly|om|qa|sa)/videohp$"
+     to="https://encrypted.google.com/videohp?hl=ar" />
+  <rule from="^http://(?:www\.)?google\.(?:at|ch|de)/videohp$"
+          to="https://encrypted.google.com/videohp?hl=de" />
+  <rule from="^http://(?:www\.)?google\.(fr|nl|it|pl|ru|bg|pt|ro|hr|fi|no)/videohp$"
+          to="https://encrypted.google.com/videohp?hl=$1" />
+  <rule from="^http://(?:www\.)?google\.com?\.(id|th|tr)/videohp$"
+          to="https://encrypted.google.com/videohp?hl=$1" />
+  <rule from="^http://(?:www\.)?google\.com\.il/videohp$"
+          to="https://encrypted.google.com/videohp?hl=he" />
+  <rule from="^http://(?:www\.)?google\.com\.kr/videohp$"
+          to="https://encrypted.google.com/videohp?hl=ko" />
+  <rule from="^http://(?:www\.)?google\.com\.kz/videohp$"
+          to="https://encrypted.google.com/videohp?hl=kk" />
+  <rule from="^http://(?:www\.)?google\.com\.jp/videohp$"
+          to="https://encrypted.google.com/videohp?hl=ja" />
+  <rule from="^http://(?:www\.)?google\.com\.vn/videohp$"
+          to="https://encrypted.google.com/videohp?hl=vi" />
+  <rule from="^http://(?:www\.)?google\.com\.br/videohp$"
+          to="https://encrypted.google.com/videohp?hl=pt-BR" />
+  <rule from="^http://(?:www\.)?google\.se/videohp$"
+          to="https://encrypted.google.com/videohp?hl=sv" />
+
+<!-- If there are URL parameters, keep them. -->
+  <rule
+   from="^http://(?:www\.)?google\.(?:com?\.)?(?:ar|bo|cl|co|cu|cr|ec|es|gt|mx|pa|pe|py|sv|uy|ve)/videohp\?"
+    to="https://encrypted.google.com/videohp?hl=es&#38;" />
+  <rule
+   from="^http://(?:www\.)?google\.(?:com\.)?(?:ae|bh|eg|jo|kw|lb|ly|om|qa|sa)/videohp\?"
+     to="https://encrypted.google.com/videohp?hl=ar&#38;" />
+  <rule from="^http://(?:www\.)?google\.(?:at|ch|de)/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=de&#38;" />
+  <rule from="^http://(?:www\.)?google\.(fr|nl|it|pl|ru|bg|pt|ro|hr|fi|no)/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=$1&#38;" />
+  <rule from="^http://(?:www\.)?google\.com?\.(id|th|tr)/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=$1&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.il/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=he&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.kr/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=ko&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.kz/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=kk&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.jp/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=ja&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.vn/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=vi&#38;" />
+  <rule from="^http://(?:www\.)?google\.com\.br/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=pt-BR&#38;" />
+  <rule from="^http://(?:www\.)?google\.se/videohp\?"
+          to="https://encrypted.google.com/videohp?hl=sv&#38;" />
+
+	<rule from="^http://video\.google\.com/ThumbnailServer2"
+		to="https://video.google.com/ThumbnailServer2" />
+
+</ruleset>

searx/https_rules/GoogleWatchBlog.xml

@@ -0,0 +1,17 @@
+<!--
+	gwbhrd.appspot.com
+
+-->
+<ruleset name="GoogleWatchBlog">
+
+	<target host="googlewatchblog.de" />
+	<target host="*.googlewatchblog.de" />
+
+
+	<securecookie host="^(?:www)?\.googlewatchblog\.de$" name=".+" />
+
+
+	<rule from="^http://(static\.|www\.)?googlewatchblog\.de/"
+		to="https://$1googlewatchblog.de/" />
+
+</ruleset>

searx/https_rules/Google_App_Engine.xml

@@ -0,0 +1,21 @@
+<!--
+	For other Google coverage, see GoogleServices.xml.
+
+-->
+<ruleset name="Google App Engine">
+
+	<target host="appspot.com" />
+	<target host="*.appspot.com" />
+		<!--
+			Redirects to http for some reason.
+								-->
+		<exclusion pattern="^http://photomunchers\.appspot\.com/" />
+
+
+	<securecookie host="^.+\.appspot\.com$" name=".+" />
+
+
+	<rule from="^http://([^@:\./]+\.)?appspot\.com/"
+		 to="https://$1appspot.com/" />
+
+</ruleset>

searx/https_rules/Googleplex.com.xml

@@ -0,0 +1,16 @@
+<!-- This rule was automatically generated based on an HSTS
+     preload rule in the Chromium browser.  See 
+     https://src.chromium.org/viewvc/chrome/trunk/src/net/base/transport_security_state.cc
+     for the list of preloads.  Sites are added to the Chromium HSTS
+     preload list on request from their administrators, so HTTPS should
+     work properly everywhere on this site.
+ 
+     Because Chromium and derived browsers automatically force HTTPS for
+     every access to this site, this rule applies only to Firefox. -->
+<ruleset name="Googleplex.com (default off)" platform="firefox" default_off="Certificate error">
+  <target host="googleplex.com" />
+
+  <securecookie host="^googleplex\.com$" name=".+" />
+
+  <rule from="^http://googleplex\.com/" to="https://googleplex.com/" />
+</ruleset>

searx/https_rules/OpenStreetMap.xml

@@ -0,0 +1,15 @@
+<ruleset name="OpenStreetMap">
+
+	<target host="openstreetmap.org"/>
+	<target host="*.openstreetmap.org"/>
+
+	<rule from="^http://(?:www\.)?openstreetmap\.org/"
+		to="https://www.openstreetmap.org/"/>
+
+	<rule from="^http://tile\.openstreetmap\.org/"
+		to="https://a.tile.openstreetmap.org/"/>
+
+	<rule from="^http://(blog|help|lists|nominatim|piwik|taginfo|[abc]\.tile|trac|wiki)\.openstreetmap\.org/"
+		to="https://$1.openstreetmap.org/"/>
+
+</ruleset>

searx/https_rules/Rawgithub.com.xml

@@ -0,0 +1,14 @@
+<!--
+	www: cert only matches ^rawgithub.com
+
+-->
+<ruleset name="rawgithub.com">
+
+	<target host="rawgithub.com" />
+	<target host="www.rawgithub.com" />
+
+
+	<rule from="^http://(?:www\.)?rawgithub\.com/"
+		to="https://rawgithub.com/" />
+
+</ruleset>

searx/https_rules/Soundcloud.xml

@@ -0,0 +1,101 @@
+<!--
+
+	CDN buckets:
+
+		- akmedia-a.akamaihd.net
+
+		- soundcloud.assistly.com
+
+			- help.soundcloud.com
+
+		- cs70.wac.edgecastcdn.net
+
+			- a1.sndcdn.com
+			- i1.sndcdn.com
+			- w1.sndcdn.com
+
+		- wpc.658D.edgecastcdn.net
+		- m-a.sndcdn.com.edgesuite.net 
+		- soundcloud.gettyimages.com
+
+		- scbackstage.wpengine.netdna-cdn.com
+
+			- ssl doesn't exist
+			- backstage.soundcloud.com
+
+		- soundcloud.wpengine.netdna-cdn.com
+
+			- -ssl doesn't exist
+			- blog.soundcloud.com
+
+		- gs1.wpc.v2cdn.netcdn.net
+		- gs1.wpc.v2cdn.net
+
+			- ec-media.soundcloud.com
+
+	Nonfunctional soundcloud.com subdomains:
+
+		- help		(redirects to http, mismatched, CN: *.assistly.com)
+		- m		(redirects to http)
+		- media
+		- status	(times out)
+
+
+	Problematic domains:
+
+		- m-a.sndcdn.com	(works, akamai)
+
+
+	Partially covered domains:
+
+		- backstage.soundcloud.com
+
+
+	Fully covered domains:
+
+		- sndcdn.com subdomains:
+
+			- a[12]
+			- api
+			- i[1-4]
+			- w[12]
+			- wis
+
+		- soundcloud.com subdomains:
+
+			- (www.)
+			- api
+			- blog
+			- connect
+			- developers
+			- ec-media
+			- eventlogger
+			- help-assets
+			- media
+			- visuals
+			- w
+
+-->
+<ruleset name="Soundcloud (partial)">
+
+	<target host="scbackstage.wpengine.netdna-cdn.com" />
+	<target host="soundcloud.wpengine.netdna-cdn.com" />
+	<target host="*.sndcdn.com" />
+	<target host="soundcloud.com" />
+	<target host="*.soundcloud.com" />
+		<exclusion pattern="^https?://(?:scbackstage\.wpengine\.netdna-cdn|backstage\.soundcloud)\.com/(?!wp-content/)" />
+
+
+	<rule from="^http://([aiw]\d|api|wis)\.sndcdn\.com/"
+		to="https://$1.sndcdn.com/" />
+
+	<rule from="^http://((?:api|backstage|blog|connect|developers|ec-media|eventlogger|help-assets|media|visuals|w|www)\.)?soundcloud\.com/"
+		to="https://$1soundcloud.com/" />
+
+	<rule from="^https?://scbackstage\.wpengine\.netdna-cdn\.com/"
+		to="https://backstage.soundcloud.com/" />
+
+	<rule from="^https?://soundcloud\.wpengine\.netdna-cdn\.com/"
+		to="https://blog.soundcloud.com/" />
+
+</ruleset>

searx/https_rules/ThePirateBay.xml

@@ -0,0 +1,36 @@
+<!--
+  Nonfunctional:
+
+    - image.bayimg.com
+    - (www.)thepiratebay.sx		(http reply)
+
+
+  For problematic rules, see ThePirateBay-mismatches.xml.
+
+-->
+<ruleset name="The Pirate Bay (partial)">
+
+  <target host="suprbay.org" />
+  <target host="*.suprbay.org" />
+  <!--	* for cross-domain cookie	-->
+  <target host="*.forum.suprbay.org" />
+  <target host="thepiratebay.org"/>
+  <target host="*.thepiratebay.org"/>
+  <target host="thepiratebay.se"/>
+  <target host="*.thepiratebay.se"/>
+
+  <securecookie host="^.*\.suprbay\.org$" name=".*" />
+  <securecookie host="^(.*\.)?thepiratebay\.se$" name=".*"/>
+
+
+  <!--	Cert doesn't match (www.), redirects like so.	-->
+  <rule from="^https?://(?:forum\.|www\.)?suprbay\.org/"
+    to="https://forum.suprbay.org/" />
+
+  <rule from="^http://(?:www\.)?thepiratebay\.(?:org|se)/"
+    to="https://thepiratebay.se/"/>
+
+  <rule from="^http://(rss|static|torrents)\.thepiratebay\.(?:org|se)/"
+    to="https://$1.thepiratebay.se/"/>
+
+</ruleset>

searx/https_rules/Torproject.xml

@@ -0,0 +1,18 @@
+<ruleset name="Tor Project">
+
+	<target host="torproject.org" />
+	<target host="*.torproject.org" />
+		<exclusion pattern="^http://torperf\.torproject\.org/" />
+
+
+	<!--	Not secured by server:
+					-->
+	<!--securecookie host="^\.blog\.torproject\.org$" name="^SESS[0-9a-f]{32}$" /-->
+
+	<securecookie host="^(?:.*\.)?torproject\.org$" name=".+" />
+
+
+	<rule from="^http://([^/:@\.]+\.)?torproject\.org/"
+		 to="https://$1torproject.org/" />
+
+</ruleset>

searx/https_rules/Twitter.xml

@@ -0,0 +1,169 @@
+<!--
+	Other Twitter rulesets:
+
+		- Twitter_Community.com.xml
+
+
+	Nonfunctional domains:
+
+		- status.twitter.com *
+		- status.twitter.jp *
+
+	* Tumblr
+
+
+	CDN buckets:
+
+		- a1095.g.akamai.net/=/1095/134446/1d/platform.twitter.com/ | platform2.twitter.com.edgesuite.net
+
+			- platform2.twitter.com
+
+		- twitter-any.s3.amazonaws.com
+		- twitter-blog.s3.amazonaws.com
+
+		- d2rdfnizen5apl.cloudfront.net
+
+			- s.twimg.com
+
+		- ssl2.twitter.com.edgekey.net
+		- twitter.github.com
+
+
+	Problematic domains:
+
+		- twimg.com subdomains:
+
+			- a5 *
+			- s		(cloudfront)
+
+		- twitter.com subdomains:
+
+			- platform[0-3]		(403, akamai)
+
+	* akamai
+
+
+	Fully covered domains:
+
+		- (www.)t.co		(www → ^)
+
+		- twimg.com subdomains:
+
+			- a[5-9]	(→ si0)
+			- a\d
+			- abs
+			- dnt
+			- ea
+			- g
+			- g2
+			- gu
+			- hca
+			- jp
+			- ma
+			- ma[0123]
+			- o
+			- p
+			- pbs
+			- r
+			- s		(→ d2rdfnizen5apl.cloudfront.net)
+			- si[0-5]
+			- syndication
+			- cdn.syndication
+			- tailfeather
+			- ton
+			- v
+			- widgets
+
+		- twitter.com subdomains:
+
+			- (www.)
+			- 201[012]
+			- about
+			- ads
+			- analytics
+			- api
+			- cdn.api
+			- urls.api
+			- blog
+			- business
+			- preview.cdn
+			- preview-dev.cdn
+			- preview-stage.cdn
+			- de
+			- dev
+			- en
+			- engineering
+			- es
+			- firefox
+			- fr
+			- it
+			- ja
+			- jp
+			- m
+			- media
+			- mobile
+			- music
+			- oauth
+			- p
+			- pic
+			- platform
+			- platform[0-3]		(→ platform)
+			- widgets.platform
+			- search
+			- static
+			- support
+			- transparency
+			- upload
+
+
+	These altnames don't exist:
+
+		- i3.twimg.com
+		- p-dev.twimg.com
+		- vmtc.twimg.com
+
+		- cdn-dev.api.twitter.com
+
+-->
+<ruleset name="Twitter">
+
+	<target host="t.co" />
+	<target host="*.t.co" />
+	<target host="*.twimg.com" />
+	<target host="twitter.com" />
+	<target host="*.twitter.com" />
+
+
+	<!--	Secured by server:
+					-->
+	<!--securecookie host="^\.twitter\.com$" name="^_twitter_sess$" /-->
+	<!--securecookie host="^support\.twitter\.com$" name="^_help_center_session$" /-->
+	<!--
+		Not secured by server:
+					-->
+	<!--securecookie host="^\.t\.co$" name="^muc$" /-->
+	<!--securecookie host="^\.twitter\.com$" name="^guest_id$" /-->
+
+	<securecookie host="^\.t\.co$" name=".+" />
+	<securecookie host="^(?:.*\.)?twitter\.com$" name=".+" />
+
+
+	<rule from="^http://(?:www\.)?t\.co/"
+		to="https://t.co/" />
+
+	<rule from="^http://a[5-9]\.twimg\.com/"
+		to="https://si0.twimg.com/" />
+
+	<rule from="^http://(abs|a\d|dnt|ea|g[2u]?|hca|jp|ma\d?|o|p|pbs|r|si\d|(?:cdn\.)?syndication|tailfeather|ton|v|widgets)\.twimg\.com/"
+		to="https://$1.twimg.com/" />
+
+	<rule from="^http://s\.twimg\.com/"
+		to="https://d2rdfnizen5apl.cloudfront.net/" />
+
+	<rule from="^http://((?:201\d|about|ads|analytics|blog|(?:cdn\.|urls\.)?api|business|preview(?:-dev|-stage)?\.cdn|de|dev|engineering|en|es|firefox|fr|it|ja|jp|m|media|mobile|music|oauth|p|pic|platform|widgets\.platform|search|static|support|transparency|upload|www)\.)?twitter\.com/"
+		to="https://$1twitter.com/" />
+
+	<rule from="^http://platform\d\.twitter\.com/"
+		to="https://platform.twitter.com/" />
+
+</ruleset>

searx/https_rules/Vimeo.xml

@@ -0,0 +1,75 @@
+<!--
+	CDN buckets:
+
+		- av.vimeo.com.edgesuite.net
+
+			- a808.g.akamai.net
+
+		- pdl.vimeocdn.com.edgesuite.net
+
+			- a1189.g.akamai.net
+
+
+	Problematic subdomains:
+
+		- av	(pdl.../crossdomain.xml restricts to port 80)
+		- pdl	(works, akamai)
+
+
+	Partially covered subdomains:
+
+		- developer	(some pages redirect to http)
+		- pdl		(→ akamai)
+
+
+	Fully covered subdomains:
+
+		- (www.)
+		- secure
+
+
+Default off per https://trac.torproject.org/projects/tor/ticket/7569 -->
+<ruleset name="Vimeo (default off)" default_off="breaks some video embedding">
+
+	<target host="vimeo.com" />
+	<target host="*.vimeo.com" />
+		<exclusion pattern="^http://av\.vimeo\.com/crossdomain\.xml" />
+		<!--exclusion pattern="^http://developer\.vimeo\.com/($|\?|(apps|guidelines|help|player)($|[?/]))" /-->
+		<exclusion pattern="^http://developer\.vimeo\.com/(?!apis(?:$|[?/])|favicon\.ico)" />
+	<target host="*.vimeocdn.com" />
+		<!--
+			Uses crossdomain.xml from s3.amazonaws.com, which sets secure="false"
+
+				https://mail1.eff.org/pipermail/https-everywhere/2012-October/001583.html
+			-->
+		<exclusion pattern="^http://a\.vimeocdn\.com/p/flash/moogaloop/" />
+
+		<!--	We cannot secure streams because crossdomain.xml
+			restricts to port 80 :(
+						-->
+		<exclusion pattern="^http://pdl\.vimeocdn\.com/(?!crossdomain\.xml)" />
+
+
+	<!--	Tracking cookies:
+					-->
+	<securecookie host="^\.(?:player\.)?vimeo\.com$" name="^__utm\w$" />
+
+
+	<rule from="^http://((?:developer|player|secure|www)\.)?vimeo\.com/"
+		to="https://$1vimeo.com/" />
+
+	<rule from="^http://av\.vimeo\.com/"
+		to="https://a248.e.akamai.net/f/808/9207/8m/av.vimeo.com/" />
+
+	<!--	a & b: Akamai	-->
+	<rule from="^http://(?:secure-)?([ab])\.vimeocdn\.com/"
+		to="https://secure-$1.vimeocdn.com/" />
+
+	<rule from="^http://i\.vimeocdn\.com/"
+		to="https://i.vimeocdn.com/" />
+
+	<rule from="^http://pdl\.vimeocdn\.com/"
+		to="https://a248.e.akamai.net/f/1189/4415/8d/pdl.vimeocdn.com/" />
+
+</ruleset>
+

searx/https_rules/WikiLeaks.xml

@@ -0,0 +1,13 @@
+<ruleset name="WikiLeaks">
+
+	<target host="wikileaks.org" />
+	<target host="*.wikileaks.org" />
+
+
+	<securecookie host="^(?:w*\.)?wikileaks\.org$" name=".+" />
+
+
+	<rule from="^http://((?:chat|search|shop|www)\.)?wikileaks\.org/"
+		to="https://$1wikileaks.org/" />
+
+</ruleset>

searx/https_rules/Wikimedia.xml

@@ -0,0 +1,107 @@
+<!--
+	Wikipedia and other Wikimedia Foundation wikis previously had no real HTTPS support, and
+	URLs had to be rewritten to https://secure.wikimedia.org/$wikitype/$language/ . This is no
+	longer the case, see https://blog.wikimedia.org/2011/10/03/native-https-support-enabled-for-all-wikimedia-foundation-wikis/ ,
+	so this file is a lot simpler these days.
+
+
+	Mixed content:
+
+		- Images, on:
+
+			- stats.wikimedia.org from upload.wikimedia.org *
+			- stats.wikimedia.org from wikimediafoundation.org *
+
+	* Secured by us
+
+-->
+<ruleset name="Wikimedia">
+
+	<target host="enwp.org" />
+	<target host="frwp.org" />
+
+	<target host="mediawiki.org" />
+	<target host="www.mediawiki.org" />
+	<target host="wikimedia.org" />
+	<target host="*.wikimedia.org" />
+		<exclusion pattern="^http://(?:apt|cs|cz|parsoid-lb\.eqiad|status|torrus|ubuntu)\.wikimedia\.org" />
+		<!-- https://mail1.eff.org/pipermail/https-everywhere-rules/2012-June/001189.html -->
+		<exclusion pattern="^http://lists\.wikimedia\.org/pipermail(?:$|/)" />
+	<target host="wikimediafoundation.org" />
+	<target host="www.wikimediafoundation.org" />
+
+	<!-- Wikimedia projects (also some wikimedia.org subdomains) -->
+	<target host="wikibooks.org" />
+	<target host="*.wikibooks.org" />
+	<target host="wikidata.org" />
+	<target host="*.wikidata.org" />
+	<target host="wikinews.org" />
+	<target host="*.wikinews.org" />
+	<target host="wikipedia.org" />
+	<target host="*.wikipedia.org" />
+	<target host="wikiquote.org" />
+	<target host="*.wikiquote.org" />
+	<target host="wikisource.org" />
+	<target host="*.wikisource.org" />
+	<target host="wikiversity.org" />
+	<target host="*.wikiversity.org" />
+	<target host="wikivoyage.org" />
+	<target host="*.wikivoyage.org" />
+	<target host="wiktionary.org" />
+	<target host="*.wiktionary.org" />
+
+	<!-- Wikimedia chapters -->
+	<target host="wikimedia.ca" />
+	<target host="www.wikimedia.ca" />
+
+	<!-- Wikimedia Tool Labs -->
+	<target host="tools.wmflabs.org" />
+	<target host="icinga.wmflabs.org" />
+	<target host="ganglia.wmflabs.org" />
+
+	<!--	Not secured by server:
+					-->
+	<!--securecookie host="^\.wiki(books|ipedia)\.org$" name="^GeoIP$" /-->
+
+	<securecookie host="^^\.wik(?:ibooks|idata|imedia|inews|ipedia|iquote|isource|iversity|ivoyage|tionary)\.org$" name="^GeoIP$" />
+	<securecookie host="^([^@:/]+\.)?wik(ibooks|idata|inews|ipedia|iquote|isource|iversity|ivoyage|tionary)\.org$" name=".*" />
+	<securecookie host="^(species|commons|meta|incubator|wikitech).wikimedia.org$" name=".*" />
+	<securecookie host="^(?:www\.)?mediawiki\.org$" name=".*" />
+	<securecookie host="^wikimediafoundation.org$" name=".*" />
+
+	<rule from="^http://(en|fr)wp\.org/"
+		to="https://$1.wikipedia.org/wiki/" />
+
+	<rule from="^http://(?:www\.)?mediawiki\.org/"
+		to="https://www.mediawiki.org/" />
+
+	<rule from="^https?://download\.wikipedia\.org/"
+		to="https://dumps.wikimedia.org/" />
+
+	<rule from="^https?://(download|dataset2|sitemap)\.wikimedia\.org/"
+		to="https://dumps.wikimedia.org/" />
+
+	<rule from="^https?://(labs-ns[01]|virt0)\.wikimedia\.org/"
+		to="https://wikitech.wikimedia.org/" />	
+
+	<rule from="^https?://noboard\.chapters\.wikimedia\.org/"
+		to="https://noboard-chapters.wikimedia.org/" />
+
+	<rule from="^https?://wg\.en\.wikipedia\.org/"
+		to="https://wg-en.wikipedia.org/" />
+
+	<rule from="^https?://arbcom\.(de|en|fi|nl)\.wikipedia\.org/"
+		to="https://arbcom-$1.wikipedia.org/" />
+
+	<rule from="^http://([^@:/]+\.)?wik(ibooks|idata|imedia|inews|ipedia|iquote|isource|iversity|ivoyage|tionary)\.org/"
+		to="https://$1wik$2.org/" />
+
+	<rule from="^http://(www\.)?wikimediafoundation\.org/"
+		to="https://$1wikimediafoundation.org/" />
+
+	<rule from="^http://(www\.)?wikimedia\.ca/"
+		to="https://wikimedia.ca/" />
+
+	<rule from="^http://([^@:/]+)\.wmflabs\.org/"
+		to="https://$1.wmflabs.org/" />
+</ruleset>

searx/https_rules/YouTube.xml

@@ -0,0 +1,46 @@
+<ruleset name="YouTube (partial)">
+
+	<target host="youtube.com" />
+	<target host="*.youtube.com" />
+		<exclusion pattern="^http://(?:www\.)?youtube\.com/crossdomain\.xml"/>
+		<exclusion pattern="^http://(?:www\.)?youtube\.com/(?:apiplayer|api_video_info)"/>
+        <exclusion pattern="^http://(?:[^/@:\.]+\.)?ytimg\.com/.*apiplayer[0-9]*\.swf"/>
+	<target host="*.ytimg.com" />
+	<target host="youtu.be" />
+	<target host="youtube-nocookie.com"/>
+	<target host="www.youtube-nocookie.com"/>
+	<target host="*.googlevideo.com"/>
+                <exclusion pattern="^http://([^/@:\.]+)\.googlevideo\.com/crossdomain\.xml"/>
+
+
+	<!--	Not secured by server:
+					-->
+	<!--securecookie host="^\.youtube\.com$" name="^(GEUP|PREF|VISITOR_INFO1_LIVE|YSC)$" /-->
+
+	<!--	observed ^. cookies:
+			- use_hitbox
+			- VISITOR_INFO1_LIVE
+			- recently_watched_video_id_list
+			- .youtube.com		-->
+	<securecookie host="^\.youtube\.com" name=".*"/>
+
+
+	<rule from="^http://(www\.)?youtube\.com/"
+		to="https://$1youtube.com/"/>
+
+	<rule from="^http://(br|de|es|fr|il|img|insight|jp|m|nl|uk)\.youtube\.com/"
+		to="https://$1.youtube.com/"/>
+
+	<rule from="^http://([^/@:\.]+)\.ytimg\.com/"
+		to="https://$1.ytimg.com/"/>
+
+	<rule from="^http://youtu\.be/"
+		to="https://youtu.be/"/>
+
+	<rule from="^http://(?:www\.)?youtube-nocookie\.com/"
+		to="https://www.youtube-nocookie.com/"/>
+
+	<rule from="^http://([^/@:\.]+)\.googlevideo\.com/"
+	        to="https://$1.googlevideo.com/"/>
+
+</ruleset>

searx/query.py

@@ -39,7 +39,8 @@ class Query(object):
         self.engines = []
         self.languages = []
 
-    # parse query, if tags are set, which change the serch engine or search-language
+    # parse query, if tags are set, which
+    # change the serch engine or search-language
     def parse_query(self):
         self.query_parts = []
 
@@ -66,11 +67,13 @@ class Query(object):
             if query_part[0] == ':':
                 lang = query_part[1:].lower()
 
-                # check if any language-code is equal with declared language-codes
+                # check if any language-code is equal with
+                # declared language-codes
                 for lc in language_codes:
                     lang_id, lang_name, country = map(str.lower, lc)
 
-                    # if correct language-code is found, set it as new search-language
+                    # if correct language-code is found
+                    # set it as new search-language
                     if lang == lang_id\
                        or lang_id.startswith(lang)\
                        or lang == lang_name\
@@ -92,19 +95,20 @@ class Query(object):
 
                 # check if prefix is equal with engine name
                 elif prefix in engines\
-                        and not prefix in self.blocked_engines:
+                        and prefix not in self.blocked_engines:
                     parse_next = True
                     self.engines.append({'category': 'none',
                                         'name': prefix})
 
                 # check if prefix is equal with categorie name
                 elif prefix in categories:
-                    # using all engines for that search, which are declared under that categorie name
+                    # using all engines for that search, which
+                    # are declared under that categorie name
                     parse_next = True
                     self.engines.extend({'category': prefix,
                                         'name': engine.name}
                                         for engine in categories[prefix]
-                                        if not engine in self.blocked_engines)
+                                        if engine not in self.blocked_engines)
 
             # append query part to query_part list
             self.query_parts.append(query_part)
@@ -124,4 +128,3 @@ class Query(object):
     def getFullQuery(self):
         # get full querry including whitespaces
         return string.join(self.query_parts, '')
-

searx/search.py

@@ -22,7 +22,7 @@ from datetime import datetime
 from operator import itemgetter
 from urlparse import urlparse, unquote
 from searx.engines import (
-    categories, engines, engine_shortcuts
+    categories, engines
 )
 from searx.languages import language_codes
 from searx.utils import gen_useragent
@@ -39,7 +39,13 @@ def default_request_params():
 
 
 # create a callback wrapper for the search engine results
-def make_callback(engine_name, results, suggestions, answers, infoboxes, callback, params):
+def make_callback(engine_name,
+                  results,
+                  suggestions,
+                  answers,
+                  infoboxes,
+                  callback,
+                  params):
 
     # creating a callback wrapper for the search engine results
     def process_callback(response, **kwargs):
@@ -126,7 +132,8 @@ def score_results(results):
 
         # strip multiple spaces and cariage returns from content
         if 'content' in res:
-            res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))
+            res['content'] = re.sub(' +', ' ',
+                                    res['content'].strip().replace('\n', ''))
 
         # get weight of this engine if possible
         if hasattr(engines[res['engine']], 'weight'):
@@ -139,8 +146,12 @@ def score_results(results):
         duplicated = False
         for new_res in results:
             # remove / from the end of the url if required
-            p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path  # noqa
+            p1 = res['parsed_url'].path[:-1]\
-            p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path  # noqa
+                if res['parsed_url'].path.endswith('/')\
+                else res['parsed_url'].path
+            p2 = new_res['parsed_url'].path[:-1]\
+                if new_res['parsed_url'].path.endswith('/')\
+                else new_res['parsed_url'].path
 
             # check if that result is a duplicate
             if res['host'] == new_res['host'] and\
@@ -153,7 +164,8 @@ def score_results(results):
         # merge duplicates together
         if duplicated:
             # using content with more text
-            if content_result_len(res.get('content', '')) > content_result_len(duplicated.get('content', '')):
+            if content_result_len(res.get('content', '')) >\
+                    content_result_len(duplicated.get('content', '')):
                 duplicated['content'] = res['content']
 
             # increase result-score
@@ -182,17 +194,25 @@ def score_results(results):
 
     for i, res in enumerate(results):
         # FIXME : handle more than one category per engine
-        category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']
+        category = engines[res['engine']].categories[0] + ':' + ''\
+            if 'template' not in res\
+            else res['template']
 
-        current = None if category not in categoryPositions else categoryPositions[category]
+        current = None if category not in categoryPositions\
+            else categoryPositions[category]
 
-        # group with previous results using the same category if the group can accept more result and is not too far from the current position
+        # group with previous results using the same category
-        if current != None and (current['count'] > 0) and (len(gresults) - current['index'] < 20):
+        # if the group can accept more result and is not too far
-            # group with the previous results using the same category with this one
+        # from the current position
+        if current is not None and (current['count'] > 0)\
+                and (len(gresults) - current['index'] < 20):
+            # group with the previous results using
+            # the same category with this one
             index = current['index']
             gresults.insert(index, res)
 
-            # update every index after the current one (including the current one)
+            # update every index after the current one
+            # (including the current one)
             for k in categoryPositions:
                 v = categoryPositions[k]['index']
                 if v >= index:
@@ -206,7 +226,7 @@ def score_results(results):
             gresults.append(res)
 
             # update categoryIndex
-            categoryPositions[category] = { 'index' : len(gresults), 'count' : 8 }
+            categoryPositions[category] = {'index': len(gresults), 'count': 8}
 
     # return gresults
     return gresults
@@ -215,7 +235,7 @@ def score_results(results):
 def merge_two_infoboxes(infobox1, infobox2):
     if 'urls' in infobox2:
         urls1 = infobox1.get('urls', None)
-        if urls1 == None:
+        if urls1 is None:
             urls1 = []
             infobox1.set('urls', urls1)
 
@@ -229,7 +249,7 @@ def merge_two_infoboxes(infobox1, infobox2):
 
     if 'attributes' in infobox2:
         attributes1 = infobox1.get('attributes', None)
-        if attributes1 == None:
+        if attributes1 is None:
             attributes1 = []
             infobox1.set('attributes', attributes1)
 
@@ -244,7 +264,7 @@ def merge_two_infoboxes(infobox1, infobox2):
     if 'content' in infobox2:
         content1 = infobox1.get('content', None)
         content2 = infobox2.get('content', '')
-        if content1 != None:
+        if content1 is not None:
             if content_result_len(content2) > content_result_len(content1):
                 infobox1['content'] = content2
         else:
@@ -257,11 +277,11 @@ def merge_infoboxes(infoboxes):
     for infobox in infoboxes:
         add_infobox = True
         infobox_id = infobox.get('id', None)
-        if infobox_id != None:
+        if infobox_id is not None:
             existingIndex = infoboxes_id.get(infobox_id, None)
-            if existingIndex != None:
+            if existingIndex is not None:
                 merge_two_infoboxes(results[existingIndex], infobox)
-                add_infobox=False
+                add_infobox = False
 
         if add_infobox:
             results.append(infobox)
@@ -311,9 +331,6 @@ class Search(object):
         if not self.request_data.get('q'):
             raise Exception('noquery')
 
-        # set query
-        self.query = self.request_data['q']
-
         # set pagenumber
         pageno_param = self.request_data.get('pageno', '1')
         if not pageno_param.isdigit() or int(pageno_param) < 1:
@@ -321,10 +338,14 @@ class Search(object):
 
         self.pageno = int(pageno_param)
 
-        # parse query, if tags are set, which change the serch engine or search-language
+        # parse query, if tags are set, which change
-        query_obj = Query(self.query, self.blocked_engines)
+        # the serch engine or search-language
+        query_obj = Query(self.request_data['q'], self.blocked_engines)
         query_obj.parse_query()
 
+        # set query
+        self.query = query_obj.getSearchQuery()
+
         # get last selected language in query, if possible
         # TODO support search with multible languages
         if len(query_obj.languages):
@@ -334,25 +355,29 @@ class Search(object):
 
         self.categories = []
 
-        # if engines are calculated from query, set categories by using that informations
+        # if engines are calculated from query,
+        # set categories by using that informations
         if self.engines:
             self.categories = list(set(engine['category']
                                        for engine in self.engines))
 
-        # otherwise, using defined categories to calculate which engines should be used
+        # otherwise, using defined categories to
+        # calculate which engines should be used
         else:
             # set used categories
             for pd_name, pd in self.request_data.items():
                 if pd_name.startswith('category_'):
                     category = pd_name[9:]
                     # if category is not found in list, skip
-                    if not category in categories:
+                    if category not in categories:
                         continue
 
                     # add category to list
                     self.categories.append(category)
 
-            # if no category is specified for this search, using user-defined default-configuration which (is stored in cookie)
+            # if no category is specified for this search,
+            # using user-defined default-configuration which
+            # (is stored in cookie)
             if not self.categories:
                 cookie_categories = request.cookies.get('categories', '')
                 cookie_categories = cookie_categories.split(',')
@@ -360,16 +385,18 @@ class Search(object):
                     if ccateg in categories:
                         self.categories.append(ccateg)
 
-            # if still no category is specified, using general as default-category
+            # if still no category is specified, using general
+            # as default-category
             if not self.categories:
                 self.categories = ['general']
 
-            # using all engines for that search, which are declared under the specific categories
+            # using all engines for that search, which are
+            # declared under the specific categories
             for categ in self.categories:
                 self.engines.extend({'category': categ,
                                      'name': x.name}
                                     for x in categories[categ]
-                                    if not x.name in self.blocked_engines)
+                                    if x.name not in self.blocked_engines)
 
     # do search-request
     def search(self, request):
@@ -386,7 +413,7 @@ class Search(object):
         number_of_searches += 1
 
         # set default useragent
-        #user_agent = request.headers.get('User-Agent', '')
+        # user_agent = request.headers.get('User-Agent', '')
         user_agent = gen_useragent()
 
         # start search-reqest for all selected engines
@@ -400,7 +427,8 @@ class Search(object):
             if self.pageno > 1 and not engine.paging:
                 continue
 
-            # if search-language is set and engine does not provide language-support, skip
+            # if search-language is set and engine does not
+            # provide language-support, skip
             if self.lang != 'all' and not engine.language_support:
                 continue
 
@@ -412,7 +440,8 @@ class Search(object):
             request_params['pageno'] = self.pageno
             request_params['language'] = self.lang
 
-            # update request parameters dependent on search-engine (contained in engines folder)
+            # update request parameters dependent on
+            # search-engine (contained in engines folder)
             request_params = engine.request(self.query.encode('utf-8'),
                                             request_params)
 
@@ -431,7 +460,8 @@ class Search(object):
                 request_params
             )
 
-            # create dictionary which contain all informations about the request
+            # create dictionary which contain all
+            # informations about the request
             request_args = dict(
                 headers=request_params['headers'],
                 hooks=dict(response=callback),

searx/settings.yml

@@ -52,6 +52,12 @@ engines:
     engine : duckduckgo
     shortcut : ddg
 
+# api-key required: http://www.faroo.com/hp/api/api.html#key
+#  - name : faroo
+#    engine : faroo
+#    shortcut : fa
+#    api_key : 'apikey' # required!
+
 # down - website is under criminal investigation by the UK
 #  - name : filecrop
 #    engine : filecrop
@@ -166,3 +172,4 @@ locales:
     es : Español
     it : Italiano
     nl : Nederlands
+    ja : 日本語 (Japanese)

searx/settings_robot.yml

@@ -4,6 +4,9 @@ server:
     debug : False
     request_timeout : 3.0 # seconds
     base_url: False
+    themes_path : ""
+    default_theme : default
+    https_rewrite : True
 
 engines:
   - name : general_dummy

searx/static/default/css/style.css

@@ -77,5 +77,5 @@ tr:hover{background:#ddd}
 #preferences{top:10px;padding:0;border:0;background:url('../img/preference-icon.png') no-repeat;background-size:28px 28px;opacity:.8;width:28px;height:30px;display:block}#preferences *{display:none}
 #pagination{clear:both;width:40em}
 #apis{margin-top:8px;clear:both}
-@media screen and (max-width:50em){#results{margin:auto;padding:0;width:90%} .github{display:none} .checkbox_container{display:block;width:90%}.checkbox_container label{border-bottom:0}}@media screen and (max-width:75em){#infoboxes{position:inherit;max-width:inherit}#infoboxes .infobox{clear:both}#infoboxes .infobox img{float:left;max-width:10em} #categories{font-size:90%;clear:both}#categories .checkbox_container{margin-top:2px;margin:auto} .right{display:none;postion:fixed !important;top:100px;right:0} #sidebar{position:static;max-width:50em;margin:0 0 2px 0;padding:0;float:none;border:none;width:auto}#sidebar input{border:0} #apis{display:none} #search_url{display:none} .result{border-top:1px solid #e8e7e6;margin:7px 0 6px 0}}.favicon{float:left;margin-right:4px;margin-top:2px}
+@media screen and (max-width:50em){#results{margin:auto;padding:0;width:90%} .github{display:none} .checkbox_container{display:block;width:90%}.checkbox_container label{border-bottom:0} .right{display:none;postion:fixed !important;top:100px;right:0}}@media screen and (max-width:75em){#infoboxes{position:inherit;max-width:inherit}#infoboxes .infobox{clear:both}#infoboxes .infobox img{float:left;max-width:10em} #categories{font-size:90%;clear:both}#categories .checkbox_container{margin-top:2px;margin:auto} #sidebar{position:static;max-width:50em;margin:0 0 2px 0;padding:0;float:none;border:none;width:auto}#sidebar input{border:0} #apis{display:none} #search_url{display:none} .result{border-top:1px solid #e8e7e6;margin:7px 0 6px 0}}.favicon{float:left;margin-right:4px;margin-top:2px}
 .preferences_back{background:none repeat scroll 0 0 #3498db;border:0 none;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;cursor:pointer;display:inline-block;margin:2px 4px;padding:4px 6px}.preferences_back a{color:#fff}

searx/static/default/less/style.less

@@ -529,6 +529,14 @@ tr {
 			border-bottom: 0;
 		}
 	}
+
+	.right {
+		display: none;
+		postion: fixed !important;
+		top: 100px;
+		right: 0px;
+	}
+
 }
 
 @media screen and (max-width: 75em) {
@@ -558,13 +566,6 @@ tr {
 		}
 	}
 
-	.right {
-		display: none;
-		postion: fixed !important;
-		top: 100px;
-		right: 0px;
-	}
-
 	#sidebar {
 		position: static;
 		max-width: @results-width;

searx/translations/de/LC_MESSAGES/messages.po

@@ -5,11 +5,12 @@
 # Translators:
 # pointhi, 2014
 # stf <stefan.marsiske@gmail.com>, 2014
+# rike, 2014
 msgid ""
 msgstr ""
 "Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
 "PO-Revision-Date: 2014-03-15 18:40+0000\n"
 "Last-Translator: pointhi\n"
 "Language-Team: German "
@@ -20,31 +21,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
-msgstr ""
+msgstr "vor {minutes} Minute(n)"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
-msgstr ""
+msgstr "vor {hours} Stunde(n), {minutes} Minute(n)"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr "Ladezeit (sek)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Trefferanzahl"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Punkte"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Punkte pro Treffer"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Fehler"
 
@@ -69,7 +70,7 @@ msgstr "Einstellungen"
 #: searx/templates/default/preferences.html:9
 #: searx/templates/oscar/preferences.html:21
 msgid "Default categories"
-msgstr "Standard Kategorien"
+msgstr "Standardkategorien"
 
 #: searx/templates/courgette/preferences.html:15
 #: searx/templates/default/preferences.html:15
@@ -93,19 +94,19 @@ msgstr "Oberflächensprache"
 #: searx/templates/default/preferences.html:36
 #: searx/templates/oscar/preferences.html:50
 msgid "Autocomplete"
-msgstr ""
+msgstr "Autovervollständigung"
 
 #: searx/templates/courgette/preferences.html:47
 #: searx/templates/default/preferences.html:47
 #: searx/templates/oscar/preferences.html:63
 msgid "Method"
-msgstr ""
+msgstr "Methode"
 
 #: searx/templates/courgette/preferences.html:56
 #: searx/templates/default/preferences.html:56
 #: searx/templates/oscar/preferences.html:73
 msgid "Themes"
-msgstr ""
+msgstr "Designs"
 
 #: searx/templates/courgette/preferences.html:66
 #: searx/templates/default/preferences.html:66
@@ -145,8 +146,8 @@ msgid ""
 "These settings are stored in your cookies, this allows us not to store "
 "this data about you."
 msgstr ""
-"Diese Informationen werden in Cookies gespeichert, damit wir keine ihrer "
+"Diese Informationen werden in Cookies auf Ihrem Rechner gespeichert, "
-"persönlichen Daten speichern müssen."
+"damit wir keine Ihrer persönlichen Daten speichern müssen."
 
 #: searx/templates/courgette/preferences.html:94
 #: searx/templates/default/preferences.html:94
@@ -155,8 +156,8 @@ msgid ""
 "These cookies serve your sole convenience, we don't use these cookies to "
 "track you."
 msgstr ""
-"Diese Cookies dienen ihrer Gemütlichkeit, wir verwenden sie nicht zum "
+"Diese Cookies dienen einzig Ihrem Komfort, wir verwenden sie nicht, um "
-"überwachen."
+"Sie zu überwachen."
 
 #: searx/templates/courgette/preferences.html:97
 #: searx/templates/default/preferences.html:97
@@ -172,30 +173,30 @@ msgstr "Zurück"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "Such-URL"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Ergebnisse herunterladen"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Vorschläge"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "vorherige Seite"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "nächste Seite"
@@ -209,7 +210,11 @@ msgstr "Suche nach..."
 #: searx/templates/courgette/stats.html:4 searx/templates/default/stats.html:4
 #: searx/templates/oscar/stats.html:5
 msgid "Engine stats"
-msgstr "Suchmaschienen Statistiken"
+msgstr "Suchmaschinenstatistik"
+
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
 
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
@@ -228,14 +233,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
 msgstr "Allgemein"
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Suchmaschienen Statistiken"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -261,11 +264,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Trefferanzahl"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -340,9 +342,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "nächste Seite"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -379,3 +380,6 @@ msgstr "IT"
 msgid "news"
 msgstr "Neuigkeiten"
 
+msgid "map"
+msgstr "Karte"
+

searx/translations/en/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
 "PO-Revision-Date: 2014-01-30 15:22+0100\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: en <LL@li.org>\n"
@@ -17,31 +17,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
 msgstr ""
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
 msgstr ""
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr ""
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr ""
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr ""
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr ""
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr ""
 
@@ -165,30 +165,30 @@ msgstr ""
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr ""
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr ""
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr ""
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr ""
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr ""
@@ -204,6 +204,10 @@ msgstr ""
 msgid "Engine stats"
 msgstr ""
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -255,7 +259,7 @@ msgstr ""
 msgid "Search results"
 msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -350,9 +354,6 @@ msgstr ""
 msgid "files"
 msgstr ""
 
-msgid "general"
-msgstr ""
-
 msgid "music"
 msgstr ""
 
@@ -371,3 +372,6 @@ msgstr ""
 msgid "news"
 msgstr ""
 
+msgid "map"
+msgstr ""
+

searx/translations/es/LC_MESSAGES/messages.po

@@ -3,14 +3,14 @@
 # This file is distributed under the same license as the  project.
 #
 # Translators:
-# niazle, 2014
+# Alejandro León Aznar, 2014
 msgid ""
 msgstr ""
 "Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
-"PO-Revision-Date: 2014-03-04 20:40+0000\n"
+"PO-Revision-Date: 2014-09-08 11:01+0000\n"
-"Last-Translator: niazle\n"
+"Last-Translator: Alejandro León Aznar\n"
 "Language-Team: Spanish "
 "(http://www.transifex.com/projects/p/searx/language/es/)\n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
@@ -19,31 +19,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
-msgstr ""
+msgstr "hace {minutes} minuto(s)"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
-msgstr ""
+msgstr "hace {hours} hora(s) y {minutes} minuto(s)"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr "Tiempo de carga (segundos)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Número de resultados"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Puntuaciones"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Puntuaciones por resultado"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Errores"
 
@@ -171,30 +171,30 @@ msgstr "Atrás"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "Buscar URL"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Descargar resultados"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Sugerencias"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "Página anterior"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "Página siguiente"
@@ -203,13 +203,17 @@ msgstr "Página siguiente"
 #: searx/templates/default/search.html:3 searx/templates/oscar/search.html:4
 #: searx/templates/oscar/search_full.html:5
 msgid "Search for..."
-msgstr ""
+msgstr "Buscar..."
 
 #: searx/templates/courgette/stats.html:4 searx/templates/default/stats.html:4
 #: searx/templates/oscar/stats.html:5
 msgid "Engine stats"
 msgstr "Estadísticas del motor de búsqueda"
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -227,14 +231,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
-msgstr "General"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Estadísticas del motor de búsqueda"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -260,11 +262,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Número de resultados"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -339,9 +340,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "Página siguiente"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -378,3 +378,6 @@ msgstr "TIC"
 msgid "news"
 msgstr "noticias"
 
+msgid "map"
+msgstr "mapa"
+

searx/translations/fr/LC_MESSAGES/messages.po

@@ -5,14 +5,14 @@
 # Translators:
 # Benjamin Sonntag <benjamin@sonntag.fr>, 2014
 # FIRST AUTHOR <EMAIL@ADDRESS>, 2014
-# rike <u@451f.org>, 2014
+# rike, 2014
 msgid ""
 msgstr ""
 "Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
-"PO-Revision-Date: 2014-03-16 07:40+0000\n"
+"PO-Revision-Date: 2014-09-07 21:24+0000\n"
-"Last-Translator: Benjamin Sonntag <benjamin@sonntag.fr>\n"
+"Last-Translator: Adam Tauber <asciimoo@gmail.com>\n"
 "Language-Team: French "
 "(http://www.transifex.com/projects/p/searx/language/fr/)\n"
 "Plural-Forms: nplurals=2; plural=(n > 1)\n"
@@ -21,31 +21,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
-msgstr "Il y a {minutes} minute(s)"
+msgstr "il y a {minutes} minute(s)"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
-msgstr "Il y a {hours} heure(s), {minutes} minute(s)"
+msgstr "il y a {hours} heure(s), {minutes} minute(s)"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr "Chargement de la page (sec)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Nombre de résultats"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Score"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Score par résultat"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Erreurs"
 
@@ -111,7 +111,7 @@ msgstr ""
 #: searx/templates/courgette/preferences.html:66
 #: searx/templates/default/preferences.html:66
 msgid "Currently used search engines"
-msgstr "Moteurs actuellement utilisés"
+msgstr "Moteurs de recherche actuellement utilisés"
 
 #: searx/templates/courgette/preferences.html:70
 #: searx/templates/default/preferences.html:70
@@ -173,30 +173,30 @@ msgstr "retour"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "URL de recherche"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Télécharger les résultats"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Suggestions"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "page précédente"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "page suivante"
@@ -212,6 +212,10 @@ msgstr "Rechercher..."
 msgid "Engine stats"
 msgstr "Statistiques du moteur"
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -229,14 +233,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
-msgstr "général"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Statistiques du moteur"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -262,11 +264,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Nombre de résultats"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -341,9 +342,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "page suivante"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -380,3 +380,6 @@ msgstr "Informatique"
 msgid "news"
 msgstr "actus"
 
+msgid "map"
+msgstr ""
+

searx/translations/hu/LC_MESSAGES/messages.po

@@ -1,47 +1,50 @@
-# Hungarian translations for PROJECT.
+# English translations for .
 # Copyright (C) 2014 ORGANIZATION
-# This file is distributed under the same license as the PROJECT project.
+# This file is distributed under the same license as the  project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
 #
+# Translators:
+# Adam Tauber <asciimoo@gmail.com>, 2014
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2014
 msgid ""
 msgstr ""
-"Project-Id-Version: PROJECT VERSION\n"
+"Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
-"PO-Revision-Date: 2014-01-21 23:33+0100\n"
+"PO-Revision-Date: 2014-09-07 21:30+0000\n"
-"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Last-Translator: Adam Tauber <asciimoo@gmail.com>\n"
-"Language-Team: hu <LL@li.org>\n"
+"Language-Team: Hungarian "
-"Plural-Forms: nplurals=1; plural=0\n"
+"(http://www.transifex.com/projects/p/searx/language/hu/)\n"
+"Plural-Forms: nplurals=2; plural=(n != 1)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
 msgstr "{minutes} perce"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
 msgstr "{hours} óra, {minutes} perce"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr "Válaszidők (sec)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Találatok száma"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Pontszámok"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Pontszámok találatonként"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Hibák"
 
@@ -167,30 +170,30 @@ msgstr "vissza"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "Keresési URL"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Találatok letöltése"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Javaslatok"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "előző oldal"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "következő oldal"
@@ -206,6 +209,10 @@ msgstr "Keresés..."
 msgid "Engine stats"
 msgstr "Kereső statisztikák"
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -223,14 +230,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
-msgstr "általános"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Kereső statisztikák"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -256,11 +261,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Találatok száma"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -335,9 +339,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "következő oldal"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -374,3 +377,6 @@ msgstr "it"
 msgid "news"
 msgstr "hírek"
 
+msgid "map"
+msgstr "térkép"
+

searx/translations/it/LC_MESSAGES/messages.po

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
-"PO-Revision-Date: 2014-03-05 13:30+0000\n"
+"PO-Revision-Date: 2014-09-08 08:19+0000\n"
 "Last-Translator: dp <d.pitrolo@gmx.com>\n"
 "Language-Team: Italian "
 "(http://www.transifex.com/projects/p/searx/language/it/)\n"
@@ -19,31 +19,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
-msgstr ""
+msgstr "di {minutes} minuti fa"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
-msgstr ""
+msgstr "di {ore} h e {minutes} minuti fa"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr " Caricamento della pagina (secondi)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Risultati ottenuti"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Punteggio"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Punteggio per risultato"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Errori"
 
@@ -171,30 +171,30 @@ msgstr "indietro"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "URL della ricerca"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Scarica i risultati"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Suggerimenti"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "pagina precedente"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "pagina successiva"
@@ -203,13 +203,17 @@ msgstr "pagina successiva"
 #: searx/templates/default/search.html:3 searx/templates/oscar/search.html:4
 #: searx/templates/oscar/search_full.html:5
 msgid "Search for..."
-msgstr ""
+msgstr "Cerca…"
 
 #: searx/templates/courgette/stats.html:4 searx/templates/default/stats.html:4
 #: searx/templates/oscar/stats.html:5
 msgid "Engine stats"
 msgstr "Statistiche dei motori"
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -227,14 +231,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
-msgstr "generale"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Statistiche dei motori"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -260,11 +262,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Risultati ottenuti"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -339,9 +340,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "pagina successiva"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -378,3 +378,6 @@ msgstr "it"
 msgid "news"
 msgstr "notizie"
 
+msgid "map"
+msgstr "mappe"
+

searx/translations/ja/LC_MESSAGES/messages.po

@@ -0,0 +1,377 @@
+# Japanese translations for PROJECT.
+# Copyright (C) 2014 ORGANIZATION
+# This file is distributed under the same license as the PROJECT project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PROJECT VERSION\n"
+"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
+"PO-Revision-Date: 2014-10-05 16:38+0200\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: ja <LL@li.org>\n"
+"Plural-Forms: nplurals=1; plural=0\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 1.3\n"
+
+#: searx/webapp.py:305
+msgid "{minutes} minute(s) ago"
+msgstr ""
+
+#: searx/webapp.py:307
+msgid "{hours} hour(s), {minutes} minute(s) ago"
+msgstr ""
+
+#: searx/engines/__init__.py:177
+msgid "Page loads (sec)"
+msgstr ""
+
+#: searx/engines/__init__.py:181
+msgid "Number of results"
+msgstr ""
+
+#: searx/engines/__init__.py:185
+msgid "Scores"
+msgstr ""
+
+#: searx/engines/__init__.py:189
+msgid "Scores per result"
+msgstr ""
+
+#: searx/engines/__init__.py:193
+msgid "Errors"
+msgstr ""
+
+#: searx/templates/courgette/index.html:8 searx/templates/default/index.html:8
+#: searx/templates/oscar/about.html:3 searx/templates/oscar/navbar.html:16
+msgid "about"
+msgstr "に関する"
+
+#: searx/templates/courgette/index.html:9 searx/templates/default/index.html:9
+#: searx/templates/oscar/navbar.html:17
+#: searx/templates/oscar/preferences.html:2
+msgid "preferences"
+msgstr "設定"
+
+#: searx/templates/courgette/preferences.html:5
+#: searx/templates/default/preferences.html:5
+#: searx/templates/oscar/preferences.html:6
+msgid "Preferences"
+msgstr "設定"
+
+#: searx/templates/courgette/preferences.html:9
+#: searx/templates/default/preferences.html:9
+#: searx/templates/oscar/preferences.html:21
+msgid "Default categories"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:15
+#: searx/templates/default/preferences.html:15
+#: searx/templates/oscar/preferences.html:27
+msgid "Search language"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:18
+#: searx/templates/default/preferences.html:18
+#: searx/templates/oscar/preferences.html:30
+msgid "Automatic"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:26
+#: searx/templates/default/preferences.html:26
+#: searx/templates/oscar/preferences.html:39
+msgid "Interface language"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:36
+#: searx/templates/default/preferences.html:36
+#: searx/templates/oscar/preferences.html:50
+msgid "Autocomplete"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:47
+#: searx/templates/default/preferences.html:47
+#: searx/templates/oscar/preferences.html:63
+msgid "Method"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:56
+#: searx/templates/default/preferences.html:56
+#: searx/templates/oscar/preferences.html:73
+msgid "Themes"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:66
+#: searx/templates/default/preferences.html:66
+msgid "Currently used search engines"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:70
+#: searx/templates/default/preferences.html:70
+msgid "Engine name"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:71
+#: searx/templates/default/preferences.html:71
+msgid "Category"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:72
+#: searx/templates/courgette/preferences.html:83
+#: searx/templates/default/preferences.html:72
+#: searx/templates/default/preferences.html:83
+#: searx/templates/oscar/preferences.html:110
+msgid "Allow"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:72
+#: searx/templates/courgette/preferences.html:84
+#: searx/templates/default/preferences.html:72
+#: searx/templates/default/preferences.html:84
+#: searx/templates/oscar/preferences.html:109
+msgid "Block"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:92
+#: searx/templates/default/preferences.html:92
+#: searx/templates/oscar/preferences.html:124
+msgid ""
+"These settings are stored in your cookies, this allows us not to store "
+"this data about you."
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:94
+#: searx/templates/default/preferences.html:94
+#: searx/templates/oscar/preferences.html:126
+msgid ""
+"These cookies serve your sole convenience, we don't use these cookies to "
+"track you."
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:97
+#: searx/templates/default/preferences.html:97
+#: searx/templates/oscar/preferences.html:129
+msgid "save"
+msgstr ""
+
+#: searx/templates/courgette/preferences.html:98
+#: searx/templates/default/preferences.html:98
+#: searx/templates/oscar/preferences.html:130
+msgid "back"
+msgstr ""
+
+#: searx/templates/courgette/results.html:12
+#: searx/templates/default/results.html:12
+#: searx/templates/oscar/results.html:70
+msgid "Search URL"
+msgstr ""
+
+#: searx/templates/courgette/results.html:16
+#: searx/templates/default/results.html:16
+#: searx/templates/oscar/results.html:75
+msgid "Download results"
+msgstr ""
+
+#: searx/templates/courgette/results.html:34
+#: searx/templates/default/results.html:42
+#: searx/templates/oscar/results.html:50
+msgid "Suggestions"
+msgstr "提案"
+
+#: searx/templates/courgette/results.html:62
+#: searx/templates/default/results.html:78
+#: searx/templates/oscar/results.html:29
+msgid "previous page"
+msgstr "前のページ"
+
+#: searx/templates/courgette/results.html:73
+#: searx/templates/default/results.html:89
+#: searx/templates/oscar/results.html:37
+msgid "next page"
+msgstr "次のページ"
+
+#: searx/templates/courgette/search.html:3
+#: searx/templates/default/search.html:3 searx/templates/oscar/search.html:4
+#: searx/templates/oscar/search_full.html:5
+msgid "Search for..."
+msgstr "検索する..."
+
+#: searx/templates/courgette/stats.html:4 searx/templates/default/stats.html:4
+#: searx/templates/oscar/stats.html:5
+msgid "Engine stats"
+msgstr ""
+
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
+#: searx/templates/oscar/base.html:61
+msgid "Powered by"
+msgstr ""
+
+#: searx/templates/oscar/base.html:61
+msgid "a privacy-respecting, hackable metasearch engine"
+msgstr ""
+
+#: searx/templates/oscar/navbar.html:6
+msgid "Toggle navigation"
+msgstr ""
+
+#: searx/templates/oscar/navbar.html:15
+msgid "home"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:11
+msgid "General"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:12
+msgid "Engines"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:36
+msgid "What language do you prefer for search?"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:47
+msgid "Change the language of the layout"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:60
+msgid "Find stuff as you type"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:70
+msgid ""
+"Change how forms are submited, <a "
+"href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\""
+" rel=\"external\">learn more about request methods</a>"
+msgstr ""
+
+#: searx/templates/oscar/preferences.html:81
+msgid "Change searx layout"
+msgstr ""
+
+#: searx/templates/oscar/results.html:6
+msgid "Search results"
+msgstr ""
+
+#: searx/templates/oscar/results.html:65
+msgid "Links"
+msgstr ""
+
+#: searx/templates/oscar/search.html:6 searx/templates/oscar/search_full.html:7
+msgid "Start search"
+msgstr ""
+
+#: searx/templates/oscar/search_full.html:11
+msgid "Show search filters"
+msgstr ""
+
+#: searx/templates/oscar/search_full.html:11
+msgid "Hide search filters"
+msgstr ""
+
+#: searx/templates/oscar/stats.html:2
+msgid "stats"
+msgstr ""
+
+#: searx/templates/oscar/messages/first_time.html:4
+#: searx/templates/oscar/messages/no_results.html:5
+#: searx/templates/oscar/messages/save_settings_successfull.html:5
+#: searx/templates/oscar/messages/unknow_error.html:5
+msgid "Close"
+msgstr ""
+
+#: searx/templates/oscar/messages/first_time.html:6
+#: searx/templates/oscar/messages/no_data_available.html:3
+msgid "Heads up!"
+msgstr ""
+
+#: searx/templates/oscar/messages/first_time.html:7
+msgid "It look like you are using searx first time."
+msgstr ""
+
+#: searx/templates/oscar/messages/js_disabled.html:2
+msgid "Warning!"
+msgstr ""
+
+#: searx/templates/oscar/messages/js_disabled.html:3
+msgid "Please enable JavaScript to use full functionality of this site."
+msgstr ""
+
+#: searx/templates/oscar/messages/no_data_available.html:4
+msgid "There is currently no data available. "
+msgstr ""
+
+#: searx/templates/oscar/messages/no_results.html:7
+msgid "Sorry!"
+msgstr ""
+
+#: searx/templates/oscar/messages/no_results.html:8
+msgid ""
+"we didn't find any results. Please use another query or search in more "
+"categories."
+msgstr ""
+
+#: searx/templates/oscar/messages/save_settings_successfull.html:7
+msgid "Well done!"
+msgstr ""
+
+#: searx/templates/oscar/messages/save_settings_successfull.html:8
+msgid "Settings saved successfully."
+msgstr ""
+
+#: searx/templates/oscar/messages/unknow_error.html:7
+msgid "Oh snap!"
+msgstr ""
+
+#: searx/templates/oscar/messages/unknow_error.html:8
+msgid "Something went wrong."
+msgstr ""
+
+#: searx/templates/oscar/result_templates/images.html:20
+msgid "Get image"
+msgstr ""
+
+#: searx/templates/oscar/result_templates/images.html:21
+msgid "View source"
+msgstr ""
+
+#: searx/templates/oscar/result_templates/torrent.html:7
+msgid "Seeder"
+msgstr ""
+
+#: searx/templates/oscar/result_templates/torrent.html:7
+msgid "Leecher"
+msgstr ""
+
+# categories - manually added
+# TODO - automatically add
+msgid "files"
+msgstr "ファイル"
+
+msgid "map"
+msgstr "地図"
+
+msgid "music"
+msgstr "音楽"
+
+msgid "social media"
+msgstr "ソーシャルメディア"
+
+msgid "images"
+msgstr "画像"
+
+msgid "videos"
+msgstr "動画"
+
+msgid "it"
+msgstr "情報技術"
+
+msgid "news"
+msgstr "ニュース"
+

searx/translations/nl/LC_MESSAGES/messages.po

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version:  searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2014-10-01 19:45+0200\n"
+"POT-Creation-Date: 2014-10-26 19:10+0100\n"
-"PO-Revision-Date: 2014-03-15 20:20+0000\n"
+"PO-Revision-Date: 2014-09-09 15:33+0000\n"
 "Last-Translator: André Koot <meneer@tken.net>\n"
 "Language-Team: Dutch "
 "(http://www.transifex.com/projects/p/searx/language/nl/)\n"
@@ -19,31 +19,31 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 1.3\n"
 
-#: searx/webapp.py:252
+#: searx/webapp.py:305
 msgid "{minutes} minute(s) ago"
-msgstr ""
+msgstr "{minutes} min geleden"
 
-#: searx/webapp.py:254
+#: searx/webapp.py:307
 msgid "{hours} hour(s), {minutes} minute(s) ago"
-msgstr ""
+msgstr "{hours} uur, {minutes} min geleden"
 
-#: searx/engines/__init__.py:164
+#: searx/engines/__init__.py:177
 msgid "Page loads (sec)"
 msgstr "Pagina laadt (sec)"
 
-#: searx/engines/__init__.py:168
+#: searx/engines/__init__.py:181
 msgid "Number of results"
 msgstr "Aantal zoekresultaten"
 
-#: searx/engines/__init__.py:172
+#: searx/engines/__init__.py:185
 msgid "Scores"
 msgstr "Scores"
 
-#: searx/engines/__init__.py:176
+#: searx/engines/__init__.py:189
 msgid "Scores per result"
 msgstr "Scores per zoekresultaat"
 
-#: searx/engines/__init__.py:180
+#: searx/engines/__init__.py:193
 msgid "Errors"
 msgstr "Fouten"
 
@@ -171,30 +171,30 @@ msgstr "terug"
 
 #: searx/templates/courgette/results.html:12
 #: searx/templates/default/results.html:12
-#: searx/templates/oscar/results.html:74
+#: searx/templates/oscar/results.html:70
 msgid "Search URL"
 msgstr "Zoek URL"
 
 #: searx/templates/courgette/results.html:16
 #: searx/templates/default/results.html:16
-#: searx/templates/oscar/results.html:79
+#: searx/templates/oscar/results.html:75
 msgid "Download results"
 msgstr "Downloaden zoekresultaten"
 
 #: searx/templates/courgette/results.html:34
-#: searx/templates/default/results.html:34
+#: searx/templates/default/results.html:42
-#: searx/templates/oscar/results.html:51
+#: searx/templates/oscar/results.html:50
 msgid "Suggestions"
 msgstr "Suggesties"
 
 #: searx/templates/courgette/results.html:62
-#: searx/templates/default/results.html:62
+#: searx/templates/default/results.html:78
 #: searx/templates/oscar/results.html:29
 msgid "previous page"
 msgstr "vorige pagina"
 
 #: searx/templates/courgette/results.html:73
-#: searx/templates/default/results.html:73
+#: searx/templates/default/results.html:89
 #: searx/templates/oscar/results.html:37
 msgid "next page"
 msgstr "volgende pagina"
@@ -210,6 +210,10 @@ msgstr "Zoeken naar..."
 msgid "Engine stats"
 msgstr "Zoekmachinestatistieken"
 
+#: searx/templates/default/results.html:34
+msgid "Answers"
+msgstr ""
+
 #: searx/templates/oscar/base.html:61
 msgid "Powered by"
 msgstr ""
@@ -227,14 +231,12 @@ msgid "home"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:11
-#, fuzzy
 msgid "General"
-msgstr "algemeen"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:12
-#, fuzzy
 msgid "Engines"
-msgstr "Zoekmachinestatistieken"
+msgstr ""
 
 #: searx/templates/oscar/preferences.html:36
 msgid "What language do you prefer for search?"
@@ -260,11 +262,10 @@ msgid "Change searx layout"
 msgstr ""
 
 #: searx/templates/oscar/results.html:6
-#, fuzzy
 msgid "Search results"
-msgstr "Aantal zoekresultaten"
+msgstr ""
 
-#: searx/templates/oscar/results.html:68
+#: searx/templates/oscar/results.html:65
 msgid "Links"
 msgstr ""
 
@@ -339,9 +340,8 @@ msgid "Something went wrong."
 msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:20
-#, fuzzy
 msgid "Get image"
-msgstr "volgende pagina"
+msgstr ""
 
 #: searx/templates/oscar/result_templates/images.html:21
 msgid "View source"
@@ -378,3 +378,6 @@ msgstr "it"
 msgid "news"
 msgstr "nieuws"
 
+msgid "map"
+msgstr "kaart"
+

searx/utils.py

@@ -1,4 +1,4 @@
-#import htmlentitydefs
+# import htmlentitydefs
 from codecs import getincrementalencoder
 from HTMLParser import HTMLParser
 from random import choice
@@ -20,6 +20,10 @@ def gen_useragent():
     return ua.format(os=choice(ua_os), version=choice(ua_versions))
 
 
+def searx_useragent():
+    return 'searx'
+
+
 def highlight_content(content, query):
 
     if not content:
@@ -64,8 +68,8 @@ class HTMLTextExtractor(HTMLParser):
         self.result.append(unichr(codepoint))
 
     def handle_entityref(self, name):
-        #codepoint = htmlentitydefs.name2codepoint[name]
+        # codepoint = htmlentitydefs.name2codepoint[name]
-        #self.result.append(unichr(codepoint))
+        # self.result.append(unichr(codepoint))
         self.result.append(name)
 
     def get_text(self):

searx/webapp.py

@@ -50,13 +50,16 @@ from searx.search import Search
 from searx.query import Query
 from searx.autocomplete import backends as autocomplete_backends
 
+from urlparse import urlparse
+import re
+
 
 static_path, templates_path, themes =\
     get_themes(settings['themes_path']
                if settings.get('themes_path')
                else searx_dir)
-default_theme = settings['default_theme'] if \
+
-    settings.get('default_theme', None) else 'default'
+default_theme = settings['server'].get('default_theme', 'default')
 
 app = Flask(
     __name__,
@@ -143,14 +146,14 @@ def render(template_name, override_theme=None, **kwargs):
 
     nonblocked_categories = set(chain.from_iterable(nonblocked_categories))
 
-    if not 'categories' in kwargs:
+    if 'categories' not in kwargs:
         kwargs['categories'] = ['general']
         kwargs['categories'].extend(x for x in
                                     sorted(categories.keys())
                                     if x != 'general'
                                     and x in nonblocked_categories)
 
-    if not 'selected_categories' in kwargs:
+    if 'selected_categories' not in kwargs:
         kwargs['selected_categories'] = []
         for arg in request.args:
             if arg.startswith('category_'):
@@ -165,7 +168,7 @@ def render(template_name, override_theme=None, **kwargs):
     if not kwargs['selected_categories']:
         kwargs['selected_categories'] = ['general']
 
-    if not 'autocomplete' in kwargs:
+    if 'autocomplete' not in kwargs:
         kwargs['autocomplete'] = autocomplete
 
     kwargs['method'] = request.cookies.get('method', 'POST')
@@ -201,23 +204,72 @@ def index():
             'index.html',
         )
 
-    search.results, search.suggestions, search.answers, search.infoboxes = search.search(request)
+    search.results, search.suggestions,\
+        search.answers, search.infoboxes = search.search(request)
 
     for result in search.results:
 
         if not search.paging and engines[result['engine']].paging:
             search.paging = True
 
+        # check if HTTPS rewrite is required
         if settings['server']['https_rewrite']\
            and result['parsed_url'].scheme == 'http':
 
-            for http_regex, https_url in https_rules:
+            skip_https_rewrite = False
-                if http_regex.match(result['url']):
+
-                    result['url'] = http_regex.sub(https_url, result['url'])
+            # check if HTTPS rewrite is possible
-                    # TODO result['parsed_url'].scheme
+            for target, rules, exclusions in https_rules:
+
+                # check if target regex match with url
+                if target.match(result['url']):
+                    # process exclusions
+                    for exclusion in exclusions:
+                        # check if exclusion match with url
+                        if exclusion.match(result['url']):
+                            skip_https_rewrite = True
+                            break
+
+                    # skip https rewrite if required
+                    if skip_https_rewrite:
+                        break
+
+                    # process rules
+                    for rule in rules:
+                        try:
+                            # TODO, precompile rule
+                            p = re.compile(rule[0])
+
+                            # rewrite url if possible
+                            new_result_url = p.sub(rule[1], result['url'])
+                        except:
+                            break
+
+                        # parse new url
+                        new_parsed_url = urlparse(new_result_url)
+
+                        # continiue if nothing was rewritten
+                        if result['url'] == new_result_url:
+                            continue
+
+                        # get domainname from result
+                        # TODO, does only work correct with TLD's like
+                        #  asdf.com, not for asdf.com.de
+                        # TODO, using publicsuffix instead of this rewrite rule
+                        old_result_domainname = '.'.join(
+                            result['parsed_url'].hostname.split('.')[-2:])
+                        new_result_domainname = '.'.join(
+                            new_parsed_url.hostname.split('.')[-2:])
+
+                        # check if rewritten hostname is the same,
+                        # to protect against wrong or malicious rewrite rules
+                        if old_result_domainname == new_result_domainname:
+                            # set new url
+                            result['url'] = new_result_url
+
+                    # target has matched, do not search over the other rules
                     break
 
-        # HTTPS rewrite
         if search.request_data.get('format', 'html') == 'html':
             if 'content' in result:
                 result['content'] = highlight_content(result['content'],
@@ -384,7 +436,7 @@ def preferences():
         for pd_name, pd in request.form.items():
             if pd_name.startswith('category_'):
                 category = pd_name[9:]
-                if not category in categories:
+                if category not in categories:
                     continue
                 selected_categories.append(category)
             elif pd_name == 'locale' and pd in settings['locales']:

setup.py

@@ -15,7 +15,7 @@ long_description = read('README.rst')
 
 setup(
     name='searx',
-    version="0.3.1",
+    version="0.4.0",
     description="A privacy-respecting, hackable metasearch engine",
     long_description=long_description,
     classifiers=[
@@ -70,6 +70,7 @@ setup(
             'translations/*/*/*',
             'templates/*/*.xml',
             'templates/*/*.html',
+            'https_rules/*.xml',
             'templates/*/result_templates/*.html',
         ],
     },