From 9b9f097adbf39d7908931203e9d8966748900bde Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 14 Sep 2014 11:09:44 +0200 Subject: [PATCH 1/5] Implementing https rewrite support #71 * parsing XML-Files which contain target, exclusions and rules * convert regex if required (is a little hack, probably does not work for all rules) * check if target rule apply for http url, and use the rules to rewrite it * add pice of code, to check if domain name has not changed during rewrite (should be rewritten, using publicsuffix instead of little hack) --- searx/__init__.py | 8 +++ searx/https_rewrite.py | 141 ++++++++++++++++++++++++++++++++++++++--- searx/webapp.py | 52 +++++++++++++-- 3 files changed, 187 insertions(+), 14 deletions(-) diff --git a/searx/__init__.py b/searx/__init__.py index 375a5414a..0acc61447 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -1,5 +1,6 @@ from os import environ from os.path import realpath, dirname, join, abspath +from searx.https_rewrite import load_https_rules try: from yaml import load except: @@ -15,6 +16,13 @@ if 'SEARX_SETTINGS_PATH' in environ: else: settings_path = join(searx_dir, 'settings.yml') +if 'SEARX_HTTPS_REWRITE_PATH' in environ: + https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH'] +else: + https_rewrite_path = join(searx_dir, 'https_rules') with open(settings_path) as settings_yaml: settings = load(settings_yaml) + +# loade https rules +load_https_rules(https_rewrite_path) diff --git a/searx/https_rewrite.py b/searx/https_rewrite.py index 44ada9450..814eda2de 100644 --- a/searx/https_rewrite.py +++ b/searx/https_rewrite.py @@ -1,14 +1,139 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2013- by Adam Tauber, +''' + import re +from lxml import etree +from os import listdir +from os.path import isfile, join + # https://gitweb.torproject.org/\ # pde/https-everywhere.git/tree/4.0:/src/chrome/content/rules # HTTPS rewrite rules -https_rules = ( - # from - (re.compile(r'^http://(www\.|m\.|)?xkcd\.(?:com|org)/', re.I | re.U), - # to - r'https://\1xkcd.com/'), - (re.compile(r'^https?://(?:ssl)?imgs\.xkcd\.com/', re.I | re.U), - r'https://sslimgs.xkcd.com/'), -) +https_rules = [] + + +# load single ruleset from a xml file +def load_single_https_ruleset(filepath): + ruleset = () + + # init parser + parser = etree.XMLParser() + + # load and parse xml-file + try: + tree = etree.parse(filepath, parser) + except: + # TODO, error message + return () + + # get root node + root = tree.getroot() + + #print(etree.tostring(tree)) + + # check if root is a node with the name ruleset + # TODO improve parsing + if root.tag != 'ruleset': + return () + + # check if rule is deactivated by default + if root.attrib.get('default_off'): + return () + + # check if rule does only work for specific platforms + if root.attrib.get('platform'): + return () + + hosts = [] + rules = [] + exclusions = [] + + # parse childs from ruleset + for ruleset in root: + # this child define a target + if ruleset.tag == 'target': + # check if required tags available + if not ruleset.attrib.get('host'): + continue + + # convert host-rule to valid regex + host = ruleset.attrib.get('host').replace('.', '\.').replace('*', '.*') + + # append to host list + hosts.append(host) + + # this child define a rule + elif ruleset.tag == 'rule': + # check if required tags available + if not ruleset.attrib.get('from')\ + or not ruleset.attrib.get('to'): + continue + + # TODO hack, which convert a javascript regex group into a valid python regex group + rule_from = ruleset.attrib.get('from').replace('$', '\\') + rule_to = ruleset.attrib.get('to').replace('$', '\\') + + # TODO, not working yet because of the hack above, currently doing that in webapp.py + #rule_from_rgx = re.compile(rule_from, re.I) + + # append rule + rules.append((rule_from, rule_to)) + + # this child define an exclusion + elif ruleset.tag == 'exclusion': + # check if required tags available + if not ruleset.attrib.get('pattern'): + continue + + exclusion_rgx = re.compile(ruleset.attrib.get('pattern')) + + # append exclusion + exclusions.append(exclusion_rgx) + + # convert list of possible hosts to a simple regex + # TODO compress regex to improve performance + try: + target_hosts = re.compile('^(' + '|'.join(hosts) + ')', re.I | re.U) + except: + return () + + # return ruleset + return (target_hosts, rules, exclusions) + + +# load all https rewrite rules +def load_https_rules(rules_path): + # add / to path if not set yet + if rules_path[-1:] != '/': + rules_path += '/' + + # search all xml files which are stored in the https rule directory + xml_files = [ join(rules_path,f) for f in listdir(rules_path) if isfile(join(rules_path,f)) and f[-4:] == '.xml' ] + + # load xml-files + for ruleset_file in xml_files: + # calculate rewrite-rules + ruleset = load_single_https_ruleset(ruleset_file) + + # skip if no ruleset returned + if not ruleset: + continue + + # append ruleset + https_rules.append(ruleset) diff --git a/searx/webapp.py b/searx/webapp.py index 2bf3afaf4..7952415af 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -49,6 +49,9 @@ from searx.languages import language_codes from searx.search import Search from searx.autocomplete import backends as autocomplete_backends +from urlparse import urlparse +import re + static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -197,16 +200,53 @@ def index(): if not search.paging and engines[result['engine']].paging: search.paging = True + # check if HTTPS rewrite is required if settings['server']['https_rewrite']\ and result['parsed_url'].scheme == 'http': - for http_regex, https_url in https_rules: - if http_regex.match(result['url']): - result['url'] = http_regex.sub(https_url, result['url']) - # TODO result['parsed_url'].scheme - break + skip_https_rewrite = False + + # check if HTTPS rewrite is possible + for target, rules, exclusions in https_rules: + + # check if target regex match with url + if target.match(result['url']): + # process exclusions + for exclusion in exclusions: + # check if exclusion match with url + if exclusion.match(result['url']): + skip_https_rewrite = True + break + + # skip https rewrite if required + if skip_https_rewrite: + break + + # process rules + for rule in rules: + # TODO, precompile rule + p = re.compile(rule[0]) + # rewrite url if possible + new_result_url = p.sub(rule[1], result['url']) + + # parse new url + new_parsed_url = urlparse(new_result_url) + + # continiue if nothing was rewritten + if result['url'] == new_result_url: + continue + + # get domainname from result + # TODO, does only work correct with TLD's like asdf.com, not for asdf.com.de + # TODO, using publicsuffix instead of this rewrite rule + old_result_domainname = '.'.join(result['parsed_url'].hostname.split('.')[-2:]) + new_result_domainname = '.'.join(new_parsed_url.hostname.split('.')[-2:]) + + # check if rewritten hostname is the same, to protect against wrong or malicious rewrite rules + if old_result_domainname == new_result_domainname: + # set new url + result['url'] = new_result_url - # HTTPS rewrite if search.request_data.get('format', 'html') == 'html': if 'content' in result: result['content'] = highlight_content(result['content'], From 6e6dbc1bb4a3e6f8746ea3677cdbc190c762b511 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 14 Sep 2014 13:24:40 +0200 Subject: [PATCH 2/5] update settings_robot.yml --- searx/settings_robot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml index 98944a811..bb91dce8f 100644 --- a/searx/settings_robot.yml +++ b/searx/settings_robot.yml @@ -4,6 +4,9 @@ server: debug : False request_timeout : 3.0 # seconds base_url: False + themes_path : "" + default_theme : default + https_rewrite : True engines: - name : general_dummy From 0616d26feb0f96b3d4fd6b2744ae0288c9fed96b Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 14 Sep 2014 14:17:12 +0200 Subject: [PATCH 3/5] improve https rewriting --- searx/__init__.py | 6 ++++-- searx/webapp.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/searx/__init__.py b/searx/__init__.py index 0acc61447..d4fc7f0bb 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -24,5 +24,7 @@ else: with open(settings_path) as settings_yaml: settings = load(settings_yaml) -# loade https rules -load_https_rules(https_rewrite_path) +# load https rules only if https rewrite is enabled +if settings.get('server', {}).get('https_rewrite'): + # loade https rules + load_https_rules(https_rewrite_path) diff --git a/searx/webapp.py b/searx/webapp.py index 7952415af..d9dc5f710 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -247,6 +247,9 @@ def index(): # set new url result['url'] = new_result_url + # target has matched, do not search over the other rules + break + if search.request_data.get('format', 'html') == 'html': if 'content' in result: result['content'] = highlight_content(result['content'], From bb126da0fb49d1c9640eeb3371d0bbcf005bcd2b Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Wed, 15 Oct 2014 14:47:03 +0200 Subject: [PATCH 4/5] improve https rewrite code --- searx/https_rewrite.py | 2 ++ searx/webapp.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/searx/https_rewrite.py b/searx/https_rewrite.py index 814eda2de..18405d87a 100644 --- a/searx/https_rewrite.py +++ b/searx/https_rewrite.py @@ -137,3 +137,5 @@ def load_https_rules(rules_path): # append ruleset https_rules.append(ruleset) + + print(' * {n} https-rules loaded'.format(n=len(https_rules))) diff --git a/searx/webapp.py b/searx/webapp.py index d9dc5f710..25c99d94c 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -224,10 +224,14 @@ def index(): # process rules for rule in rules: - # TODO, precompile rule - p = re.compile(rule[0]) - # rewrite url if possible - new_result_url = p.sub(rule[1], result['url']) + try: + # TODO, precompile rule + p = re.compile(rule[0]) + + # rewrite url if possible + new_result_url = p.sub(rule[1], result['url']) + except: + break # parse new url new_parsed_url = urlparse(new_result_url) From 1e3e6465845236b027ce8df9ab5fcc78ffe3f1d1 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Wed, 15 Oct 2014 14:52:27 +0200 Subject: [PATCH 5/5] import basic https rules from https-everywhere repository * licenced under GPLv3+ * downloaded from https://github.com/EFForg/https-everywhere --- searx/https_rules/00README | 17 + searx/https_rules/Bing.xml | 56 + searx/https_rules/Dailymotion.xml | 69 + searx/https_rules/Deviantart.xml | 53 + searx/https_rules/DuckDuckGo.xml | 38 + searx/https_rules/Flickr.xml | 44 + searx/https_rules/Github-Pages.xml | 11 + searx/https_rules/Github.xml | 94 + searx/https_rules/Google-mismatches.xml | 26 + searx/https_rules/Google.org.xml | 14 + searx/https_rules/GoogleAPIs.xml | 143 ++ searx/https_rules/GoogleCanada.xml | 6 + searx/https_rules/GoogleImages.xml | 65 + searx/https_rules/GoogleMainSearch.xml | 78 + searx/https_rules/GoogleMaps.xml | 67 + searx/https_rules/GoogleMelange.xml | 6 + searx/https_rules/GoogleSearch.xml | 135 ++ searx/https_rules/GoogleServices.xml | 345 ++++ searx/https_rules/GoogleShopping.xml | 28 + searx/https_rules/GoogleSorry.xml | 7 + searx/https_rules/GoogleTranslate.xml | 8 + searx/https_rules/GoogleVideos.xml | 83 + searx/https_rules/GoogleWatchBlog.xml | 17 + searx/https_rules/Google_App_Engine.xml | 21 + searx/https_rules/Googleplex.com.xml | 16 + searx/https_rules/OpenStreetMap.xml | 15 + searx/https_rules/Rawgithub.com.xml | 14 + searx/https_rules/Soundcloud.xml | 101 + searx/https_rules/ThePirateBay.xml | 36 + searx/https_rules/Torproject.xml | 18 + searx/https_rules/Twitter.xml | 169 ++ searx/https_rules/Vimeo.xml | 75 + searx/https_rules/WikiLeaks.xml | 13 + searx/https_rules/Wikimedia.xml | 107 + searx/https_rules/Yahoo.xml | 2450 +++++++++++++++++++++++ searx/https_rules/YouTube.xml | 46 + 36 files changed, 4491 insertions(+) create mode 100644 searx/https_rules/00README create mode 100644 searx/https_rules/Bing.xml create mode 100644 searx/https_rules/Dailymotion.xml create mode 100644 searx/https_rules/Deviantart.xml create mode 100644 searx/https_rules/DuckDuckGo.xml create mode 100644 searx/https_rules/Flickr.xml create mode 100644 searx/https_rules/Github-Pages.xml create mode 100644 searx/https_rules/Github.xml create mode 100644 searx/https_rules/Google-mismatches.xml create mode 100644 searx/https_rules/Google.org.xml create mode 100644 searx/https_rules/GoogleAPIs.xml create mode 100644 searx/https_rules/GoogleCanada.xml create mode 100644 searx/https_rules/GoogleImages.xml create mode 100644 searx/https_rules/GoogleMainSearch.xml create mode 100644 searx/https_rules/GoogleMaps.xml create mode 100644 searx/https_rules/GoogleMelange.xml create mode 100644 searx/https_rules/GoogleSearch.xml create mode 100644 searx/https_rules/GoogleServices.xml create mode 100644 searx/https_rules/GoogleShopping.xml create mode 100644 searx/https_rules/GoogleSorry.xml create mode 100644 searx/https_rules/GoogleTranslate.xml create mode 100644 searx/https_rules/GoogleVideos.xml create mode 100644 searx/https_rules/GoogleWatchBlog.xml create mode 100644 searx/https_rules/Google_App_Engine.xml create mode 100644 searx/https_rules/Googleplex.com.xml create mode 100644 searx/https_rules/OpenStreetMap.xml create mode 100644 searx/https_rules/Rawgithub.com.xml create mode 100644 searx/https_rules/Soundcloud.xml create mode 100644 searx/https_rules/ThePirateBay.xml create mode 100644 searx/https_rules/Torproject.xml create mode 100644 searx/https_rules/Twitter.xml create mode 100644 searx/https_rules/Vimeo.xml create mode 100644 searx/https_rules/WikiLeaks.xml create mode 100644 searx/https_rules/Wikimedia.xml create mode 100644 searx/https_rules/Yahoo.xml create mode 100644 searx/https_rules/YouTube.xml diff --git a/searx/https_rules/00README b/searx/https_rules/00README new file mode 100644 index 000000000..fcd8a7724 --- /dev/null +++ b/searx/https_rules/00README @@ -0,0 +1,17 @@ + diff --git a/searx/https_rules/Bing.xml b/searx/https_rules/Bing.xml new file mode 100644 index 000000000..8b403f108 --- /dev/null +++ b/searx/https_rules/Bing.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Dailymotion.xml b/searx/https_rules/Dailymotion.xml new file mode 100644 index 000000000..743100cb7 --- /dev/null +++ b/searx/https_rules/Dailymotion.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Deviantart.xml b/searx/https_rules/Deviantart.xml new file mode 100644 index 000000000..7830fc20f --- /dev/null +++ b/searx/https_rules/Deviantart.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/DuckDuckGo.xml b/searx/https_rules/DuckDuckGo.xml new file mode 100644 index 000000000..173a9ad9f --- /dev/null +++ b/searx/https_rules/DuckDuckGo.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Flickr.xml b/searx/https_rules/Flickr.xml new file mode 100644 index 000000000..85c6e8065 --- /dev/null +++ b/searx/https_rules/Flickr.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Github-Pages.xml b/searx/https_rules/Github-Pages.xml new file mode 100644 index 000000000..d3be58a4c --- /dev/null +++ b/searx/https_rules/Github-Pages.xml @@ -0,0 +1,11 @@ + + + + + + + + diff --git a/searx/https_rules/Github.xml b/searx/https_rules/Github.xml new file mode 100644 index 000000000..a9a3a1e53 --- /dev/null +++ b/searx/https_rules/Github.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Google-mismatches.xml b/searx/https_rules/Google-mismatches.xml new file mode 100644 index 000000000..de9d3eb18 --- /dev/null +++ b/searx/https_rules/Google-mismatches.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Google.org.xml b/searx/https_rules/Google.org.xml new file mode 100644 index 000000000..d6cc47881 --- /dev/null +++ b/searx/https_rules/Google.org.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/GoogleAPIs.xml b/searx/https_rules/GoogleAPIs.xml new file mode 100644 index 000000000..85a5a8081 --- /dev/null +++ b/searx/https_rules/GoogleAPIs.xml @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleCanada.xml b/searx/https_rules/GoogleCanada.xml new file mode 100644 index 000000000..d5eefe816 --- /dev/null +++ b/searx/https_rules/GoogleCanada.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/searx/https_rules/GoogleImages.xml b/searx/https_rules/GoogleImages.xml new file mode 100644 index 000000000..0112001e0 --- /dev/null +++ b/searx/https_rules/GoogleImages.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMainSearch.xml b/searx/https_rules/GoogleMainSearch.xml new file mode 100644 index 000000000..df504d90c --- /dev/null +++ b/searx/https_rules/GoogleMainSearch.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMaps.xml b/searx/https_rules/GoogleMaps.xml new file mode 100644 index 000000000..0f82c5267 --- /dev/null +++ b/searx/https_rules/GoogleMaps.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMelange.xml b/searx/https_rules/GoogleMelange.xml new file mode 100644 index 000000000..ec23cd45f --- /dev/null +++ b/searx/https_rules/GoogleMelange.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/searx/https_rules/GoogleSearch.xml b/searx/https_rules/GoogleSearch.xml new file mode 100644 index 000000000..66b7ffdb0 --- /dev/null +++ b/searx/https_rules/GoogleSearch.xml @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleServices.xml b/searx/https_rules/GoogleServices.xml new file mode 100644 index 000000000..704646b53 --- /dev/null +++ b/searx/https_rules/GoogleServices.xml @@ -0,0 +1,345 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleShopping.xml b/searx/https_rules/GoogleShopping.xml new file mode 100644 index 000000000..6ba69a91d --- /dev/null +++ b/searx/https_rules/GoogleShopping.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleSorry.xml b/searx/https_rules/GoogleSorry.xml new file mode 100644 index 000000000..72a19210d --- /dev/null +++ b/searx/https_rules/GoogleSorry.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/searx/https_rules/GoogleTranslate.xml b/searx/https_rules/GoogleTranslate.xml new file mode 100644 index 000000000..a004025ae --- /dev/null +++ b/searx/https_rules/GoogleTranslate.xml @@ -0,0 +1,8 @@ + + + + + + + diff --git a/searx/https_rules/GoogleVideos.xml b/searx/https_rules/GoogleVideos.xml new file mode 100644 index 000000000..a5e88fcf0 --- /dev/null +++ b/searx/https_rules/GoogleVideos.xml @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleWatchBlog.xml b/searx/https_rules/GoogleWatchBlog.xml new file mode 100644 index 000000000..afec70c97 --- /dev/null +++ b/searx/https_rules/GoogleWatchBlog.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Google_App_Engine.xml b/searx/https_rules/Google_App_Engine.xml new file mode 100644 index 000000000..851e051d1 --- /dev/null +++ b/searx/https_rules/Google_App_Engine.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Googleplex.com.xml b/searx/https_rules/Googleplex.com.xml new file mode 100644 index 000000000..7ddbb5ba9 --- /dev/null +++ b/searx/https_rules/Googleplex.com.xml @@ -0,0 +1,16 @@ + + + + + + + + diff --git a/searx/https_rules/OpenStreetMap.xml b/searx/https_rules/OpenStreetMap.xml new file mode 100644 index 000000000..58a661823 --- /dev/null +++ b/searx/https_rules/OpenStreetMap.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/searx/https_rules/Rawgithub.com.xml b/searx/https_rules/Rawgithub.com.xml new file mode 100644 index 000000000..3868f332a --- /dev/null +++ b/searx/https_rules/Rawgithub.com.xml @@ -0,0 +1,14 @@ + + + + + + + + + + diff --git a/searx/https_rules/Soundcloud.xml b/searx/https_rules/Soundcloud.xml new file mode 100644 index 000000000..0baa5832b --- /dev/null +++ b/searx/https_rules/Soundcloud.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/ThePirateBay.xml b/searx/https_rules/ThePirateBay.xml new file mode 100644 index 000000000..010387b6b --- /dev/null +++ b/searx/https_rules/ThePirateBay.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Torproject.xml b/searx/https_rules/Torproject.xml new file mode 100644 index 000000000..69269af7e --- /dev/null +++ b/searx/https_rules/Torproject.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Twitter.xml b/searx/https_rules/Twitter.xml new file mode 100644 index 000000000..3285f44e0 --- /dev/null +++ b/searx/https_rules/Twitter.xml @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Vimeo.xml b/searx/https_rules/Vimeo.xml new file mode 100644 index 000000000..f2a3e5764 --- /dev/null +++ b/searx/https_rules/Vimeo.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/WikiLeaks.xml b/searx/https_rules/WikiLeaks.xml new file mode 100644 index 000000000..977709d2d --- /dev/null +++ b/searx/https_rules/WikiLeaks.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Wikimedia.xml b/searx/https_rules/Wikimedia.xml new file mode 100644 index 000000000..9f25831a2 --- /dev/null +++ b/searx/https_rules/Wikimedia.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Yahoo.xml b/searx/https_rules/Yahoo.xml new file mode 100644 index 000000000..33548c4ab --- /dev/null +++ b/searx/https_rules/Yahoo.xml @@ -0,0 +1,2450 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/YouTube.xml b/searx/https_rules/YouTube.xml new file mode 100644 index 000000000..bddc2a5f3 --- /dev/null +++ b/searx/https_rules/YouTube.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +