From 0226ae69d361dc28cfbb4a89a29a86ea02963b11 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 4 Dec 2020 16:47:43 +0100 Subject: [PATCH 1/3] [fix] dbpedia autocomplete (and use HTTPS) --- searx/autocomplete.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 420b8a461..fbe634a5b 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -113,7 +113,7 @@ def searx_bang(full_query): def dbpedia(query, lang): # dbpedia autocompleter, no HTTPS - autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' + autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' response = get(autocomplete_url + urlencode(dict(QueryString=query))) @@ -121,8 +121,7 @@ def dbpedia(query, lang): if response.ok: dom = etree.fromstring(response.content) - results = dom.xpath('//a:Result/a:Label//text()', - namespaces={'a': 'http://lookup.dbpedia.org/'}) + results = dom.xpath('//Result/Label//text()') return results From 244e812f37940800cee06f222b6cac26c729131e Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 4 Dec 2020 16:48:15 +0100 Subject: [PATCH 2/3] [fix] remove searx/engines/filecrop.py (dead code) --- searx/engines/filecrop.py | 85 --------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 searx/engines/filecrop.py diff --git a/searx/engines/filecrop.py b/searx/engines/filecrop.py deleted file mode 100644 index 0331e7b19..000000000 --- a/searx/engines/filecrop.py +++ /dev/null @@ -1,85 +0,0 @@ -from html.parser import HTMLParser -from urllib.parse import urlencode - - -url = 'http://www.filecrop.com/' -search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa - -paging = True - - -class FilecropResultParser(HTMLParser): # pylint: disable=W0223 # (see https://bugs.python.org/issue31844) - - def __init__(self): - HTMLParser.__init__(self) - self.__start_processing = False - - self.results = [] - self.result = {} - - self.tr_counter = 0 - self.data_counter = 0 - - def handle_starttag(self, tag, attrs): - - if tag == 'tr': - if ('bgcolor', '#edeff5') in attrs or\ - ('bgcolor', '#ffffff') in attrs: - self.__start_processing = True - - if not self.__start_processing: - return - - if tag == 'label': - self.result['title'] = [attr[1] for attr in attrs - if attr[0] == 'title'][0] - elif tag == 'a' and ('rel', 'nofollow') in attrs\ - and ('class', 'sourcelink') in attrs: - if 'content' in self.result: - self.result['content'] += [attr[1] for attr in attrs - if attr[0] == 'title'][0] - else: - self.result['content'] = [attr[1] for attr in attrs - if attr[0] == 'title'][0] - self.result['content'] += ' ' - elif tag == 'a': - self.result['url'] = url + [attr[1] for attr in attrs - if attr[0] == 'href'][0] - - def handle_endtag(self, tag): - if self.__start_processing is False: - return - - if tag == 'tr': - self.tr_counter += 1 - - if self.tr_counter == 2: - self.__start_processing = False - self.tr_counter = 0 - self.data_counter = 0 - self.results.append(self.result) - self.result = {} - - def handle_data(self, data): - if not self.__start_processing: - return - - if 'content' in self.result: - self.result['content'] += data + ' ' - else: - self.result['content'] = data + ' ' - - self.data_counter += 1 - - -def request(query, params): - index = 1 + (params['pageno'] - 1) * 30 - params['url'] = search_url.format(query=urlencode({'w': query}), index=index) - return params - - -def response(resp): - parser = FilecropResultParser() - parser.feed(resp.text) - - return parser.results From 38d32337e7600446c04ecd601b56d6915a4046fd Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 4 Dec 2020 16:50:26 +0100 Subject: [PATCH 3/3] [mod] documentations & comments: update http://* URL to https://*. About http://lesscss.org see https://github.com/less/less-docs/issues/520 --- CONTRIBUTING.md | 4 ++-- dockerfiles/docker-entrypoint.sh | 2 +- docs/admin/buildhosts.rst | 2 +- docs/admin/installation-nginx.rst | 2 +- docs/admin/settings.rst | 2 +- docs/conf.py | 6 +++--- docs/dev/contribution_guide.rst | 4 ++-- docs/dev/engine_overview.rst | 2 +- docs/dev/quickstart.rst | 2 +- docs/dev/reST.rst | 24 ++++++++++++------------ utils/makefile.python | 2 +- 11 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6850ab405..300349f3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,8 +2,8 @@ ## Resources in the documentation -* [Development quickstart](http://searx.github.io/searx/dev/contribution_guide.html) -* [Contribution guide](http://searx.github.io/searx/dev/contribution_guide.html) +* [Development quickstart](https://searx.github.io/searx/dev/contribution_guide.html) +* [Contribution guide](https://searx.github.io/searx/dev/contribution_guide.html) ## Submitting PRs diff --git a/dockerfiles/docker-entrypoint.sh b/dockerfiles/docker-entrypoint.sh index a8f18f05b..accc015f7 100755 --- a/dockerfiles/docker-entrypoint.sh +++ b/dockerfiles/docker-entrypoint.sh @@ -66,7 +66,7 @@ patch_searx_settings() { CONF="$1" # Make sure that there is trailing slash at the end of BASE_URL - # see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion + # see https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion export BASE_URL="${BASE_URL%/}/" # update settings.yml diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index a727d25b9..1f6eb472e 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -67,7 +67,7 @@ to ``imgmath``: If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ - graphviz from http://www.graphviz.org + graphviz from https://www.graphviz.org .. WARNING: LaTeX command 'latex' cannot be run (needed for math \ display), check the imgmath_latex setting diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst index 65fd73573..589c40ada 100644 --- a/docs/admin/installation-nginx.rst +++ b/docs/admin/installation-nginx.rst @@ -9,7 +9,7 @@ Install with nginx .. _nginx server configuration: https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers .. _nginx beginners guide: - http://nginx.org/en/docs/beginners_guide.html + https://nginx.org/en/docs/beginners_guide.html .. _Getting Started wiki: https://www.nginx.com/resources/wiki/start/ .. _uWSGI support from nginx: diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index 532b99752..7bd990fbf 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -112,7 +112,7 @@ Global Settings specific instance of searx, a locale can be defined using an ISO language code, like ``fr``, ``en``, ``de``. -.. _requests proxies: http://requests.readthedocs.io/en/latest/user/advanced/#proxies +.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies .. _PySocks: https://pypi.org/project/PySocks/ ``proxies`` : diff --git a/docs/conf.py b/docs/conf.py index 4b348ae0e..d6fde9bec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,11 +49,11 @@ extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '') #extlinks['role'] = ( # 'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '') extlinks['duref'] = ( - 'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '') + 'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '') extlinks['durole'] = ( - 'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '') + 'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '') extlinks['dudir'] = ( - 'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '') + 'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '') extlinks['ctan'] = ( 'https://ctan.org/pkg/%s', 'CTAN: ') diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 26f8d2bb7..90b22670c 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -117,8 +117,8 @@ Translation currently takes place on :ref:`transifex `. Documentation ============= -.. _Sphinx: http://www.sphinx-doc.org -.. _reST: http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html +.. _Sphinx: https://www.sphinx-doc.org +.. _reST: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html .. sidebar:: The reST sources diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 0f58af765..b9f5fcbbf 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -264,7 +264,7 @@ latitude latitude of result (in decimal format) longitude longitude of result (in decimal format) boundingbox boundingbox of result (array of 4. values ``[lat-min, lat-max, lon-min, lon-max]``) -geojson geojson of result (http://geojson.org) +geojson geojson of result (https://geojson.org/) osm.type type of osm-object (if OSM-Result) osm.id id of osm-object (if OSM-Result) address.name name of object diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index 3e1a5e344..14af03fa6 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -60,7 +60,7 @@ read :ref:`make test`. How to compile styles and javascript ==================================== -.. _less: http://lesscss.org/ +.. _less: https://lesscss.org/ .. _NodeJS: https://nodejs.org How to build styles diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 906a0e9af..963378748 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -1391,27 +1391,27 @@ The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac`` .. _readability: https://docs.python-guide.org/writing/style/ .. _Sphinx-Primer: - http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html + https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html .. _reST: https://docutils.sourceforge.io/rst.html .. _Sphinx Roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html -.. _Sphinx: http://www.sphinx-doc.org -.. _`sphinx-doc FAQ`: http://www.sphinx-doc.org/en/stable/faq.html +.. _Sphinx: https://www.sphinx-doc.org +.. _`sphinx-doc FAQ`: https://www.sphinx-doc.org/en/stable/faq.html .. _Sphinx markup constructs: - http://www.sphinx-doc.org/en/stable/markup/index.html + https://www.sphinx-doc.org/en/stable/markup/index.html .. _`sphinx cross references`: - http://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations + https://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations .. _sphinx.ext.extlinks: https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html -.. _intersphinx: http://www.sphinx-doc.org/en/stable/ext/intersphinx.html -.. _sphinx config: http://www.sphinx-doc.org/en/stable/config.html -.. _Sphinx's autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html +.. _intersphinx: https://www.sphinx-doc.org/en/stable/ext/intersphinx.html +.. _sphinx config: https://www.sphinx-doc.org/en/stable/config.html +.. _Sphinx's autodoc: https://www.sphinx-doc.org/en/stable/ext/autodoc.html .. _Sphinx's Python domain: - http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain + https://www.sphinx-doc.org/en/stable/domains.html#the-python-domain .. _Sphinx's C domain: - http://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs + https://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs .. _doctree: - http://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases + https://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases .. _docutils: http://docutils.sourceforge.net/docs/index.html .. _docutils FAQ: http://docutils.sourceforge.net/FAQ.html .. _linuxdoc: https://return42.github.io/linuxdoc @@ -1424,5 +1424,5 @@ The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac`` .. _ImageMagick: https://www.imagemagick.org .. _`Emacs Table Mode`: https://www.emacswiki.org/emacs/TableMode -.. _`Online Tables Generator`: http://www.tablesgenerator.com/text_tables +.. _`Online Tables Generator`: https://www.tablesgenerator.com/text_tables .. _`OASIS XML Exchange Table Model`: https://www.oasis-open.org/specs/tm9901.html diff --git a/utils/makefile.python b/utils/makefile.python index 6c6696964..668b0894b 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -252,7 +252,7 @@ pyenv-python: pyenv-install # PyPi is required and since uploads via setuptools is not recommended, we have # to imstall / use twine ... its really a mess. # -# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi +# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi # [2] https://github.com/pypa/pip/pull/1519 # https://github.com/pypa/twine