Merge pull request #375 from dalf/searxng_extra

SearXNG: searx_extra

.github/workflows/data-update.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: Fetch data
         env:
-          FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
+          FETCH_SCRIPT: ./searxng_extra/update/${{ matrix.fetch }}
         run: |
           V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
 

docs/index.rst

@@ -32,7 +32,7 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`.
    user/index
    admin/index
    dev/index
-   searx_extra/index
+   searxng_extra/index
    utils/index
    src/index
 

docs/searx_extra/standalone_searx.py.rst

@@ -1,9 +0,0 @@
-
-.. _standalone_searx.py:
-
-===================================
-``searx_extra/standalone_searx.py``
-===================================
-
-.. automodule:: searx_extra.standalone_searx
-  :members:

docs/searxng_extra/index.rst

@@ -1,10 +1,10 @@
-.. _searx_extra:
+.. _searxng_extra:
 
 ======================================================
-Tooling box ``searx_extra`` for developers and users
+Tooling box ``searxng_extra`` for developers and users
 ======================================================
 
-In the folder :origin:`searx_extra/` we maintain some tools useful for
+In the folder :origin:`searxng_extra/` we maintain some tools useful for
 developers and users.
 
 .. toctree::

docs/searxng_extra/standalone_searx.py.rst

@@ -0,0 +1,9 @@
+
+.. _standalone_searx.py:
+
+=====================================
+``searxng_extra/standalone_searx.py``
+=====================================
+
+.. automodule:: searxng_extra.standalone_searx
+  :members:

manage

@@ -25,7 +25,7 @@ pylint.FILES() {
     # These py files are linted by test.pylint(), all other files are linted by
     # test.pep8()
 
-    grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searx_extra tests
+    grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
 }
 
 YAMLLINT_FILES=()
@@ -328,11 +328,11 @@ data.all() {
         data.useragents
         data.osm_keys_tags
         build_msg DATA "update searx/data/ahmia_blacklist.txt"
-        python searx_extra/update/update_ahmia_blacklist.py
+        python searxng_extra/update/update_ahmia_blacklist.py
         build_msg DATA "update searx/data/wikidata_units.json"
-        python searx_extra/update/update_wikidata_units.py
+        python searxng_extra/update/update_wikidata_units.py
         build_msg DATA "update searx/data/currencies.json"
-        python searx_extra/update/update_currencies.py
+        python searxng_extra/update/update_currencies.py
     )
 }
 
@@ -341,7 +341,7 @@ data.languages() {
     (   set -e
         pyenv.activate
         build_msg ENGINES "fetch languages .."
-        python searx_extra/update/update_languages.py
+        python searxng_extra/update/update_languages.py
         build_msg ENGINES "update update searx/languages.py"
         build_msg DATA "update searx/data/engines_languages.json"
     )
@@ -350,13 +350,13 @@ data.languages() {
 
 data.useragents() {
     build_msg DATA "update searx/data/useragents.json"
-    pyenv.cmd python searx_extra/update/update_firefox_version.py
+    pyenv.cmd python searxng_extra/update/update_firefox_version.py
     dump_return $?
 }
 
 data.osm_keys_tags() {
     build_msg DATA "update searx/data/osm_keys_tags.json"
-    pyenv.cmd python searx_extra/update/update_osm_keys_tags.py
+    pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
     dump_return $?
 }
 
@@ -524,8 +524,8 @@ node.clean() {
 }
 
 pygments.less() {
-    build_msg PYGMENTS "searx_extra/update/update_pygments.py"
+    build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
-    if ! pyenv.cmd python searx_extra/update/update_pygments.py; then
+    if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
         build_msg PYGMENTS "building LESS files for pygments failed"
         return 1
     fi

searx/data/__init__.py

@@ -31,7 +31,7 @@ def ahmia_blacklist_loader():
     """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion
     names.  The MD5 values are fetched by::
 
-      searx_extra/update/update_ahmia_blacklist.py
+      searxng_extra/update/update_ahmia_blacklist.py
 
     This function is used by :py:mod:`searx.plugins.ahmia_filter`.
 

searx/static/themes/oscar/css/logicodev-dark.css

@@ -383,7 +383,7 @@
   width: 10rem;
 }
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 .code-highlight .linenos {

searx/static/themes/oscar/css/logicodev.css

@@ -383,7 +383,7 @@
   width: 10rem;
 }
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 .code-highlight .linenos {

searx/static/themes/oscar/css/pointhi.css

@@ -383,7 +383,7 @@
   width: 10rem;
 }
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 .code-highlight .linenos {

searx/static/themes/oscar/src/generated/pygments-logicodev.less

@@ -1,5 +1,5 @@
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 

searx/static/themes/oscar/src/generated/pygments-pointhi.less

@@ -1,5 +1,5 @@
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 

searx/static/themes/simple/css/searxng-rtl.css

@@ -829,7 +829,7 @@ template {
   border: inherit;
 }
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 .code-highlight .linenos {

searx/static/themes/simple/css/searxng.css

@@ -829,7 +829,7 @@ template {
   border: inherit;
 }
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 .code-highlight .linenos {

searx/static/themes/simple/src/generated/pygments.less

@@ -1,5 +1,5 @@
 /*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version 2.10.0
 */
 

searxng_extra/google_search.py

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 from sys import argv, exit
 
 if not len(argv) > 1:

searxng_extra/standalone_searx.py

@@ -1,5 +1,11 @@
 #!/usr/bin/env python
-"""Script to run searx from terminal.
+# lint: pylint
+
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# (C) Copyright Contributors to the SearXNG project.
+# (C) Copyright Contributors to the searx project (2014 - 2021)
+
+"""Script to run SearXNG from terminal.
 
 Getting categories without initiate the engine will only return `['general']`
 
@@ -15,7 +21,7 @@ Example to use this script:
 
 .. code::  bash
 
-    $ python3 searx_extra/standalone_searx.py rain
+    $ python3 searxng_extra/standalone_searx.py rain
 
 Example to run it from python:
 
@@ -31,7 +37,7 @@ Example to run it from python:
 ... engine_cs = list(searx.engines.categories.keys())
 ... # load module
 ... spec = importlib.util.spec_from_file_location(
-...     'utils.standalone_searx', 'searx_extra/standalone_searx.py')
+...     'utils.standalone_searx', 'searxng_extra/standalone_searx.py')
 ... sas = importlib.util.module_from_spec(spec)
 ... spec.loader.exec_module(sas)
 ... # use function from module
@@ -56,25 +62,8 @@ Example to run it from python:
     },
     "suggestions": [...]
 }
-"""  # noqa: E501
+""" # pylint: disable=line-too-long
-# pylint: disable=pointless-string-statement
-'''
-searx is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
 
-searx is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
-
-(C) 2016- by Alexandre Flament, <alex@al-f.net>
-'''
-# pylint: disable=wrong-import-position
 import argparse
 import sys
 from datetime import datetime

searxng_extra/update/update_ahmia_blacklist.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 # This script saves Ahmia's blacklist for onion sites.
 # More info in https://ahmia.fi/blacklist/

searxng_extra/update/update_currencies.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 import re
 import unicodedata

searxng_extra/update/update_engine_descriptions.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 import json
 from urllib.parse import urlparse

searxng_extra/update/update_external_bangs.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # lint: pylint
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
 Update searx/data/external_bangs.json using the duckduckgo bangs.
 

searxng_extra/update/update_firefox_version.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 import json
 import requests
@@ -16,7 +17,7 @@ NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?$')
 # BETA_REGEX = re.compile('.*[0-9]b([0-9\-a-z]+)$')
 # ESR_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?esr$')
 
-# 
+#
 useragents = {
     "versions": (),
     "os": ('Windows NT 10.0; WOW64',

searxng_extra/update/update_languages.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 # This script generates languages.py from intersecting each engine's supported languages.
 #

searxng_extra/update/update_osm_keys_tags.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # lint: pylint
-# pylint: disable=missing-function-docstring
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """Fetch OSM keys and tags.
 
 To get the i18n names, the scripts uses `Wikidata Query Service`_ instead of for

searxng_extra/update/update_pygments.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
 Update pygments style
 
@@ -111,7 +112,7 @@ def get_output_filename(relative_name):
 
 def get_css(cssclass, style):
     result = f"""/*
-   this file is generated automatically by searx_extra/update/update_pygments.py
+   this file is generated automatically by searxng_extra/update/update_pygments.py
    using pygments version {pygments.__version__}
 */\n\n"""
     css_text = HtmlFormatter(style=style).get_style_defs(cssclass)

searxng_extra/update/update_wikidata_units.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+# pylint: disable=missing-module-docstring
 
 import json
 import collections
@@ -51,5 +54,5 @@ def get_wikidata_units_filename():
     return join(join(searx_dir, "data"), "wikidata_units.json")
 
 
-with open(get_wikidata_units_filename(), 'w') as f:
+with open(get_wikidata_units_filename(), 'w', encoding="utf8") as f:
     json.dump(get_data(), f, indent=4, ensure_ascii=False)

setup.py

@@ -38,7 +38,7 @@ setup(
     author='Adam Tauber',
     author_email='asciimoo@gmail.com',
     license='GNU Affero General Public License',
-    packages=find_packages(exclude=["tests*", "searx_extra"]),
+    packages=find_packages(exclude=["tests*", "searxng_extra"]),
     zip_safe=False,
     install_requires=requirements,
     extras_require={

tests/unit/test_standalone_searx.py

@@ -8,7 +8,7 @@ from mock import Mock, patch
 from nose2.tools import params
 
 from searx.search import SearchQuery, EngineRef, initialize
-from searx_extra import standalone_searx as sas
+from searxng_extra import standalone_searx as sas
 from tests import SearxTestCase