Replace Flask by Starlette

.github/workflows/integration.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
     steps:
     - name: Checkout
       uses: actions/checkout@v2

requirements.txt

@@ -9,8 +9,13 @@ python-dateutil==2.8.2
 pyyaml==5.4.1
 httpx[http2]==0.17.1
 Brotli==1.0.9
-uvloop==0.16.0; python_version >= '3.7'
+uvloop==0.16.0
-uvloop==0.14.0; python_version < '3.7'
 httpx-socks[asyncio]==0.3.1
 langdetect==1.0.9
 setproctitle==1.2.2
+starlette==0.16.0
+starlette-i18n==1.0.0
+starlette-context==0.3.3
+uvicorn[standard]==0.14.0
+gunicorn==20.1.0
+python-multipart==0.0.5

searx/__init__.py

@@ -17,11 +17,13 @@ if settings is not None:
     settings = settings_set_defaults(settings)
 
 searx_debug = settings['general']['debug']
+logging.basicConfig()
 if searx_debug:
-    logging.basicConfig(level=logging.DEBUG)
+    logging.root.setLevel(logging.DEBUG)
+    for logger_name in ('hpack.hpack', 'hpack.table', 'multipart.multipart'):
+        logging.getLogger(logger_name).setLevel(logging.WARNING)
 else:
-    logging.basicConfig(level=logging.WARNING)
+    logging.root.setLevel(logging.WARNING)
-
 logger = logging.getLogger('searx')
 logger.info(settings_load_message)
 

searx/network/client.py

@@ -226,18 +226,16 @@ def new_client(
 
 def get_loop():
     global LOOP
+    if LOOP:
         return LOOP
 
+    loop_ready = threading.Lock()
+    loop_ready.acquire()
 
-def init():
-    # log
-    for logger_name in ('hpack.hpack', 'hpack.table'):
-        logging.getLogger(logger_name).setLevel(logging.WARNING)
-
-    # loop
     def loop_thread():
         global LOOP
         LOOP = asyncio.new_event_loop()
+        loop_ready.release()
         LOOP.run_forever()
 
     thread = threading.Thread(
@@ -246,6 +244,5 @@ def init():
         daemon=True,
     )
     thread.start()
-
+    loop_ready.acquire()
-
+    return LOOP
-init()

searx/network/network.py

@@ -11,6 +11,7 @@ from itertools import cycle
 import httpx
 
 from .client import new_client, get_loop
+from searx import logger
 
 
 DEFAULT_NAME = '__DEFAULT__'
@@ -307,6 +308,8 @@ def done():
             future = asyncio.run_coroutine_threadsafe(Network.aclose_all(), loop)
             # wait 3 seconds to close the HTTP clients
             future.result(3)
+    except:
+        logger.exception('Exception while closing clients')
     finally:
         NETWORKS.clear()
 

searx/run.py

@@ -0,0 +1,71 @@
+import multiprocessing
+import uvicorn
+import uvicorn.workers
+import gunicorn.app.base
+
+from searx import settings
+
+
+class CustomUvicornWorker(uvicorn.workers.UvicornWorker):
+    CONFIG_KWARGS = {}
+
+
+class StandaloneApplication(gunicorn.app.base.BaseApplication):
+
+    def __init__(self, app, options=None):
+        self.options = options or {}
+        self.application = app
+        super().__init__()
+
+    def load_config(self):
+        config = {key: value for key, value in self.options.items()
+                if key in self.cfg.settings and value is not None}
+        for key, value in config.items():
+            self.cfg.set(key.lower(), value)
+
+    def load(self):
+        return self.application
+
+
+def number_of_workers():
+    return 1 # (multiprocessing.cpu_count() * 2) + 1
+
+
+def run_production(app):
+    config_kwargs = {
+        "loop": "uvloop",
+        "http": "httptools",
+    }
+    base_url = settings['server']['base_url'] or None
+    if base_url:
+        # ? config_kwargs['proxy_headers'] = True
+        config_kwargs['root_path'] = settings['server']['base_url']
+
+    CustomUvicornWorker.CONFIG_KWARGS.update(config_kwargs)
+
+    options = {
+        'proc_name': 'searxng',
+        'bind': '%s:%s' % (settings['server']['bind_address'], settings['server']['port']),
+        'workers': number_of_workers(),
+        'worker_class': 'searx.run.CustomUvicornWorker',
+        'loglevel': 'debug',
+        'capture_output': True,
+    }
+    StandaloneApplication(app, options).run()
+
+
+def run_debug():
+    kwargs = {
+        'reload': True,
+        'loop': 'auto',
+        'http': 'auto',
+        'ws': 'none',
+        'host': settings['server']['bind_address'],
+        'port': settings['server']['port'],
+    }
+    base_url = settings['server']['base_url']
+    if base_url:
+        kwargs['proxy_headers'] = True
+        kwargs['root_path'] = settings['server']['base_url']
+
+    uvicorn.run('searx.__main__:app', **kwargs)