[mod] add a process manager infrastructure (based on hapless)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

requirements.txt

@@ -16,3 +16,4 @@ setproctitle==1.3.2
 redis==4.3.4
 markdown-it-py==2.1.0
 typing_extensions==4.3.0
+hapless @ git+https://github.com/bmwant/hapless.git#egg=hapless

searx/services/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Services managed by SearXNG"""
+
+from pathlib import Path
+import tempfile
+
+# hapless files are stored in /tmp/SearXNG .. may be we should store them on a
+# different location.
+SEARXNG_HAP_DIR = Path(tempfile.gettempdir()) / "SearXNG"

searx/services/__main__.py

@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""A hapless command line wrapper suitable for SearXNG.
+
+In a development environment try::
+
+    $ ./manage pyenv.cmd bash
+    (py3) $ python -m searx.services --help
+"""
+
+import os
+import asyncio
+import time
+
+from pathlib import Path
+
+try:
+    from shlex import join as shlex_join
+except ImportError:
+    # Fallback for Python 3.7
+    from hapless.utils import shlex_join_backport as shlex_join
+
+from string import Template
+from typing import Optional
+
+from hapless import cli
+from hapless.main import Hapless
+
+from searx.settings_loader import load_yaml
+from . import SEARXNG_HAP_DIR
+
+SEARXNG_SERVICES_CONFIG = Path(__file__).parent / 'config.yml'
+CONFIG_ENV = {
+    'SEARXNG_ROOT': SEARXNG_SERVICES_CONFIG.parent.parent.parent,
+    'HOME': os.environ['HOME'],
+}
+
+
+class SearXNGHapless(Hapless):
+    """Adjustments to :py:class:`Hapless` class
+
+    ToDo fix upstrem: the methods should never call sys.exit
+    """
+
+    def run(self, cmd: str, name: Optional[str] = None, check: bool = False):
+        hap = self.create_hap(cmd=cmd, name=name)
+        pid = os.fork()
+        if pid == 0:
+            coro = self.run_hap(hap)
+            asyncio.run(coro)
+        else:
+            if check:
+                self._check_fast_failure(hap)
+            # sys.exit(0)
+
+
+def _parse_cmd(service_cfg):
+    cmd = []
+    for item in service_cfg.get('cmd', []):
+        cmd.append(Template(item).substitute(**CONFIG_ENV))
+    return shlex_join(cmd)
+
+
+@cli.cli.command(short_help="Start SearXNG services from YAML config (ToDo)")
+@cli.click.argument("config", metavar="config", default=SEARXNG_SERVICES_CONFIG)
+def sxng_start(config):
+    # print("START services from YAML config file --> %s" % config)
+    cfg = load_yaml(config).get('services', {})
+    cli.hapless.clean()
+    for name, service_cfg in cfg.items():
+        hap = cli.hapless.get_hap(name)
+        if hap is not None:
+            cli.console.print(
+                f"{cli.config.ICON_INFO} Hap with such name already exists: {hap}",
+                style=f"{cli.config.COLOR_ERROR} bold",
+            )
+            continue
+        cmd = _parse_cmd(service_cfg)
+        cli.hapless.run(cmd, name=name)
+
+
+@cli.cli.command(short_help="TODO: Stop SearXNG services from YAML config (ToDo)")
+@cli.click.argument("config", metavar="config", default=SEARXNG_SERVICES_CONFIG)
+def sxng_stop(config):
+    # print("STOP services from YAML config file --> %s" % config)
+    cfg = load_yaml(config).get('services', {})
+    hap_list = []
+    for name, _ in cfg.items():
+        hap = cli.hapless.get_hap(name)
+        if hap is not None:
+            hap_list.append(hap)
+    if hap_list:
+        cli.hapless.kill(hap_list)
+        # wait a second to close open handles
+        time.sleep(1)
+        cli.hapless.clean()
+
+
+#    import pdb
+#    pdb.set_trace()
+
+if __name__ == "__main__":
+    cli.hapless = SearXNGHapless(hapless_dir=SEARXNG_HAP_DIR)
+    cli.cli()  # pylint: disable=no-value-for-parameter

searx/services/config.yml

@@ -0,0 +1,13 @@
+# YAML file to configure SearXNG services
+
+services:
+
+  # service named 'test001'
+  test001:
+    cmd:
+      - python
+      - $SEARXNG_ROOT/searx/services/dummy.py
+
+  # service named 'test002'
+  test002:
+    cmd: [python, -m, searx.services.dummy]

searx/services/dummy.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""A dummy service to demonstrate SearXNG service management"""
+
+import os
+import sys
+import time
+
+
+def main():
+
+    print("This is arguments")
+    print(f"{sys.argv}")
+    print("This is environment", flush=True)
+    for key, value in os.environ.items():
+        print(f"{key} : {value}", flush=True)
+
+    # this should be running for about 2 hours
+    for i in range(1000):
+        print(f"Iteration {i}...", flush=True)
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()