Merge pull request #932 from dalf/static-file-hash

static files: add a hash as a query

dockerfiles/uwsgi.ini

@@ -44,9 +44,9 @@ logto = /var/log/uwsgi/uwsgi.log
 add-header = Connection: close
 
 # uwsgi serves the static files
-# expires set to one day as Flask does
+# expires set to one year since there are hashes
 static-map = /static=/usr/local/searxng/searx/static
-static-expires = /* 864000
+static-expires = /* 31557600
 static-gzip-all = True
 offload-threads = %k
 

searx/webapp.py

@@ -16,7 +16,7 @@ from timeit import default_timer
 from html import escape
 from io import StringIO
 import typing
-from typing import List, Dict, Iterable
+from typing import List, Dict, Iterable, Optional
 
 import urllib
 import urllib.parse
@@ -348,7 +348,7 @@ def code_highlighter(codelines, language=None):
     return html_code
 
 
-def get_current_theme_name(override: str = None) -> str:
+def get_current_theme_name(override: Optional[str] = None) -> str:
     """Returns theme name.
 
     Checks in this order:
@@ -373,14 +373,16 @@ def get_result_template(theme_name: str, template_name: str):
     return 'result_templates/' + template_name
 
 
-def url_for_theme(endpoint: str, override_theme: str = None, **values):
+def url_for_theme(endpoint: str, override_theme: Optional[str] = None, **values):
+    suffix = ""
     if endpoint == 'static' and values.get('filename'):
         theme_name = get_current_theme_name(override=override_theme)
         filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
-        if filename_with_theme in static_files:
+        file_hash = static_files.get(filename_with_theme)
+        if file_hash:
             values['filename'] = filename_with_theme
-    url = url_for(endpoint, **values)
-    return url
+            suffix = "?" + file_hash
+    return url_for(endpoint, **values) + suffix
 
 
 def proxify(url: str):

searx/webutils.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 import os
+import pathlib
 import csv
 import hashlib
 import hmac
 import re
 import inspect
 import itertools
-from typing import Iterable, List, Tuple
+from typing import Iterable, List, Tuple, Dict
 
 from io import StringIO
 from codecs import getincrementalencoder
@@ -58,13 +59,29 @@ def get_themes(templates_path):
     return themes
 
 
-def get_static_files(static_path):
-    static_files = set()
-    static_path_length = len(static_path) + 1
-    for directory, _, files in os.walk(static_path):
-        for filename in files:
-            f = os.path.join(directory[static_path_length:], filename)
-            static_files.add(f)
+def get_hash_for_file(file: pathlib.Path) -> str:
+    m = hashlib.sha1()
+    with file.open('rb') as f:
+        m.update(f.read())
+    return m.hexdigest()
+
+
+def get_static_files(static_path: str) -> Dict[str, str]:
+    static_files: Dict[str, str] = {}
+    static_path_path = pathlib.Path(static_path)
+
+    def walk(path: pathlib.Path):
+        for file in path.iterdir():
+            if file.name.startswith('.'):
+                # ignore hidden file
+                continue
+            if file.is_file():
+                static_files[str(file.relative_to(static_path_path))] = get_hash_for_file(file)
+            if file.is_dir() and file.name not in ('node_modules', 'src'):
+                # ignore "src" and "node_modules" directories
+                walk(file)
+
+    walk(static_path_path)
     return static_files
 
 

tests/unit/test_webapp.py

@@ -23,6 +23,11 @@ class ViewsTestCase(SearxTestCase):
         webapp.app.config['TESTING'] = True  # to get better error messages
         self.app = webapp.app.test_client()
 
+        # remove sha for the static file
+        # so the tests don't have to care about the changing URLs
+        for k in webapp.static_files:
+            webapp.static_files[k] = None
+
         # set some defaults
         test_results = [
             {