version based on the git repository

This commit remove the need to update the brand for GIT_URL and GIT_BRANCH:
there are read from the git repository.

It is possible to call python -m searx.version freeze to freeze the current version.
Useful when the code is installed outside git (distro package, docker, etc...)
This commit is contained in:
Alexandre Flament 2021-07-27 18:37:46 +02:00
parent 24fcd7475a
commit 4b43775c91
16 changed files with 199 additions and 98 deletions

View File

@ -60,9 +60,6 @@ ARG VERSION_GITCOMMIT=unknown
RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
if [ ! -z $VERSION_GITCOMMIT ]; then\
echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \
fi; \
find /usr/local/searx/searx/static -a \( -name '*.html' -o -name '*.css' -o -name '*.js' \
-o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \
-type f -exec gzip -9 -k {} \+ -exec brotli --best {} \+

View File

@ -5,7 +5,7 @@ import sys, os
from pallets_sphinx_themes import ProjectLink
from searx import get_setting
from searx.version import VERSION_STRING
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
# Project --------------------------------------------------------------
@ -15,8 +15,6 @@ author = u'Adam Tauber'
release, version = VERSION_STRING, VERSION_STRING
SEARX_URL = get_setting('server.base_url') or 'https://example.org/searx'
GIT_URL = get_setting('brand.git_url')
GIT_BRANCH = get_setting('brand.git_branch')
ISSUE_URL = get_setting('brand.issue_url')
DOCS_URL = get_setting('brand.docs_url')
PUBLIC_INSTANCES = get_setting('brand.public_instances')

View File

@ -107,8 +107,8 @@ The ``make buildenv`` target will update the *build environment* in:
Tasks running outside of an *installed instance*, need the following settings
from the YAML configuration:
- ``GIT_URL`` from :ref:`brand.git_url <settings global brand>`
- ``GIT_BRANCH`` from :ref:`brand.git_branch <settings global brand>`
- ``GIT_URL`` from git configuration
- ``GIT_BRANCH`` from git configuration
- ``SEARX_URL`` from :ref:`server.base_url <settings global server>` (aka
``PUBLIC_URL``)

36
manage
View File

@ -115,8 +115,11 @@ buildenv() {
fi
fi
export SEARX_SETTINGS_PATH
(
set -e
SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
| prefix_stdout "${_Blue}BUILDENV${_creset} "
)
return "${PIPESTATUS[0]}"
}
@ -200,7 +203,6 @@ docker.build() {
local SEARX_GIT_VERSION
local VERSION_GITCOMMIT
local SEARX_PYTHON_VERSION
local GITHUB_USER
local SEARX_IMAGE_NAME
local BUILD
@ -227,34 +229,14 @@ docker.build() {
fi
# This is a git repository
# "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
# awk to remove the "v" and the "g"
SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
# add the suffix "-dirty" if the repository has uncommited change
# /!\ HACK for searx/searx: ignore utils/brand.env
git update-index -q --refresh
if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
fi
pyenv.cmd python -m searx.version freeze
eval "$(pyenv.cmd python -m searx.version)"
# Get the last git commit id, will be added to the Searx version (see Dockerfile)
VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
# Get the last git commit id
VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3)
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
# Check consistency between the git tag and the searx/version.py file
# /! HACK : parse Python file with bash /!
# otherwise it is not possible build the docker image without all Python
# dependencies ( version.py loads __init__.py )
# SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
err_msg "git tag: $SEARX_GIT_VERSION"
err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
die 1 "Inconsistency between the last git tag and the searx/version.py file"
fi
# define the docker image name
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
@ -274,14 +256,14 @@ docker.build() {
docker $BUILD \
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
--build-arg GIT_URL="${GIT_URL}" \
--build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
--build-arg SEARX_GIT_VERSION="${VERSION_STRING}" \
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
--build-arg LABEL_VCS_URL="${GIT_URL}" \
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
-t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
-t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${VERSION_STRING}" .
if [ "$1" = "push" ]; then
docker push "${SEARX_IMAGE_NAME}:latest"

View File

@ -10,11 +10,6 @@ brand:
wiki_url: https://github.com/searxng/searxng/wiki
issue_url: https://github.com/searxng/searxng/issues
# If you change a value below don't forget to rebuild instance's enviroment
# (make buildenv)
git_url: https://github.com/searxng/searxng
git_branch: master
search:
# Filter results. 0: None, 1: Moderate, 2: Strict
safe_search: 0

View File

@ -130,8 +130,6 @@ SCHEMA = {
'contact_url': SettingsValue((None, False, str), None),
},
'brand': {
'git_url': SettingsValue(str),
'git_branch': SettingsValue(str),
'issue_url': SettingsValue(str, None),
'new_issue_url': SettingsValue(str, None),
'docs_url': SettingsValue(str, None),

View File

@ -11,7 +11,7 @@
<p>More about SearXNG ...</p>
<ul>
<li><a href="{{ get_setting('brand.git_url') }}">SearXNG sources</a></li>
<li><a href="{{ searx_git_url }}">SearXNG sources</a></li>
<li><a href="https://www.transifex.com/projects/p/searx/">transifex</a></li>
</ul>
@ -30,7 +30,7 @@
</li>
<li>
SearXNG is free software, the code is 100% open and you can help to make
it better. See more on <a href="{{ get_setting('brand.git_url')
it better. See more on <a href="{{ searx_git_url
}}">SearXNG sources</a>.
</li>
</ul>
@ -81,7 +81,7 @@
<p>
SearXNG appreciates your concern regarding logs, so take the code from
the <a href="{{ get_setting('brand.git_url') }}">SearXNG project</a> and
the <a href="{{ searx_git_url }}">SearXNG project</a> and
run it yourself!
</p>
<p>
@ -95,7 +95,7 @@
<p>
See the <a href="{{ get_setting('brand.docs_url') }}">SearXNG docs</a>
and <a href="{{ get_setting('brand.git_url') }}">SearXNG sources</a>
and <a href="{{ searx_git_url }}">SearXNG sources</a>
</p>
</div>

View File

@ -83,7 +83,7 @@
<p class="text-muted">
<small>
{{ _('Powered by') }} <a href="{{ get_setting('brand.docs_url') }}">SearXNG</a> - {{ searx_version }} - {{ _('a privacy-respecting, hackable metasearch engine') }}<br/>
<a href="{{ get_setting('brand.git_url') }}">{{ _('Source code') }}</a> |
<a href="{{ searx_git_url }}">{{ _('Source code') }}</a> |
<a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> |
<a href="{{ get_setting('brand.public_instances') }}">{{ _('Public instances') }}</a>{% if get_setting('general.contact_url') %} |
<a href="{{ get_setting('general.contact_url') }}">{{ _('Contact instance maintainer') }}</a>{% endif %}

View File

@ -51,7 +51,7 @@
<footer>
<p>
{{ _('Powered by') }} <a href="{{ url_for('about') }}">searxng</a> - {{ searx_version }} — {{ _('a privacy-respecting, hackable metasearch engine') }}<br/>
<a href="{{ get_setting('brand.git_url') }}">{{ _('Source code') }}</a> |
<a href="{{ searx_git_url }}">{{ _('Source code') }}</a> |
<a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> |
<a href="{{ get_setting('brand.public_instances') }}">{{ _('Public instances') }}</a>{% if get_setting('general.contact_url') %} |
<a href="{{ get_setting('general.contact_url') }}">{{ _('Contact instance maintainer') }}</a>{% endif %}

View File

@ -16,7 +16,7 @@ from babel.core import get_global
from searx import settings
from searx.data import USER_AGENTS
from searx.version import VERSION_STRING
from searx.version import VERSION_TAG
from searx.languages import language_codes
from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
from searx import logger
@ -44,7 +44,7 @@ NOTSET = NotSetClass()
def searx_useragent():
"""Return the searx User Agent"""
return 'searx/{searx_version} {suffix}'.format(
searx_version=VERSION_STRING,
searx_version=VERSION_TAG,
suffix=settings['outgoing']['useragent_suffix']).strip()

View File

@ -1,26 +1,142 @@
# -*- coding: utf-8 -*-
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=missing-function-docstring,missing-module-docstring,missing-class-docstring
searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
import re
import os
import shlex
import subprocess
import logging
You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.
# fallback values
# if there is searx.version_frozen module, and it is not possible to get the git tag
VERSION_STRING = "1.0.0"
VERSION_TAG = "1.0.0"
GIT_URL = "unknow"
GIT_BRANCH = "unknow"
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
logger = logging.getLogger("searx")
# version of searx
VERSION_MAJOR = 1
VERSION_MINOR = 0
VERSION_BUILD = 0
SUBPROCESS_RUN_ENV = {
"PATH": os.environ["PATH"],
"LC_ALL": "C",
"LANGUAGE": "",
}
VERSION_STRING = "{0}.{1}.{2}".format(VERSION_MAJOR,
VERSION_MINOR,
VERSION_BUILD)
def subprocess_run(args, **kwargs):
"""Call :py:func:`subprocess.run` and return (striped) stdout. If returncode is
non-zero, raise a :py:func:`subprocess.CalledProcessError`.
"""
if not isinstance(args, (list, tuple)):
args = shlex.split(args)
kwargs["env"] = kwargs.get("env", SUBPROCESS_RUN_ENV)
kwargs["encoding"] = kwargs.get("encoding", "utf-8")
kwargs["stdout"] = subprocess.PIPE
kwargs["stderr"] = subprocess.PIPE
# raise CalledProcessError if returncode is non-zero
kwargs["check"] = True
proc = subprocess.run(args, **kwargs) # pylint: disable=subprocess-run-check
return proc.stdout.strip()
def get_git_url_and_branch():
try:
ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}")
except subprocess.CalledProcessError:
ref = subprocess_run("git rev-parse --abbrev-ref master@{upstream}")
origin, git_branch = ref.split("/", 1)
git_url = subprocess_run(["git", "remote", "get-url", origin])
# get https:// url from git@ url
if git_url.startswith("git@"):
git_url = git_url.replace(":", "/", 2).replace("git@", "https://", 1)
if git_url.endswith(".git"):
git_url = git_url.replace(".git", "", 1)
return git_url, git_branch
def get_git_version():
try:
tag = subprocess_run("git describe HEAD")
# a. HEAD is on tag name, example: tag = "v1.0.1"
# b. HEAD is not a tag name, example "<tag>-<distance>-g<commit>"
tag_version, tag_distance, tag_commit = (tag.split("-") + ["", ""])[:3]
if re.match(r"v[0-9]+\.[0-9]+\.[0-9]+", tag_version):
# tag_version "v1.0.0" becomes "1.0.0" (without the v)
# other patterns are kept untouched
tag_version = tag_version[1:]
# remove "g" prefix from tag_commit
if tag_commit and tag_commit[0] == "g":
tag_commit = tag_commit[1:]
# set git_version to "1.0.0-590-0686e274" or '1.0.0'
git_version = "-".join(filter(bool, [tag_version, tag_distance, tag_commit]))
except subprocess.CalledProcessError:
# fall back to "YYYY.MM.DD.Hash" if there is no tag at all
git_version = subprocess_run(r"git show -s --format='%as-%h'")
# PEP 440: replace - with .
tag_version = git_version = git_version.replace("-", ".")
# add "-dirty" suffix if there are uncommited changes except searx/settings.yml
try:
subprocess_run(
"git diff --quiet -- . ':!searx/settings.yml' ':!utils/brand.env'"
)
except subprocess.CalledProcessError as e:
if e.returncode == 1:
git_version += "-dirty"
else:
logger.warning(
'"%s" returns an unexpected return code %i', e.returncode, e.cmd
)
return git_version, tag_version
try:
from searx.version_frozen import VERSION_STRING, VERSION_TAG, GIT_URL, GIT_BRANCH
except ImportError:
try:
try:
VERSION_STRING, VERSION_TAG = get_git_version()
except subprocess.CalledProcessError as ex:
logger.error("Error while getting the version: %s", ex.stderr)
try:
GIT_URL, GIT_BRANCH = get_git_url_and_branch()
except subprocess.CalledProcessError as ex:
logger.error("Error while getting the git URL & branch: %s", ex.stderr)
except FileNotFoundError as ex:
logger.error("%s is not found, fallback to the default version", ex.filename)
logger.info("version: %s", VERSION_STRING)
if __name__ == "__main__":
import sys
if len(sys.argv) >= 2 and sys.argv[1] == "freeze":
# freeze the version (to create an archive outside a git repository)
python_code = f"""# SPDX-License-Identifier: AGPL-3.0-or-later
# this file is generated automatically by searx/version.py
VERSION_STRING = "{VERSION_STRING}"
VERSION_TAG = "{VERSION_TAG}"
GIT_URL = "{GIT_URL}"
GIT_BRANCH = "{GIT_BRANCH}"
"""
with open(
os.path.join(os.path.dirname(__file__), "version_frozen.py"), "w"
) as f:
f.write(python_code)
print(f"{f.name} created")
else:
# output shell code to set the variables
# usage: eval "$(python -m searx.version)"
shell_code = f"""
VERSION_STRING="{VERSION_STRING}"
VERSION_TAG="{VERSION_TAG}"
GIT_URL="{GIT_URL}"
GIT_BRANCH="{GIT_BRANCH}"
"""
print(shell_code)

View File

@ -82,7 +82,7 @@ from searx.utils import (
dict_subset,
match_language,
)
from searx.version import VERSION_STRING
from searx.version import VERSION_STRING, GIT_URL
from searx.query import RawTextQuery
from searx.plugins import plugins
from searx.plugins.oa_doi_rewrite import get_doi_resolver
@ -475,6 +475,7 @@ def render(template_name, override_theme=None, **kwargs):
]
kwargs['instance_name'] = get_setting('general.instance_name')
kwargs['searx_version'] = VERSION_STRING
kwargs['searx_git_url'] = GIT_URL
kwargs['get_setting'] = get_setting
# helpers to create links to other pages
@ -1322,7 +1323,7 @@ def config():
'version': VERSION_STRING,
'brand': {
'CONTACT_URL': get_setting('general.contact_url'),
'GIT_URL': get_setting('brand.git_url'),
'GIT_URL': GIT_URL,
'DOCS_URL': get_setting('brand.docs_url'),
},
'doi_resolvers': list(settings['doi_resolvers'].keys()),

View File

@ -4,10 +4,7 @@
from setuptools import setup
from setuptools import find_packages
import os
import sys
from searx.version import VERSION_STRING
from searx.version import VERSION_TAG, GIT_URL
from searx import get_setting
with open('README.rst', encoding='utf-8') as f:
@ -21,12 +18,12 @@ with open('requirements-dev.txt') as f:
setup(
name='searx',
version=VERSION_STRING,
version=VERSION_TAG,
description="A privacy-respecting, hackable metasearch engine",
long_description=long_description,
url=get_setting('brand.docs_url'),
project_urls={
"Code": get_setting('brand.git_url'),
"Code": GIT_URL,
"Issue tracker": get_setting('brand.issue_url')
},
classifiers=[

View File

@ -1,5 +1,5 @@
export GIT_URL='https://github.com/searxng/searxng'
export GIT_BRANCH='master'
export SEARX_URL=''
export SEARX_PORT='8888'
export SEARX_BIND_ADDRESS='127.0.0.1'
export GIT_URL='https://github.com/searxng/searxng'
export GIT_BRANCH='master'

View File

@ -7,6 +7,8 @@ import sys
import os
from os.path import realpath, dirname, join, sep, abspath
from searx.version import GIT_URL, GIT_BRANCH
repo_root = realpath(dirname(realpath(__file__)) + sep + '..')
sys.path.insert(0, repo_root)
@ -30,9 +32,6 @@ def _env(*arg, **kwargs):
name_val = [
('GIT_URL' , 'brand.git_url'),
('GIT_BRANCH' , 'brand.git_branch'),
('SEARX_URL' , 'server.base_url'),
('SEARX_PORT' , 'server.port'),
('SEARX_BIND_ADDRESS' , 'server.bind_address'),
@ -57,8 +56,9 @@ from searx import get_setting
print('build %s (settings from: %s)' % (brand_env, os.environ['SEARX_SETTINGS_PATH']))
sys.path.insert(0, repo_root)
from searx import settings
with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f:
for name, option in name_val:
print("export %s='%s'" % (name, _env(option)), file=f)
print(f"export GIT_URL='{GIT_URL}'", file=f)
print(f"export GIT_BRANCH='{GIT_BRANCH}'", file=f)

View File

@ -429,13 +429,11 @@ EOF
popd > /dev/null
}
# shellcheck disable=SC2034
prompt_installation_status(){
local state branch remote remote_url instance_setting
local state branch remote remote_url GIT_URL GIT_BRANCH VERSION_STRING VERSION_TAG
state="$(install_searx_get_state)"
branch="$(git name-rev --name-only HEAD)"
remote="$(git config branch."${branch}".remote)"
remote_url="$(git config remote."${remote}".url)"
case $state in
missing-searx-clone)
@ -445,14 +443,16 @@ prompt_installation_status(){
*)
warn_msg "SearXNG instance already installed at: $SEARX_SRC"
warn_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} "
instance_setting="$(prompt_installation_setting brand.git_url)"
if ! [ "$instance_setting" = "$remote_url" ]; then
warn_msg "instance's brand.git_url: '${instance_setting}'" \
branch="$(git name-rev --name-only HEAD)"
remote="$(git config branch."${branch}".remote)"
remote_url="$(git config remote."${remote}".url)"
eval "$(get_installed_version_variables)"
if ! [ "$GIT_URL" = "$remote_url" ]; then
warn_msg "instance's git URL: '${GIT_URL}'" \
"differs from local clone's remote URL: ${remote_url}"
fi
instance_setting="$(prompt_installation_setting brand.git_branch)"
if ! [ "$instance_setting" = "$branch" ]; then
warn_msg "instance brand.git_branch: ${instance_setting}" \
if ! [ "$GIT_BRANCH" = "$branch" ]; then
warn_msg "instance git branch: ${GIT_BRANCH}" \
"differs from local clone's branch: ${branch}"
fi
return 42
@ -469,7 +469,7 @@ verify_continue_install(){
prompt_installation_setting(){
# usage: prompt_installation_setting brand.git_url
# usage: prompt_installation_setting brand.docs_url
#
# Prompts the value of the (YAML) setting in the SearXNG instance.
@ -496,6 +496,23 @@ EOF
esac
}
get_installed_version_variables() {
# usage: eval "$(get_installed_version_variables)"
#
# Set variables VERSION_STRING, VERSION_TAG, GIT_URL, GIT_BRANCH
local _state
_state="$(install_searx_get_state)"
case $_state in
python-installed|installer-modified)
sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python -m searx.version";;
*)
return 42
;;
esac
}
init_SEARX_SRC(){
rst_title "Update instance: ${SEARX_SRC}/" section