diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 000000000..d7ec87921 --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,133 @@ +;;; .dir-locals.el +;; +;; If you get ``*** EPC Error ***`` (even after a jedi:install-server) in your +;; emacs session, mostly you have jedi-mode enabled but the python enviroment is +;; missed. The python environment has to be next to the +;; ``/.dir-locals.el`` in:: +;; +;; ./local/py3 +;; +;; In Emacs, some buffer locals are referencing the project environment: +;; +;; - prj-root --> / +;; - python-environment-directory --> /local +;; - python-environment-default-root-name --> py3 +;; - python-shell-virtualenv-root --> /local/py3 +;; When this variable is set with the path of the virtualenv to use, +;; `process-environment' and `exec-path' get proper values in order to run +;; shells inside the specified virtualenv, example:: +;; (setq python-shell-virtualenv-root "/path/to/env/") +;; +;; To setup such an environment build target 'pyenv' or 'pyenvinstall':: +;; +;; $ make pyenvinstall +;; +;; Alternatively create the virtualenv, source it and install jedi + epc +;; (required by `emacs-jedi `_):: +;; +;; $ virtualenv --python=python3 "--no-site-packages" ./local/py3 +;; ... +;; $ source ./local/py3/bin/activate +;; (py3)$ # now install into the activated 'py3' environment .. +;; (py3)$ pip install jedi epc +;; ... +;; +;; Here is what also I found useful to add to my .emacs:: +;; +;; (global-set-key [f6] 'flycheck-mode) +;; (add-hook 'python-mode-hook 'my:python-mode-hook) +;; +;; (defun my:python-mode-hook () +;; (add-to-list 'company-backends 'company-jedi) +;; (require 'jedi-core) +;; (jedi:setup) +;; (define-key python-mode-map (kbd "C-c C-d") 'jedi:show-doc) +;; (define-key python-mode-map (kbd "M-.") 'jedi:goto-definition) +;; (define-key python-mode-map (kbd "M-,") 'jedi:goto-definition-pop-marker) +;; ) +;; + +((nil + . ((fill-column . 80) + )) + (python-mode + . ((indent-tabs-mode . nil) + + ;; project root folder is where the `.dir-locals.el' is located + (eval . (setq-local + prj-root (locate-dominating-file default-directory ".dir-locals.el"))) + + (eval . (setq-local + python-environment-directory (expand-file-name "./local" prj-root))) + + ;; use 'py3' enviroment as default + (eval . (setq-local + python-environment-default-root-name "py3")) + + (eval . (setq-local + python-shell-virtualenv-root + (concat python-environment-directory + "/" + python-environment-default-root-name))) + + ;; python-shell-virtualenv-path is obsolete, use python-shell-virtualenv-root! + ;; (eval . (setq-local + ;; python-shell-virtualenv-path python-shell-virtualenv-root)) + + (eval . (setq-local + python-shell-interpreter + (expand-file-name "bin/python" python-shell-virtualenv-root))) + + (eval . (setq-local + python-environment-virtualenv + (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root) + ;;"--system-site-packages" + "--quiet"))) + + (eval . (setq-local + pylint-command + (expand-file-name "bin/pylint" python-shell-virtualenv-root))) + + ;; pylint will find the '.pylintrc' file next to the CWD + ;; https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options + (eval . (setq-local + flycheck-pylintrc ".pylintrc")) + + ;; flycheck & other python stuff should use the local py3 environment + (eval . (setq-local + flycheck-python-pylint-executable python-shell-interpreter)) + + ;; use 'M-x jedi:show-setup-info' and 'M-x epc:controller' to inspect jedi server + + ;; https://tkf.github.io/emacs-jedi/latest/#jedi:environment-root -- You + ;; can specify a full path instead of a name (relative path). In that case, + ;; python-environment-directory is ignored and Python virtual environment + ;; is created at the specified path. + (eval . (setq-local jedi:environment-root python-shell-virtualenv-root)) + + ;; https://tkf.github.io/emacs-jedi/latest/#jedi:server-command + (eval .(setq-local + jedi:server-command + (list python-shell-interpreter + jedi:server-script) + )) + + ;; jedi:environment-virtualenv --> see above 'python-environment-virtualenv' + ;; is set buffer local! No need to setup jedi:environment-virtualenv: + ;; + ;; Virtualenv command to use. A list of string. If it is nil, + ;; python-environment-virtualenv is used instead. You must set non-nil + ;; value to jedi:environment-root in order to make this setting work. + ;; + ;; https://tkf.github.io/emacs-jedi/latest/#jedi:environment-virtualenv + ;; + ;; (eval . (setq-local + ;; jedi:environment-virtualenv + ;; (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root) + ;; ;;"--python" + ;; ;;"/usr/bin/python3.4" + ;; ))) + + ;; jedi:server-args + + ))) diff --git a/.gitignore b/.gitignore index db20da83e..828856f4c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ setup.cfg node_modules/ .tx/ + +local/ +searx.egg-info/ diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..3b4adb2ca --- /dev/null +++ b/.pylintrc @@ -0,0 +1,444 @@ +# -*- coding: utf-8; mode: conf -*- +# lint Python modules using external checkers. +# +# This is the main checker controlling the other ones and the reports +# generation. It is itself both a raw checker and an astng checker in order +# to: +# * handle message activation / deactivation at the module level +# * handle some basic but necessary stats'data (number of classes, methods...) +# +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS, .git, .svn + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=bad-whitespace, duplicate-code + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details + +# HINT: do not set this here, use argument --msg-template=... +#msg-template={path}:{line}: [{msg_id}({symbol}),{obj}] {msg} + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. + +# HINT: do not set this here, use argument --output-format=... +#output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Naming hint for argument names +argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*)|([A-Z0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$ +#const-rgx=[f]?[A-Z_][a-zA-Z0-9_]{2,30}$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,log,cfg,id + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct module names +#module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +module-rgx=([a-z_][a-z0-9_]*)$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*)|([a-z]))$ + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=2000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement.No config file found, using default configuration + +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Maximum number of attributes for a class (see R0902). +max-attributes=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..77ffe489f --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +PYOBJECTS = searx +PY_SETUP_EXTRAS ?= \[test\] + +include utils/makefile.include +include utils/makefile.python + +all: clean install + +PHONY += help +help: + @echo ' test - run developer tests' + @echo ' run - run developer instance' + @echo ' install - developer install (./local)' + @echo ' uninstall - uninstall (./local)' + @echo '' + @$(MAKE) -s -f utils/makefile.include make-help + @echo '' + @$(MAKE) -s -f utils/makefile.python python-help + +PHONY += install +install: pyenvinstall + +PHONY += uninstall +uninstall: pyenvuninstall + +PHONY += clean +clean: pyclean + $(call cmd,common_clean) + +PHONY += run +run: pyenvinstall + $(Q) ( \ + sed -i -e "s/debug : False/debug : True/g" ./searx/settings.yml ; \ + sleep 2 ; \ + xdg-open http://127.0.0.1:8888/ ; \ + sleep 3 ; \ + sed -i -e "s/debug : True/debug : False/g" ./searx/settings.yml ; \ + ) & + $(PY_ENV)/bin/python ./searx/webapp.py + +# test +# ---- + +PHONY += test test.pylint test.pep8 test.unit test.robot + +# TODO: balance linting with pylint +test: test.pep8 test.unit test.robot + - make pylint + +test.pep8: pyenvinstall + $(PY_ENV_ACT); ./manage.sh pep8_check + +test.unit: pyenvinstall + $(PY_ENV_ACT); ./manage.sh unit_tests + +test.robot: pyenvinstall + $(PY_ENV_ACT); ./manage.sh install_geckodriver + $(PY_ENV_ACT); ./manage.sh robot_tests + +.PHONY: $(PHONY) diff --git a/setup.py b/setup.py index 7333551fe..bd3dd5d1c 100644 --- a/setup.py +++ b/setup.py @@ -11,14 +11,14 @@ import sys sys.path.insert(0, './searx') from version import VERSION_STRING +with open('README.rst') as f: + long_description = f.read() -def read(*rnames): - return open(os.path.join(os.path.dirname(__file__), *rnames)).read() +with open('requirements.txt') as f: + requirements = [ l.strip() for l in f.readlines()] - -long_description = read('README.rst') -requirements = map(str.strip, open('requirements.txt').readlines()) -dev_requirements = map(str.strip, open('requirements-dev.txt').readlines()) +with open('requirements-dev.txt') as f: + dev_requirements = [ l.strip() for l in f.readlines()] setup( name='searx', diff --git a/utils/makefile.include b/utils/makefile.include new file mode 100644 index 000000000..716889c02 --- /dev/null +++ b/utils/makefile.include @@ -0,0 +1,128 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +make-help: + @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' + @echo ' make V=2 [targets] 2 => give reason for rebuild of target' + +quiet_cmd_common_clean = CLEAN $@ + cmd_common_clean = \ + rm -rf tests/build ;\ + find . -name '*.orig' -exec rm -f {} + ;\ + find . -name '*.rej' -exec rm -f {} + ;\ + find . -name '*~' -exec rm -f {} + ;\ + find . -name '*.bak' -exec rm -f {} + ;\ + +FMT = cat +ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0) +FMT = fmt +endif + +# MS-Windows +# +# For a minimal *make-environment*, I'am using the gnu-tools from: +# +# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc. +# https://github.com/gnu-mcu-eclipse/windows-build-tools/releases +# +# - git for Windows, which brings 'find', 'grep' etc. +# https://git-scm.com/download/win + + +# normpath +# +# System-dependent normalization of the path name +# +# usage: $(call normpath,/path/to/file) + +normpath = $1 +ifeq ($(OS),Windows_NT) + normpath = $(subst /,\,$1) +endif + + +# stolen from linux/Makefile +# + +ifeq ("$(origin V)", "command line") + KBUILD_VERBOSE = $(V) +endif +ifndef KBUILD_VERBOSE + KBUILD_VERBOSE = 0 +endif + +ifeq ($(KBUILD_VERBOSE),1) + quiet = + Q = +else + quiet=quiet_ + Q = @ +endif + +# stolen from linux/scripts/Kbuild.include +# + +# Convenient variables +comma := , +quote := " +#" this comment is only for emacs highlighting +squote := ' +#' this comment is only for emacs highlighting +empty := +space := $(empty) $(empty) +space_escape := _-_SPACE_-_ + +# Find any prerequisites that is newer than target or that does not exist. +# PHONY targets skipped in both cases. +any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^) +# +### +# why - tell why a a target got build +# enabled by make V=2 +# Output (listed in the order they are checked): +# (1) - due to target is PHONY +# (2) - due to target missing +# (3) - due to: file1.h file2.h +# (4) - due to command line change +# (5) - due to missing .cmd file +# (6) - due to target not in $(targets) +# (1) PHONY targets are always build +# (2) No target, so we better build it +# (3) Prerequisite is newer than target +# (4) The command line stored in the file named dir/.target.cmd +# differed from actual command line. This happens when compiler +# options changes +# (5) No dir/.target.cmd file (used to store command line) +# (6) No dir/.target.cmd file and target not listed in $(targets) +# This is a good hint that there is a bug in the kbuild file +ifeq ($(KBUILD_VERBOSE),2) +why = \ + $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ + $(if $(wildcard $@), \ + $(if $(strip $(any-prereq)),- due to: $(any-prereq), \ + $(if $(arg-check), \ + $(if $(cmd_$@),- due to command line change, \ + $(if $(filter $@, $(targets)), \ + - due to missing .cmd file, \ + - due to $(notdir $@) not in $$(targets) \ + ) \ + ) \ + ) \ + ), \ + - due to target missing \ + ) \ + ) + +echo-why = $(call escsq, $(strip $(why))) +endif +# +### +# Escape single quote for use in echo statements +escsq = $(subst $(squote),'\$(squote)',$1) +# +# echo command. +# Short version is used, if $(quiet) equals `quiet_', otherwise full one. +echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) +# +# printing commands +cmd = @$(echo-cmd) $(cmd_$(1)) + diff --git a/utils/makefile.python b/utils/makefile.python new file mode 100644 index 000000000..228eb3f80 --- /dev/null +++ b/utils/makefile.python @@ -0,0 +1,290 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +# list of python packages (folders) or modules (files) of this build +PYOBJECTS ?= + +SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python +export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH + +# folder where the python distribution takes place +PYDIST ?= ./py_dist +# folder where the python intermediate build files take place +PYBUILD ?= ./py_build +# python version to use +PY ?=3 +PYTHON ?= python$(PY) +PIP ?= pip$(PY) +PIP_INST ?= --user + +# https://www.python.org/dev/peps/pep-0508/#extras +#PY_SETUP_EXTRAS ?= \[develop,test\] +PY_SETUP_EXTRAS ?= + +PYDEBUG ?= --pdb +PYLINT_RC ?= .pylintrc + +TEST_FOLDER ?= ./tests +TEST ?= . + +VTENV_OPTS = "--no-site-packages" +PY_ENV = ./local/py$(PY) +PY_ENV_BIN = $(PY_ENV)/bin +PY_ENV_ACT = . $(PY_ENV_BIN)/activate + +ifeq ($(OS),Windows_NT) + PYTHON = python + PY_ENV_BIN = $(PY_ENV)/Scripts + PY_ENV_ACT = $(PY_ENV_BIN)/activate +endif + +ifeq ($(PYTHON),python) + VIRTUALENV = virtualenv +else + VIRTUALENV = virtualenv --python=$(PYTHON) +endif + +ifeq ($(KBUILD_VERBOSE),1) + PIP_VERBOSE = + VIRTUALENV_VERBOSE = +else + PIP_VERBOSE = "-q" + VIRTUALENV_VERBOSE = "-q" +endif + +python-help:: + @echo 'makefile.python:' + @echo ' pyenv | pyenv[un]install' + @echo ' build $(PY_ENV) & [un]install python objects' + @echo ' targts using pyenv $(PY_ENV):' + @echo ' pylint - run pylint *linting*' + @echo ' pytest - run *tox* test on python objects' + @echo ' pydebug - run tests within a PDB debug session' + @echo ' pybuild - build python packages' + @echo ' pyclean - clean intermediate python objects' + @echo ' targets using system users environment:' + @echo ' py[un]install - [un]install python objects in editable mode' + @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi' + @echo 'options:' + @echo ' make PY=2 [targets] => to eval targets with python 2 ($(PY))' + @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))' + @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)' + @echo ' make DEBUG= => target "debug": do not invoke PDB on errors' + @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]' + @echo ' when using target "pydebug", set breakpoints within py-source by adding::' + @echo ' DEBUG()' + +# ------------------------------------------------------------------------------ +# OS requirements +# ------------------------------------------------------------------------------ + +PHONY += msg-python-exe python-exe +msg-python-exe: + @echo "\n $(PYTHON) is required\n\n\ + Make sure you have $(PYTHON) installed, grab it from\n\ + https://www.python.org or install it from your package\n\ + manager. On debian based OS these requirements are\n\ + installed by::\n\n\ + sudo -H apt-get install $(PYTHON)\n" | $(FMT) + +ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) +python-exe: msg-python-exe + $(error The '$(PYTHON)' command was not found) +else +python-exe: + @: +endif + +msg-pip-exe: + @echo "\n $(PIP) is required\n\n\ + Make sure you have updated pip installed, grab it from\n\ + https://pip.pypa.io or install it from your package\n\ + manager. On debian based OS these requirements are\n\ + installed by::\n\n\ + sudo -H apt-get install python$(PY)-pip\n" | $(FMT) + +ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1) +pip-exe: msg-pip-exe + $(error The '$(PIP)' command was not found) +else +pip-exe: + @: +endif + +PHONY += msg-virtualenv-exe virtualenv-exe +msg-virtualenv-exe: + @echo "\n virtualenv is required\n\n\ + Make sure you have an updated virtualenv installed, grab it from\n\ + https://virtualenv.pypa.io/en/stable/installation/ or install it\n\ + via pip by::\n\n\ + pip install --user https://github.com/pypa/virtualenv/tarball/master\n" | $(FMT) + +ifeq ($(shell which virtualenv >/dev/null 2>&1; echo $$?), 1) +virtualenv-exe: msg-virtualenv-exe + $(error The 'virtualenv' command was not found) +else +virtualenv-exe: + @: +endif + +# ------------------------------------------------------------------------------ +# commands +# ------------------------------------------------------------------------------ + +# $2 path to folder with setup.py, this uses pip from the OS +quiet_cmd_pyinstall = INSTALL $2 + cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS) + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvinstall = PYENV install $2 + cmd_pyenvinstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) + +# Uninstall the package. Since pip does not uninstall the no longer needed +# depencies (something like autoremove) the depencies remain. + +# $2 package name to uninstall, this uses pip from the OS. +quiet_cmd_pyuninstall = UNINSTALL $2 + cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvuninstall = PYENV uninstall $2 + cmd_pyenvuninstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder where virtualenv take place +quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate + cmd_virtualenv = \ + if [ ! -d "./$(PY_ENV)" ];then \ + $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \ + else \ + echo " PYENV using virtualenv from $2"; \ + fi + +# $2 path to lint +quiet_cmd_pylint = LINT $@ + cmd_pylint = $(PY_ENV_BIN)/pylint --rcfile $(PYLINT_RC) $2 + +quiet_cmd_pytest = TEST $@ + cmd_pytest = $(PY_ENV_BIN)/tox -vv + +# setuptools, pip, easy_install its a mess full of cracks, a documentation hell +# and broken by design ... all sucks, I really, really hate all this ... aaargh! +# +# About python packaging see `Python Packaging Authority`_. Most of the names +# here are mapped to ``setup(=..., =...)`` arguments in +# ``setup.py``. See `Packaging and distributing projects`_ about ``setup(...)`` +# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_. +# +# Further read: +# +# - pythonwheels_ +# - setuptools_ +# - packaging_ +# - sdist_ +# - installing_ +# +# .. _`Python Packaging Authority`: https://www.pypa.io +# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/ +# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ +# .. _pythonwheels: https://pythonwheels.com/ +# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html +# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects +# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions +# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels +# .. _installing: https://packaging.python.org/tutorials/installing-packages/ +# +quiet_cmd_pybuild = BUILD $@ + cmd_pybuild = $(PY_ENV_BIN)/$(PYTHON) setup.py \ + sdist -d $(PYDIST) \ + bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) + +quiet_cmd_pyclean = CLEAN $@ +# remove 'build' folder since bdist_wheel does not care the --bdist-dir + cmd_pyclean = \ + rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info ;\ + find . -name '*.pyc' -exec rm -f {} + ;\ + find . -name '*.pyo' -exec rm -f {} + ;\ + find . -name __pycache__ -exec rm -rf {} + + +# ------------------------------------------------------------------------------ +# targets +# ------------------------------------------------------------------------------ + +# for installation use the pip from the OS! +PHONY += pyinstall +pyinstall: pip-exe + $(call cmd,pyinstall,.) + +PHONY += pyuninstall +pyuninstall: pip-exe + $(call cmd,pyuninstall,$(PYOBJECTS)) + +# for installation use the pip from PY_ENV (not the OS)! +PHONY += pyenvinstall +pyenvinstall: $(PY_ENV) + $(call cmd,pyenvinstall,.) + +PHONY += pyenvuninstall +pyenvuninstall: $(PY_ENV) + $(call cmd,pyenvuninstall,$(PYOBJECTS)) + +PHONY += pyclean +pyclean: + $(call cmd,pyclean) + +# to build *local* environment, python and virtualenv from the OS is needed! +pyenv: $(PY_ENV) +$(PY_ENV): virtualenv-exe python-exe + $(call cmd,virtualenv,$(PY_ENV)) + @$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) -r requirements.txt + +PHONY += pylint-exe +pylint-exe: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install pylint + +PHONY += pylint +pylint: pylint-exe + $(call cmd,pylint,$(PYOBJECTS)) + +PHONY += pybuild +pybuild: $(PY_ENV) + $(call cmd,pybuild) + +PHONY += pytest +pytest: $(PY_ENV) + $(call cmd,pytest) + +PHONY += pydebug +# set breakpoint with: +# DEBUG() +# e.g. to run tests in debug mode in emacs use: +# 'M-x pdb' ... 'make pydebug' +pydebug: $(PY_ENV) + DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST) + +# install / uninstall python objects into virtualenv (PYENV) +pyenv-install: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e . + @echo " ACTIVATE $(call normpath,$(PY_ENV_ACT)) " + +pyenv-uninstall: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes . + +# runs python interpreter from ./local/py/bin/python +pyenv-python: pyenv-install + cd ./local; ../$(PY_ENV_BIN)/python -i + +# With 'dependency_links=' setuptools supports dependencies on packages hosted +# on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big +# drawback is, due to security reasons (I don't know where the security gate on +# PyPi is), this feature is not supported by pip [2]. Thats why an upload to +# PyPi is required and since uploads via setuptools is not recommended, we have +# to imstall / use twine ... its really a mess. +# +# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi +# [2] https://github.com/pypa/pip/pull/1519 + +# https://github.com/pypa/twine +PHONY += upload-pypi +upload-pypi: pyclean pybuild + @$(PY_ENV_BIN)/twine upload $(PYDIST)/* + +.PHONY: $(PHONY)