[Buildroot] [PATCH 2/9] support/scripts/check-package: new script
Ricardo Martincoski
ricardo.martincoski at gmail.com
Sat Dec 31 03:21:03 UTC 2016
Create the infra to check the style of new packages before submitting.
The overall function of the script is described inside a txt file.
It is designed to process the actual files and NOT the patch files
generated by git format-patch.
Also add the first check function, to warn if a file (Config.in*, *.mk,
*.hash, *.patch) has no newline at the last line of the file, see [1].
Basic usage for simple packages:
support/scripts/check-package -vvv package/newpackage/*
Basic usage for packages with subdirs:
support/scripts/check-package -vvv $(find package/newpackage/ -type f)
See "checkpackage" in [2].
[1] http://patchwork.ozlabs.org/patch/631129/
[2] http://elinux.org/Buildroot#Todo_list
Signed-off-by: Ricardo Martincoski <ricardo.martincoski at gmail.com>
---
Notes:
$ time support/scripts/check-package $(find package -type f) >/dev/null 2>/dev/null
real 0m0.415s
user 0m0.392s
sys 0m0.024s
CHECK_NEWLINE_AT_EOF:
support/scripts/check-package --include-only check_newline_at_eof \
$(find package -type f) 2>/dev/null | wc -l
0
(cd support/scripts/check-package-example && \
../check-package --include-only check_newline_at_eof -vv package/*/*)
package/package1/wrong-name.patch:13: missing newline at end of file
#include <curses.h>
159 lines processed
1 warnings generated
support/scripts/check-package | 136 ++++++++++++++++++++++++++++++
support/scripts/check-package.txt | 76 +++++++++++++++++
support/scripts/checkpackagelib.py | 18 ++++
support/scripts/checkpackagelib_config.py | 7 ++
support/scripts/checkpackagelib_hash.py | 7 ++
support/scripts/checkpackagelib_mk.py | 8 ++
support/scripts/checkpackagelib_patch.py | 7 ++
7 files changed, 259 insertions(+)
create mode 100755 support/scripts/check-package
create mode 100644 support/scripts/check-package.txt
create mode 100644 support/scripts/checkpackagelib.py
create mode 100644 support/scripts/checkpackagelib_config.py
create mode 100644 support/scripts/checkpackagelib_hash.py
create mode 100644 support/scripts/checkpackagelib_mk.py
create mode 100644 support/scripts/checkpackagelib_patch.py
diff --git a/support/scripts/check-package b/support/scripts/check-package
new file mode 100755
index 000000000..42af82e2c
--- /dev/null
+++ b/support/scripts/check-package
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# See support/scripts/check-package.txt before editing this file.
+
+from __future__ import print_function
+import argparse
+import inspect
+import re
+import sys
+
+import checkpackagelib_config
+import checkpackagelib_hash
+import checkpackagelib_mk
+import checkpackagelib_patch
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+
+ # Do not use argparse.FileType("r") here because only files with known
+ # format will be open based on the filename.
+ parser.add_argument("files", metavar="F", type=str, nargs="*",
+ help="list of files")
+
+ parser.add_argument("--manual-url", action="store",
+ default="http://nightly.buildroot.org/",
+ help="default: %(default)s")
+ parser.add_argument("--verbose", "-v", action="count", default=0)
+
+ # Now the debug options in the order they are processed.
+ parser.add_argument("--include-only", dest="include_list", action="append",
+ help="run only the specified functions (debug)")
+ parser.add_argument("--exclude", dest="exclude_list", action="append",
+ help="do not run the specified functions (debug)")
+ parser.add_argument("--dry-run", action="store_true", help="print the "
+ "functions that would be called for each file (debug)")
+
+ return parser.parse_args()
+
+
+CONFIG_IN_FILENAME = re.compile("/Config\.in(\.host)?$")
+FILE_IS_FROM_A_PACKAGE = re.compile("package/[^/]*/")
+
+
+def get_lib_from_filename(fname):
+ if FILE_IS_FROM_A_PACKAGE.search(fname) is None:
+ return None
+ if CONFIG_IN_FILENAME.search(fname):
+ return checkpackagelib_config
+ if fname.endswith(".hash"):
+ return checkpackagelib_hash
+ if fname.endswith(".mk"):
+ return checkpackagelib_mk
+ if fname.endswith(".patch"):
+ return checkpackagelib_patch
+ return None
+
+
+def call_check_function(function, fname, args, **kwargs):
+ warnings = function(fname, args, **kwargs)
+
+ # Avoid the need to use 'return []' at the end of every check function.
+ if warnings is None:
+ return 0 # No warning generated.
+
+ for level, message in enumerate(warnings):
+ if args.verbose >= level:
+ print(message.replace("\t", "< tab >").rstrip())
+ return 1 # One more warning to count.
+
+
+def check_file_using_lib(fname, args):
+ # Count number of warnings generated and lines processed.
+ nwarnings = 0
+ nlines = 0
+
+ lib = get_lib_from_filename(fname)
+ if not lib:
+ if args.verbose >= 3:
+ print("{}: ignored".format(fname))
+ return nwarnings, nlines
+ callbacks = inspect.getmembers(lib, inspect.isfunction)
+
+ # Do not call helper functions.
+ callbacks = [cb for cb in callbacks if not cb[0].startswith("_")]
+
+ if args.include_list:
+ callbacks = [cb for cb in callbacks if cb[0] in args.include_list]
+ if args.exclude_list:
+ callbacks = [cb for cb in callbacks if cb[0] not in args.exclude_list]
+
+ if args.dry_run:
+ functions_to_run = [cb[0] for cb in callbacks]
+ print("{}: would run: {}".format(fname, functions_to_run))
+ return nwarnings, nlines
+
+ for cb in callbacks:
+ nwarnings += call_check_function(cb[1], fname, args, start=True)
+ for lineno, text in enumerate(open(fname, "r").readlines()):
+ nlines += 1
+ for cb in callbacks:
+ nwarnings += call_check_function(cb[1], fname, args,
+ lineno=lineno + 1, text=text)
+ for cb in callbacks:
+ nwarnings += call_check_function(cb[1], fname, args, end=True)
+
+ return nwarnings, nlines
+
+
+def __main__():
+ args = parse_args()
+
+ if len(args.files) == 0:
+ print("No files to check style")
+ sys.exit(1)
+
+ # Accumulate number of warnings generated and lines processed.
+ total_warnings = 0
+ total_lines = 0
+
+ for fname in args.files:
+ nwarnings, nlines = check_file_using_lib(fname, args)
+ total_warnings += nwarnings
+ total_lines += nlines
+
+ # The warning messages are printed to stdout and can be post-processed
+ # (e.g. counted by 'wc'), so for stats use stderr. Wait all warnings are
+ # printed, for the case there are many of them, before printing stats.
+ sys.stdout.flush()
+ print("{} lines processed".format(total_lines), file=sys.stderr)
+ print("{} warnings generated".format(total_warnings), file=sys.stderr)
+
+ if total_warnings > 0:
+ sys.exit(1)
+
+
+__main__()
diff --git a/support/scripts/check-package.txt b/support/scripts/check-package.txt
new file mode 100644
index 000000000..2e71323a2
--- /dev/null
+++ b/support/scripts/check-package.txt
@@ -0,0 +1,76 @@
+How the scripts are structured:
+- check-package is the main engine, called by the user.
+ For each input file, this script decides which parser should be used and it
+ collects all functions declared in the library file. It opens the input files
+ and it serves each raw line (including newline!) to every check function in
+ the library. Two special parameters are used to call the initialization of
+ each check function (for the case it needs to keep data across calls) and the
+ equivalent finalization (e.g. for the case a message must be issued if some
+ pattern is not in the input file).
+- checkpackagelib.py contains common check functions.
+ Each check function is explicitly included in a given type-parsing library.
+ Do not include every single check function in this file, a function that will
+ only parse hash files should be implemented in the hash-parsing library.
+ When a warning must be issued, the check function returns an array of strings.
+ Each string is a warning message and is displayed if the corresponding verbose
+ level is active. When the script is called without --verbose only the first
+ warning in the returned array is printed; when called with --verbose both
+ first and second warnings are printed; when called with -vv until the third
+ warning is printed; an so on.
+ Helper functions can be defined by starting the name with "_"; they will not
+ be called by the main script.
+- checkpackagelib_type.py contains check functions specific to files of this
+ type.
+
+Some hints when changing this code:
+- prefer O(n) algorithms, where n is the total number of lines in the files
+ processed.
+- when there is no other reason for ordering, use alphabetical order (e.g. keep
+ the check functions in alphabetical order, keep the imports in alphabetical
+ order, and so on).
+- use pyflakes to detect and fix potential problems.
+- use pep8 formatting.
+- keep in mind each function will be called with start=True before any line is
+ served to be checked. A function that checks the filename should only
+ implement this behavior. A function that needs to keep data across calls (e.g.
+ keep the last line before the one being processed) should initialize all
+ "static" variables at this stage.
+- keep in mind each function will be called with end=True after all lines were
+ served to be checked. A function that checks the absence of a pattern in the
+ file will need to use this stage.
+- try to avoid false warnings. It's better to not issue a warning message to a
+ corner case than have too many false warnings. The second can make users stop
+ using the script.
+- do not check spacing in the input line in every single function. Trailing
+ whitespace and wrong indentation should be checked by separate functions.
+- avoid duplicate tests. Try to test only one thing in each function.
+- in the warning message, include the url to a section from the manual, when
+ applicable. It potentially will make more people know the manual.
+- use short sentences in the warning messages. A complete explanation can be
+ added to show when --verbose is used.
+- when testing, verify the error message is displayed when the error pattern is
+ found, but also verify the error message is not displayed for few
+ well-formatted packages... there are many of these, just pick your favorite
+ as golden package that should not trigger any warning message.
+- check the url displayed by the warning message works.
+
+Usage examples:
+- to get a list of check functions that would be called without actually
+ calling them you can use the --dry-run option:
+$ support/scripts/check-package --dry-run package/yourfavorite/*
+
+- when you just added a new check function, e.g. check_something, check how it
+ behaves for all current packages:
+$ support/scripts/check-package --include-only check_something \
+ $(find package -type f)
+
+- the effective processing time (when the .pyc were already generated and all
+ files to be processed are cached in the RAM) should stay in the order of few
+ seconds:
+$ support/scripts/check-package $(find package -type f) >/dev/null ; \
+ time support/scripts/check-package $(find package -type f) >/dev/null
+
+- vim users can navigate the warnings (most editors probably have similar
+ function) since warnings are generated in the form 'path/file:line: warning':
+$ find package/ -name 'Config.*' > filelist && vim -c \
+ 'set makeprg=support/scripts/check-package\ $(cat\ filelist)' -c make -c copen
diff --git a/support/scripts/checkpackagelib.py b/support/scripts/checkpackagelib.py
new file mode 100644
index 000000000..987399954
--- /dev/null
+++ b/support/scripts/checkpackagelib.py
@@ -0,0 +1,18 @@
+# See support/scripts/check-package.txt before editing this file.
+
+
+def check_newline_at_eof(
+ fname, args, lineno=0, text=None, start=False, end=False):
+ if start:
+ check_newline_at_eof.lastlineno = 0
+ check_newline_at_eof.lastline = "\n"
+ return
+ if end:
+ line = check_newline_at_eof.lastline
+ if line == line.rstrip("\r\n"):
+ return ["{}:{}: missing newline at end of file"
+ .format(fname, check_newline_at_eof.lastlineno),
+ line]
+ return
+ check_newline_at_eof.lastlineno = lineno
+ check_newline_at_eof.lastline = text
diff --git a/support/scripts/checkpackagelib_config.py b/support/scripts/checkpackagelib_config.py
new file mode 100644
index 000000000..2660cccd7
--- /dev/null
+++ b/support/scripts/checkpackagelib_config.py
@@ -0,0 +1,7 @@
+# See support/scripts/check-package.txt before editing this file.
+# Kconfig generates errors if someone introduces a typo like "boool" instead of
+# "bool", so below check functions don't need to check for things already
+# checked by running "make menuconfig".
+
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from checkpackagelib import check_newline_at_eof
diff --git a/support/scripts/checkpackagelib_hash.py b/support/scripts/checkpackagelib_hash.py
new file mode 100644
index 000000000..590d5a7c4
--- /dev/null
+++ b/support/scripts/checkpackagelib_hash.py
@@ -0,0 +1,7 @@
+# See support/scripts/check-package.txt before editing this file.
+# The validity of the hashes itself is checked when building, so below check
+# functions don't need to check for things already checked by running
+# "make package-dirclean package-source".
+
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from checkpackagelib import check_newline_at_eof
diff --git a/support/scripts/checkpackagelib_mk.py b/support/scripts/checkpackagelib_mk.py
new file mode 100644
index 000000000..ad7362335
--- /dev/null
+++ b/support/scripts/checkpackagelib_mk.py
@@ -0,0 +1,8 @@
+# See support/scripts/check-package.txt before editing this file.
+# There are already dependency checks during the build, so below check
+# functions don't need to check for things already checked by exploring the
+# menu options using "make menuconfig" and by running "make" with appropriate
+# packages enabled.
+
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from checkpackagelib import check_newline_at_eof
diff --git a/support/scripts/checkpackagelib_patch.py b/support/scripts/checkpackagelib_patch.py
new file mode 100644
index 000000000..0fb3685a7
--- /dev/null
+++ b/support/scripts/checkpackagelib_patch.py
@@ -0,0 +1,7 @@
+# See support/scripts/check-package.txt before editing this file.
+# The format of the patch files is tested during the build, so below check
+# functions don't need to check for things already checked by running
+# "make package-dirclean package-patch".
+
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from checkpackagelib import check_newline_at_eof
--
2.11.0
More information about the buildroot
mailing list