[Buildroot] [PATCH v2 1/2] support/scripts/pkg-stats: URL checking support

Matthew Weber matthew.weber at rockwellcollins.com
Fri Sep 21 15:37:22 UTC 2018


All,

On Fri, Sep 21, 2018 at 10:36 AM Matt Weber
<matthew.weber at rockwellcollins.com> wrote:
>
>  - Adds support to check if a package has a URL and if that URL
>    is valid by doing a header request.
>  - Reports this information as part of the generated html output
>
> The URL data is currently gathered from the URL string provided
> in the Kconfig help sections for each package.
>
> This check helps ensure the URLs are valid and can be used
> for other scripting purposes as the product's home site/URL.
> CPE XML generation is an example of a case that could use this
> product URL as part of an automated update generation script.
>

~ 3x the time to execute the script with this change.

> Signed-off-by: Matt Weber <matthew.weber at rockwellcollins.com>
> ---
> Changes
> v1 -> v2
>  - Dropped disabling of SSL cert verifing
>
> [Thomas
>  - Moved URL report to new column
>  - Added color coding
>  - Better info on if a package doesn't have a Config.in to search
>    or the Config.in is missing the URL
> ---
>  support/scripts/pkg-stats | 57 +++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 57 insertions(+)
>
> diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats
> index b7b00e8..37f89ef 100755
> --- a/support/scripts/pkg-stats
> +++ b/support/scripts/pkg-stats
> @@ -24,6 +24,7 @@ from collections import defaultdict
>  import re
>  import subprocess
>  import sys
> +import requests  # URL checking
>
>  INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)")
>
> @@ -43,10 +44,32 @@ class Package:
>          self.patch_count = 0
>          self.warnings = 0
>          self.current_version = None
> +        self.url = None
> +        self.url_status = None
>
>      def pkgvar(self):
>          return self.name.upper().replace("-", "_")
>
> +    def set_url(self):
> +        """
> +        Fills in the .url field
> +        """
> +        in_help_section = False
> +        self.url_status = "No Config.in"
> +        for filename in os.listdir(os.path.dirname(self.path)):
> +            if fnmatch.fnmatch(filename, 'Config.*'):
> +                fp = open(os.path.join(os.path.dirname(self.path), filename), "r")
> +                for config_line in fp:
> +                    if config_line.strip() == "help":
> +                        in_help_section = True
> +                    if in_help_section and re.match("(.*)https?://", config_line):
> +                        self.url = ''.join(config_line.split())
> +                        self.url_status = "Found"
> +                        fp.close()
> +                        return
> +                self.url_status = "Missing"
> +                fp.close()
> +
>      def set_infra(self):
>          """
>          Fills in the .infras field
> @@ -254,6 +277,14 @@ def package_init_make_info():
>
>          Package.all_versions[pkgvar] = value
>
> +def check_url_status(pkg):
> +    if pkg.url_status != "Missing" and pkg.url_status != "No Config.in":
> +        try:
> +            url_status_code = requests.head(pkg.url, timeout=5).status_code
> +            if url_status_code >= 400:
> +                pkg.url_status = "Invalid(%s)" % str(url_status_code)
> +        except requests.exceptions.RequestException as e:
> +            return
>
>  def calculate_stats(packages):
>      stats = defaultdict(int)
> @@ -311,6 +342,15 @@ td.somepatches {
>  td.lotsofpatches {
>    background: #ff9a69;
>  }
> +td.good_url {
> +  background: #d2ffc4;
> +}
> +td.missing_url {
> +  background: #ffd870;
> +}
> +td.invalid_url {
> +  background: #ff9a69;
> +}
>  </style>
>  <title>Statistics of Buildroot packages</title>
>  </head>
> @@ -422,6 +462,20 @@ def dump_html_pkg(f, pkg):
>      f.write("  <td class=\"%s\">%d</td>\n" %
>              (" ".join(td_class), pkg.warnings))
>
> +    # URL status
> +    td_class = ["centered"]
> +    url_str = pkg.url_status
> +    if pkg.url_status == "Missing" or pkg.url_status == "No Config.in":
> +        td_class.append("missing_url")
> +    elif pkg.url_status.startswith("Invalid"):
> +        td_class.append("invalid_url")
> +        url_str = "<a href=%s>%s</a>" % (pkg.url, pkg.url_status)
> +    else:
> +        td_class.append("good_url")
> +        url_str = "<a href=%s>Link</a>" % pkg.url
> +    f.write("  <td class=\"%s\">%s</td>\n" %
> +            (" ".join(td_class), url_str))
> +
>      f.write(" </tr>\n")
>
>
> @@ -437,6 +491,7 @@ def dump_html_all_pkgs(f, packages):
>  <td class=\"centered\">Hash file</td>
>  <td class=\"centered\">Current version</td>
>  <td class=\"centered\">Warnings</td>
> +<td class=\"centered\">URL</td>
>  </tr>
>  """)
>      for pkg in sorted(packages):
> @@ -517,6 +572,8 @@ def __main__():
>          pkg.set_patch_count()
>          pkg.set_check_package_warnings()
>          pkg.set_current_version()
> +        pkg.set_url()
> +        check_url_status(pkg)
>      print("Calculate stats")
>      stats = calculate_stats(packages)
>      print("Write HTML")
> --
> 1.9.1
>


-- 
Matthew L Weber / Pr Software Engineer
Airborne Information Systems / RC Linux Secure Platforms
MS 131-100, C Ave NE, Cedar Rapids, IA, 52498, USA
www.rockwellcollins.com

Note: Any Export License Required Information and License Restricted
Third Party Intellectual Property (TPIP) content must be encrypted and
sent to matthew.weber at corp.rockwellcollins.com.



More information about the buildroot mailing list