[Buildroot] [PATCH v7 08/10] support/scripts/cpe-report: add NIST xml generation
Matt Weber
matthew.weber at rockwellcollins.com
Wed Oct 17 21:42:00 UTC 2018
From: Bryce Ferguson <bryce.ferguson at rockwellcollins.com>
Hooks to generate the NIST xml files required to propose an
update to the database. The generation uses existing Buildroot
information to create the submission information.
This design uses the generated cpe-info csv and a path to the relevant
buildroot clone for access to the packages/dir structure.
Signed-off-by: Paresh Chaudhary <paresh.chaudhary at rockwellcollins.com>
Signed-off-by: Bryce Ferguson <bryce.ferguson at rockwellcollins.com>
Signed-off-by: Matt Weber <matthew.weber at rockwellcollins.com>
---
Changes
v7
- New
---
support/scripts/cpe-report | 17 ++++--
support/scripts/cpedb.py | 128 +++++++++++++++++++++++++++++++++++++++++----
2 files changed, 132 insertions(+), 13 deletions(-)
diff --git a/support/scripts/cpe-report b/support/scripts/cpe-report
index 036eab2..da4ad96 100755
--- a/support/scripts/cpe-report
+++ b/support/scripts/cpe-report
@@ -6,7 +6,7 @@ import csv
from cpedb import CPEDB
-def get_target_cpe_report(cpe_report_file, cpedb):
+def get_target_cpe_report(cpe_report_file, pkg_dir, cpedb):
report_cpe_exact_match = ""
report_cpe_needing_update = ""
report_cpe_missing = ""
@@ -21,9 +21,9 @@ def get_target_cpe_report(cpe_report_file, cpedb):
if not result:
result = cpedb.find_partial(cpedb.get_cpe_no_version(cpe[0]))
if not result:
- report_cpe_missing += cpe[0] + "\n"
+ report_cpe_missing += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
else:
- report_cpe_needing_update += cpe[0] + "\n"
+ report_cpe_needing_update += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
else:
report_cpe_exact_match += cpe[0] + "\n"
except (OSError, IOError) as e:
@@ -34,11 +34,20 @@ def get_target_cpe_report(cpe_report_file, cpedb):
print("CPE: Found but REQUIRES UPDATE:\n" + report_cpe_needing_update)
print("CPE: Not found (proposing the following to be added):\n" + report_cpe_missing)
+ for cpe in report_cpe_needing_update.splitlines():
+ cpedb.update(cpe, pkg_dir, 1)
+ for cpe in report_cpe_missing.splitlines():
+ cpedb.update(cpe, pkg_dir, 1)
+ print("XML Generation Complete of NIST update files, see ./cpe/*")
+
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('-c', dest='cpe_report', action='store', required=True,
help='CPE Report generated by make cpe-info (csv format)')
+ parser.add_argument('-d', dest='pkg_dir', action='store', required=True,
+ help='Path to dir(s) of Buildroot and BR2_EXTERNAL(s) seperated with ":". ' +
+ 'By default the script looks for a ./package/<pkgname> and ./<pkgname> in each path)')
return parser.parse_args()
@@ -47,7 +56,7 @@ def __main__():
cpedb = CPEDB()
cpedb.get_xml_dict()
print("Performing Target CPE Report Analysis...")
- get_target_cpe_report(args.cpe_report, cpedb)
+ get_target_cpe_report(args.cpe_report, args.pkg_dir, cpedb)
__main__()
diff --git a/support/scripts/cpedb.py b/support/scripts/cpedb.py
index 77d1d17..9e2ea91 100644
--- a/support/scripts/cpedb.py
+++ b/support/scripts/cpedb.py
@@ -1,16 +1,78 @@
import sys
import urllib2
+from collections import OrderedDict
import xmltodict
import gzip
from StringIO import StringIO
+import re
+import os
CPE_XML_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
+class CPE:
+ cpe_str = None
+ cpe_str_short = None
+ cpe_desc = None
+ references = {}
+
+ def __init__(self, cpe_str, product_ref=None, version_ref=None):
+ self.cpe_str = cpe_str
+ self.cpe_str_short = ":".join(self.cpe_str.split(":")[2:6])
+ pkg_name = "".join(self.cpe_str.split(":")[4:5]).replace("_", " ")
+ pkg_ver = "".join(self.cpe_str.split(":")[5:6])
+ self.cpe_desc = pkg_name.title() + " " + pkg_ver
+ if product_ref and version_ref:
+ self.references = {
+ 'references': {
+ 'reference': [{
+ '@href': product_ref,
+ '#text': 'PRODUCT'
+ }, {
+ '@href': version_ref,
+ '#text': 'VERSION'
+ }]
+ }
+ }
+
+ def update_ref(self, new_ref):
+ self.references = new_ref
+
+ def to_dict(self):
+ cpe_dict = OrderedDict([
+ ('cpe-item', OrderedDict([
+ ('@name', 'cpe:/' + self.cpe_str_short),
+ ('title', OrderedDict([
+ ('@xml:lang', 'en-US'),
+ ('#text', self.cpe_desc)
+ ])),
+ ('references', self.references),
+ ('cpe-23:cpe23-item', OrderedDict([
+ ('@name', self.cpe_str)
+ ]))
+ ]))
+ ])
+ return cpe_dict
+
+
+def find_url_in_config_in(pkg_config_in):
+ product_url = "Not Found"
+ in_help_section = False
+ if os.path.exists(pkg_config_in):
+ fp = open(pkg_config_in, "r")
+ for config_line in fp:
+ if config_line.strip() == "help":
+ in_help_section = True
+ if in_help_section and re.match("(.*)https?://", config_line):
+ product_url = ''.join(config_line.split())
+ break
+ fp.close()
+ return product_url
+
+
class CPEDB:
all_cpedb = dict()
- all_cpes = set()
- all_cpes_no_version = set()
+ all_cpes = dict()
def get_xml_dict(self):
print("CPE: Fetching xml manifest...")
@@ -23,9 +85,9 @@ class CPEDB:
for cpe in self.all_cpedb['cpe-list']['cpe-item']:
cpe_str = cpe['cpe-23:cpe23-item']['@name']
+ item = CPE(cpe_str)
cpe_str_no_version = self.get_cpe_no_version(cpe_str)
- self.all_cpes.add(cpe_str)
- self.all_cpes_no_version.add(cpe_str_no_version)
+ self.all_cpes.update({cpe_str_no_version: {cpe_str: item}})
except urllib2.HTTPError:
print("CPE: HTTP Error: %s" % CPE_XML_URL)
@@ -36,17 +98,65 @@ class CPEDB:
def find_partial(self, cpe_str):
cpe_str_no_version = self.get_cpe_no_version(cpe_str)
- if cpe_str_no_version in self.all_cpes_no_version:
+ if cpe_str_no_version in self.all_cpes:
return cpe_str_no_version
def find(self, cpe_str):
- if cpe_str in self.all_cpes:
- return cpe_str
+ if self.find_partial(cpe_str):
+ cpe_str_no_version = self.get_cpe_no_version(cpe_str)
+ if cpe_str in self.all_cpes[cpe_str_no_version]:
+ return cpe_str
- def get_cpe_no_version(self, cpe):
- return ":".join(cpe.split(":")[:5])
+ def check_package(self, cpe_str, pkg_dir):
+ product_site = "Not Found"
+ # For each dir, check the boot/<pkgname>/, package/<pkgname>/ and <pkgname>/ folders for Config.in
+ # This should cover regular packages, linux and boot related packages.
+ for pkg_dir_path in pkg_dir.split(":"):
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'package', cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'boot', cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ return product_site
+
+ def update(self, cpe_str, pkg_dir=None, cpe_full_str=None):
+ if cpe_full_str:
+ cpe_str = cpe_str.split(",")
+ to_update = CPE(cpe_str[0], self.check_package(cpe_str[1], pkg_dir), cpe_str[2])
+ xml = self.__gen_xml__(to_update.to_dict())
+ if not os.path.exists("cpe"):
+ os.makedirs("cpe")
+ fp = open(os.path.join('cpe', cpe_str[1] + ".xml"), "w+")
+ fp.write(xmltodict.unparse(xml, pretty=True))
+ fp.close()
def get_nvd_url(self, cpe_str):
return "https://nvd.nist.gov/products/cpe/search/results?keyword=" + \
urllib2.quote(cpe_str) + \
"&status=FINAL&orderBy=CPEURI&namingFormat=2.3"
+
+ def get_cpe_no_version(self, cpe):
+ return ":".join(cpe.split(":")[:5])
+
+ def __gen_xml__(self, cpe_list):
+ list_header = {
+ "cpe-list": {
+ "@xmlns:config": "http://scap.nist.gov/schema/configuration/0.1",
+ "@xmlns": "http://cpe.mitre.org/dictionary/2.0",
+ "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+ "@xmlnsscap-core": "http://scap.nist.gov/schema/scap-core/0.3",
+ "@xmlns:cpe-23": "http://scap.nist.gov/schema/cpe-extension/2.3",
+ "@xmlns:ns6": "http://scap.nist.gov/schema/scap-core/0.1",
+ "@xmlns:meta": "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
+ "@xsi:schemaLocation": "http://scap.nist.gov/schema/cpe-extension/2.3 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd http://cpe.mitre.org/dictionary/2.0 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2 https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd http://scap.nist.gov/schema/scap-core/0.3 https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd http://scap.nist.gov/schema/configuration/0.1 https://scap.nist.gov/schema/nvd/configuration_0.1.xsd http://scap.nist.gov/schema/scap-core/0.1 https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"
+ }
+ }
+ list_header['cpe-list'].update(cpe_list)
+ return list_header
--
1.9.1
More information about the buildroot
mailing list