[Buildroot] [RFC] external toolchain scanner

Hollis Blanchard hollis_blanchard at mentor.com
Wed Dec 7 18:14:41 UTC 2016


Newer copy attached. I've been using this version for a couple weeks.

This can be run with defconfig as an extra option, like so:

    ./scan-ext-toolchain <toolchain> [<files>]

That will prepend the content of <files> to the toolchain metadata, so 
that it can be used like so:

    ./scan-ext-toolchain -o output/combined_defconfig <toolchain> my-board-defconfig
    make BR2_DEFCONFIG=output/combined_defconfig defconfig


I still haven't tried to probe SSP, RPC, etc, for non-glibc toolchains.

Hollis Blanchard <hollis_blanchard at mentor.com>
Mentor Graphics Emulation Division

On 11/22/2016 05:32 PM, Hollis Blanchard wrote:
> The attached python script inspects an external toolchain and spits 
> out BR config settings, like so (with a Linaro toolchain):
>
>     aurora:buildroot$ ./support/scripts/scan-ext-toolchain /foo/gcc-linaro-aarch64-linux-gnu-4.9-2014.09_linux
>     BR2_TOOLCHAIN_EXTERNAL=y
>     BR2_TOOLCHAIN_EXTERNAL_GCC_4_9=y
>     BR2_TOOLCHAIN_EXTERNAL_HEADERS_3_7=y
>     BR2_TOOLCHAIN_EXTERNAL_CXX=y
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y
>     BR2_TOOLCHAIN_EXTERNAL_GF=y
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="aarch64-linux-gnu"
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y
>     BR2_TOOLCHAIN_EXTERNAL_PATH="/foo/gcc-linaro-aarch64-linux-gnu-4.9-2014.09_linux"
>
> It also works with multi-arch toolchains (this one from Mentor Graphics):
>
>     aurora:buildroot$ ./support/scripts/scan-ext-toolchain /foo/codesourcery/codebench/
>     Toolchain supports multiple targets. Please choose one of the following: ['aarch64-linux-gnu', 'arm-none-eabi', 'arm-none-linux-gnueabi']
>     aurora:buildroot$ ./support/scripts/scan-ext-toolchain -t arm-none-linux-gnueabi /foo/codesourcery/codebench/
>     BR2_TOOLCHAIN_EXTERNAL=y
>     BR2_TOOLCHAIN_EXTERNAL_HEADERS_3_16=y
>     BR2_TOOLCHAIN_EXTERNAL_GCC_4_9=y
>     BR2_TOOLCHAIN_EXTERNAL_CXX=y
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX=arm-none-linux-gnueabi
>     BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC=y
>     BR2_TOOLCHAIN_EXTERNAL_PATH="/foo/codesourcery/codebench/"
>
> It complains about bare-metal toolchains:
>
>     aurora:buildroot$ ./support/scripts/scan-ext-toolchain -t arm-none-eabi /foo/codesourcery/codebench/
>     Is this a Linux toolchain? Couldn't find the sysroot in:
>              /foo/codesourcery/codebench/arm-none-eabi/libc
>              /foo/codesourcery/codebench/arm-none-eabi/sysroot
>
>
> Current limitations that I know of:
>
> 1. I haven't run through a full build with it yet, but it looks like 
> it's doing the right thing.
>
> 2. It detects MUSL and UCLIBC toolchains, but it looks like further 
> work is needed to detect SSP, RPC, etc, for those toolchain types.
>
> 3. There is no guarantee that BR2_arch matches 
> BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX.
>
> 4. Users would run it something like this:
>
>     ./support/scripts/scan-ext-toolchain > toolchain.defconfig
>     cat board.defconfig toolchain.defconfig > defconfig
>     make ... BR2_DEFCONFIG=defconfig
>
> Creating and managing board.defconfig without toolchain configuration 
> data is left as an (awkward?) exercise for the user.
>
>
> Comments?
>
> -- 
> Hollis Blanchard<hollis_blanchard at mentor.com>
> Mentor Graphics Emulation Division

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20161207/2a49114a/attachment.html>
-------------- next part --------------
#!/usr/bin/env python

# Copyright 2016 Mentor Graphics Corporation
# All Rights Reserved
#
# THIS WORK CONTAINS TRADE SECRET AND PROPRIETARY
# INFORMATION WHICH ARE THE PROPERTY OF MENTOR
# GRAPHICS CORPORATION OR ITS LICENSORS AND IS
# SUBJECT TO LICENSE TERMS.

import sys
import os
import subprocess
import re
from optparse import OptionParser

class ExtToolchainMetadata:
	def __init__(self, base_path):
		self.cfg = {}
		self.tools = {}
		self.base_path = base_path
		self.sysroot_path = None

	def get_buildroot_cfg(self):
		result = 'BR2_TOOLCHAIN_EXTERNAL=y\n'
		for k, v in self.cfg.items():
			result += 'BR2_TOOLCHAIN_EXTERNAL_%s=%s\n' % (k, v)
		return result

def probe_prefix(path, target, metadata):
	libexec_path = os.path.join(path, 'libexec', 'gcc') 

	if not os.path.isdir(libexec_path):
		raise RuntimeError("Couldn't examine directory %s" % libexec_path)

	targets = os.listdir(libexec_path)
	if len(targets) == 0:
		raise RuntimeError("Couldn't find any targets in %s" % libexec_path)

	if not target:
		if len(targets) > 1:
			raise RuntimeError('Toolchain supports multiple targets. '
					'Please choose one of the following: %s' % (targets))
		target = targets[0]
	else:
		if target not in targets:
			raise RuntimeError('Toolchain does not support target %s.' % target)
		target = target

	# XXX use ARCH instead?
	# cpu, vendor_os = target.split('-', 1)
	# metadata.cfg['CUSTOM_PREFIX'] = '"$(ARCH)-%s"' % vendor_os
	metadata.cfg['CUSTOM_PREFIX'] = '"%s"' % target

def probe_tools(path, metadata):
	class Tool:
		def __init__(self, executable, cfgname=None):
			self.executable = executable
			self.cfgname = cfgname

	tools = (
		Tool('gcc'),
		Tool('readelf'),
		Tool('g++', 'CXX'),
		Tool('gfortran', 'GF'),
	)

	prefix = metadata.cfg['CUSTOM_PREFIX'].strip('"')

	for tool in tools:
		full_name = '%s-%s' % (prefix, tool.executable)
		full_path = os.path.join(path, 'bin', full_name)

		if os.path.exists(full_path):
			metadata.tools[tool.executable] = full_path
			if tool.cfgname:
				metadata.cfg[tool.cfgname] = 'y'

def probe_gcc_version(metadata):
	argv = [
		metadata.tools['gcc'],
		'--version',
	]

	proc = subprocess.Popen(argv, stdout=subprocess.PIPE)
	output = proc.communicate()[0]
	line1 = output.splitlines()[0]

	m = re.match('^[^)]+\) ([^ ]+)', line1)
	if not m:
		raise RuntimeError("%s\n\tdidn't report a recognizable version:\n%s" %
				(metadata.tools['gcc'], line1))

	version = m.group(1) # E.g. 4.9.2
	major, minor = [ int(i) for i in version.split('.', 2)[:2] ]
	metadata.cfg['GCC_%d_%d' % (major, minor)] = 'y'

def probe_gcc_sysroot(metadata):
	# Sysroot directories could have a couple names:
	subdirs = ('libc', 'sysroot')

	# Construct a list of full paths so that in case of failure we can tell the
	# user exactly where we searched.
	base = metadata.base_path
	prefix = metadata.cfg['CUSTOM_PREFIX'].strip('"')
	sysroot_paths = [ os.path.join(base, prefix, d) for d in subdirs ]

	sysroot_path = None
	for path in sysroot_paths:
		if os.path.exists(path):
			sysroot_path = path
			break

	if not sysroot_path:
		msg = "Is this a Linux toolchain? Couldn't find the sysroot in:\n\t%s"
		raise RuntimeError(msg % '\n\t'.join(sysroot_paths))

	metadata.sysroot_path = sysroot_path

def probe_gcc_headers(metadata):
	version_path = os.path.join(metadata.sysroot_path,
		'usr',
		'include',
		'linux',
		'version.h'
	)
	version_re = '#define LINUX_VERSION_CODE ([0-9]+)'

	with open(version_path) as version_file:
		linux_version_code = version_file.readline()

	m = re.match(version_re, linux_version_code)
	if not m:
		msg = "Didn't recognize LINUX_VERSION_CODE in %s"
		raise RuntimeError(msg % version_path)

	version = int(m.group(1))
	major = (version >> 16) & 0xff
	minor = (version >> 8) & 0xff
	metadata.cfg['HEADERS_%d_%d' % (major, minor)] = 'y'

def probe_libc(metadata):
	libc_re = '  -m(glibc|musl|eglibc)\s+\[enabled\]'
	argv = [
		metadata.tools['gcc'],
		'-Q',
		'--help=target',
	]

	proc = subprocess.Popen(argv, stdout=subprocess.PIPE)
	output = proc.communicate()[0].splitlines()
	for line in output:
		m = re.match(libc_re, line)
		if m:
			libc = m.group(1)
			metadata.cfg['CUSTOM_%s' % libc.upper()] = 'y'
			break

def probe_rpc(metadata):
	rpc_path = os.path.join(metadata.sysroot_path,
		'usr',
		'include',
		'rpc',
		'rpc.h')
	if os.path.exists(rpc_path):
		metadata.cfg['HAS_RPC'] = 'y'

def probe(path, options, metadata):
	metadata.cfg['CUSTOM'] = 'y'
	metadata.cfg['PATH'] = '"%s"' % path

	probe_prefix(path, options.target, metadata)
	probe_tools(path, metadata)
	probe_gcc_version(metadata)
	probe_gcc_sysroot(metadata)
	probe_gcc_headers(metadata)
	probe_libc(metadata)
	probe_rpc(metadata)

def cat_files(out_file, files):
	for path in files:
		with open(path) as f:
			for line in f.readlines():
				out_file.write(line)

def main():
	parser = OptionParser()
	parser.add_option('-t', '--target', dest='target')
	parser.add_option('-o', '--output', default='/dev/stdout')

	(options, arguments) = parser.parse_args()

	if len(arguments) < 1:
		print "Missing path to toolchain base directory"
		sys.exit(1)

	toolchain_dir = arguments[0]
	if not os.path.isdir(toolchain_dir):
		print "Not a directory: %s" % toolchain_dir
		sys.exit(2)

	metadata = ExtToolchainMetadata(toolchain_dir)
	probe(toolchain_dir, options, metadata)

	with open(options.output, 'w') as out_file:
		if len(arguments) >= 1:
			cat_files(out_file, arguments[1:])

		out_file.write(metadata.get_buildroot_cfg())

if __name__ == '__main__':
	try:
		main()
	except RuntimeError, e:
		print e


More information about the buildroot mailing list