[Buildroot] [PATCH 1/3] scanpypi.py: new utility
Denis THULIN
denis.thulin at openwide.fr
Mon Jun 15 10:06:03 UTC 2015
Signed-off-by: Denis THULIN <denis.thulin at openwide.fr>
---
v0: initial commit
python-pacakage-generator.py is an utility for automatically generating a
python package. It fetches packages info from http://pypi.python.org and
generates corresponding packages files.
v1:
- renamed python-package-generator to scanpypi
- split the huge script into a lot of functions
- fixed mistakes and small bugs
I did not know where to put the script so I put it in support/scripts.
I have updated the python-package section of the manual as well.
Signed-off-by: Denis THULIN <denis.thulin at openwide.fr>
---
docs/manual/adding-packages-python.txt | 36 ++
support/scripts/scanpypi.py | 607 +++++++++++++++++++++++++++++++++
2 files changed, 643 insertions(+)
create mode 100755 support/scripts/scanpypi.py
diff --git a/docs/manual/adding-packages-python.txt b/docs/manual/adding-packages-python.txt
index f81d625..647cb67 100644
--- a/docs/manual/adding-packages-python.txt
+++ b/docs/manual/adding-packages-python.txt
@@ -7,6 +7,42 @@ This infrastructure applies to Python packages that use the standard
Python setuptools mechanism as their build system, generally
recognizable by the usage of a +setup.py+ script.
+[[scanpypi]]
+
+==== generating a +python-package+ from a pypi repository
+
+You may want to use the +scanpypi.py+ located in
++support/script+ to generate a package from an existing pypi(pip) package.
+
+you can find the list of existing pypi package here: (https://pypi.python.org).
+
+Please keep in mind that you most likely need
+to manually check the package for any mistakes
+as there are things that cannot be guessed by the generator (e.g.
+dependencies on any of the python core modules
+such as BR2_PACKAGE_PYTHON_ZLIB).
+
+When at the root of your buildroot directory just do :
+
+-----------------------
+./support/script/scanpypi.py foo bar -o package
+-----------------------
+
+This will generate packages +python-foo+ and +python-bar+ in the package
+folder if they exist on https://pypi.python.org.
+
+You will need to manually write the path to the package inside
+the +package/Config.in+ file:
+
+Find the +external python modules+ menu and insert your package inside.
+Keep in mind that the items inside a menu should be in alphabetical order.
+
+Option +-h+ wil list the options available
+
+-----------------------
+./support/script/scanpypi.py -h
+-----------------------
+
[[python-package-tutorial]]
==== +python-package+ tutorial
diff --git a/support/scripts/scanpypi.py b/support/scripts/scanpypi.py
new file mode 100755
index 0000000..953f8d2
--- /dev/null
+++ b/support/scripts/scanpypi.py
@@ -0,0 +1,607 @@
+#!/usr/bin/python2
+"""
+Utility for building buildroot packages for existing pypi packages
+
+Any package built by scanpypi should be manually checked for
+errors.
+"""
+from __future__ import print_function
+import argparse
+import json
+import urllib2
+import sys
+import os
+import shutil
+import StringIO
+import tarfile
+import errno
+import hashlib
+import re
+import magic
+import tempfile
+from functools import wraps
+
+
+# private global
+_calls = {}
+
+
+def setup_info(pkg_name):
+ """Get a package info from _calls
+
+ Keyword arguments:
+ pkg_name -- the name of the package
+ """
+ return _calls[pkg_name]
+
+
+def setup_decorator(func, method):
+ """
+ Decorator for distutils.core.setup and setuptools.setup.
+ Puts the args of setup as a dict inside global private dict _calls.
+ Add key 'method' which should be either 'setuptools' or 'distutils'.
+
+ Keyword arguments:
+ func -- either setuptools.setup or distutils.core.setup
+ method -- either 'setuptools' or 'distutils'
+ """
+
+ @wraps(func)
+ def closure(*args, **kwargs):
+ _calls[kwargs['name']] = kwargs
+ _calls[kwargs['name']]['method'] = method
+ return closure
+
+
+# monkey patch
+import setuptools
+setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
+import distutils
+distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
+
+
+def find_file_upper_case(filenames, path='./'):
+ """
+ List generator:
+ Recursively find files that matches one of the specified filenames.
+ Returns absolute path
+
+ Keyword arguments:
+ filenames -- List of filenames to be found
+ path -- Path to the directory to search
+ """
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ if file.upper() in filenames:
+ yield (os.path.join(root, file))
+
+
+def pkg_buildroot_name(pkg_name):
+ """
+ Returns name to avoid troublesome characters.
+ Remove all non alphanumeric characters except -
+ Also lowers the name
+
+ Keyword arguments:
+ pkg_name -- String to rename
+ """
+ name = re.sub('[^\w-]', '', pkg_name.lower())
+ name = re.sub('^python-', '', name)
+ return name
+
+
+def find_setup(package_name, version, archive):
+ """
+ Search for setup.py file in an archive and returns True if found
+ Used for finding the correct path to the setup.py
+
+ Keyword arguments:
+ package_name -- base name of the package to search (e.g. Flask)
+ version -- version of the package to search (e.g. 0.8.1)
+ archive -- tar archive to search in
+ """
+ try:
+ archive.getmember('{name}-{version}/setup.py'.format(
+ name=package_name,
+ version=version))
+ except KeyError:
+ return False
+ else:
+ return True
+
+
+def fetch_package_info(pkg_name):
+ """
+ Fetch a package's metadata for the python package index
+
+ Keyword arguments:
+ pkg_name -- the name of the package
+ """
+ url = 'https://pypi.python.org/pypi/{pkg}/json'.format(
+ pkg=pkg_name)
+ print('URL:', url)
+ try:
+ pkg_json = urllib2.urlopen(url).read().decode()
+ except (urllib2.HTTPError) as error:
+ print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
+ print('ERROR: Could not find package {pkg}.\n'
+ 'Check syntax inside the python package index:\n'
+ 'https://pypi.python.org/pypi/ '.format(pkg=pkg_name))
+ return None, None
+ except urllib2.URLError:
+ print('ERROR: Could not find package {pkg}.\n'
+ 'Check syntax inside the python package index:\n'
+ 'https://pypi.python.org/pypi/ '.format(pkg=pkg_name))
+ return None, None
+
+ else:
+ return pkg_json, url
+
+
+def download_package(package):
+ """
+ Download a package using metadata from pypi
+
+ Keyword arguments:
+ package -- a dictionary containing info from the pypi json api
+ """
+ try:
+ targz = package['urls'][0]['filename']
+ except IndexError:
+ print(
+ 'Non conventional package, ',
+ 'please check manually after creation')
+ download_url = package['info']['download_url']
+ try:
+ download = urllib2.urlopen(download_url)
+ except urllib2.HTTPError:
+ targz = None
+ download = None
+ as_file = None
+ used_url = None
+ else:
+ used_url = {'url': download_url}
+ as_file = StringIO.StringIO(download.read())
+ as_file.seek(0)
+ extension = 'tar.gz'
+ if 'gzip' not in magic.from_buffer(as_file.read()):
+ extension = 'tar.bz2'
+ targz = '{name}-{version}.{extension}'.format(
+ name=package['info']['name'],
+ version=package['info']['version'], extension=extension)
+ as_file.seek(0)
+ used_url['filename'] = targz
+ else:
+ for download_url in package['urls']:
+ try:
+ download = urllib2.urlopen(download_url['url'])
+ except urllib2.HTTPError:
+ targz = None
+ download = None
+ as_file = None
+ used_url = None
+ else:
+ used_url = download_url
+ as_file = StringIO.StringIO(download.read())
+ md5_sum = hashlib.md5(as_file.read()).hexdigest()
+ if md5_sum == download_url['md5_digest']:
+ break
+ targz = used_url['filename']
+ return download, targz, used_url, as_file
+
+
+def extract_package(pkg_name, as_file, tmp_path):
+ """
+ Create folders used for extracting a package as file object and extract it
+
+ pkg_name -- name of the package to be extracted
+ as_file -- file object to extract
+ tmp_path -- folder where you want the package to be extracted
+ """
+ as_file.seek(0)
+ as_tarfile = tarfile.open(fileobj=as_file)
+ tmp_pkg = '/'.join([tmp_path, pkg_name])
+ try:
+ os.makedirs(tmp_pkg)
+ except OSError as exception:
+ if exception.errno != errno.EEXIST:
+ print("ERROR: ", exception.message, file=sys.stderr)
+ return None, None
+ print('WARNING:', exception.message, file=sys.stderr)
+ print('Removing {pkg}...'.format(pkg=tmp_pkg))
+ shutil.rmtree(tmp_pkg)
+ os.makedirs(tmp_pkg)
+ version = package['info']['version']
+ tar_folder = package['info']['name']
+ if not find_setup(tar_folder, version, as_tarfile):
+ return None, None
+ as_tarfile.extractall(tmp_pkg)
+ as_tarfile.close()
+ as_file.close()
+ tmp_extract = '{folder}/{name}-{version}'.format(
+ folder=tmp_pkg,
+ name=tar_folder,
+ version=package['info']['version'])
+ return tar_folder, tmp_extract
+
+
+def get_requirements(package_name):
+ """
+ Retrieve dependencies of from a metadata found in the setup.py script of
+ a pypi package.
+
+ Keyword Arguments:
+ package_name -- name of the package found in the pypi metadata of the
+ package.
+ """
+ pkg_req = setup_info(package_name)['install_requires']
+ pkg_req = [re.sub('([\w-]+)[><=]*.*', r'\1', req).lower()
+ for req in pkg_req]
+ pkg_req = map(pkg_buildroot_name, pkg_req)
+ req_not_found = [
+ pkg for pkg in pkg_req
+ if 'python-{name}'.format(name=pkg)
+ not in os.listdir(pkg_folder)
+ ]
+ req_not_found = [pkg for pkg in req_not_found
+ if pkg not in packages]
+ if (req_not_found) != 0:
+ print(
+ 'Error: could not find packages \'{packages}\''
+ 'required by {current_package}'.format(
+ packages=", ".join(req_not_found),
+ current_package=pkg_name))
+ return pkg_req
+
+
+def create_mk_header(pkg_name):
+ """
+ Create the header of the <package_name>.mk file
+
+ Keyword arguments:
+ pkg_name -- name of the package
+ """
+ header = ['#' * 80 + '\n']
+ header.append('#\n')
+ header.append('# python-{name}\n'.format(name=pkg_name))
+ header.append('#\n')
+ header.append('#' * 80 + '\n')
+ header.append('\n')
+ return header
+
+
+def create_mk_download_info(pkg_name, version, targz, url):
+ """
+ Create the lines refering to the download information of the
+ <package_name>.mk file
+
+ Keyword arguments:
+ pkg_name -- name of the package
+ version -- version of the package
+ targz -- name of the archive corresponding to the package
+ url -- url to be used for downloading the package
+ """
+ lines = []
+ version_line = 'PYTHON_{name}_VERSION = {version}\n'.format(
+ name=pkg_name.upper(),
+ version=version)
+ lines.append(version_line)
+
+ targz = targz.replace(
+ version,
+ '$(PYTHON_{name}_VERSION)'.format(name=pkg_name.upper()))
+ targz_line = 'PYTHON_{name}_SOURCE = {filename}\n'.format(
+ name=pkg_name.upper(),
+ filename=targz)
+ lines.append(targz_line)
+
+ site_line = ('PYTHON_{name}_SITE = {url}\n'.format(
+ name=pkg_name.upper(),
+ url=url['url'].replace(url['filename'], '')))
+ if 'sourceforge' in site_line:
+ site_line = ('PYTHON_{name}_SITE = {url}\n'.format(
+ name=pkg_name.upper(),
+ url=url['url']))
+ lines.append(site_line)
+ return lines
+
+
+def create_mk_setup(pkg_name, tar_folder):
+ """
+ Create the line refering to the setup method of the package of the
+ <package_name>.mk file
+
+ There are two things you can use to make an installer
+ for a python package: distutils or setuptools
+ distutils comes with python but does not support dependancies.
+ distutils is mostly still there for backward support.
+ setuptools is what smart people use,
+ but it is not shipped with python :(
+
+ Keyword Arguments:
+ pkg_name -- name of the package
+ tar_folder -- name of the folder where the setup.py can be found
+ """
+ lines = []
+ setup_type_line = 'PYTHON_{name}_SETUP_TYPE = {method}\n'.format(
+ name=pkg_name.upper(),
+ method=setup_info(tar_folder)['method'])
+ lines.append(setup_type_line)
+ return lines
+
+
+def create_mk_license(pkg_name, license_name, package_location):
+ """
+ Create the lines referring to the package's license informations of the
+ <package_name>.mk file
+
+ The license's files are found by searching the package for files named
+ license or license.txt (case insensitive).
+ If more than one license file is found, the user is asked to select which
+ ones he wants to use.
+
+ Keyword Arguments:
+ pkg_name -- name of the package
+ license_name -- name of the license
+ package_location -- where to look for the licenses
+ """
+ lines = []
+ license_line = 'PYTHON_{name}_LICENSE = {license}\n'.format(
+ name=pkg_name.upper(),
+ license=license_name)
+ lines.append(license_line)
+ print('WARNING: License has been set to "{license}",'
+ ' please change it manually if necessary'.format(
+ license=license_name))
+ filenames = ['LICENSE', 'LICENSE.TXT']
+ license_files = list(find_file_upper_case(filenames, package_location))
+ license_files = [license.replace(package_location, '')[1:]
+ for license in license_files]
+ if len(license_files) > 1:
+ print('More than one file found for license: ')
+ for index, item in enumerate(license_files):
+ print('\t{index})'.format(index=index), item)
+ license_choices = raw_input(
+ 'specify file numbers separated by spaces(default 0): ')
+ license_choices = [int(choice)
+ for choice in license_choices.split(' ')
+ if choice.isdigit() and int(choice) in
+ range(len(license_files))]
+ if len(license_choices) == 0:
+ license_choices = [0]
+ license_files = [file
+ for index, file in enumerate(license_files)
+ if index in license_choices]
+ elif len(license_files) == 0:
+ print('WARNING: No license file found,'
+ ' please specify it manually afterward')
+
+ license_file_line = ('PYTHON_{name}_LICENSE_FILES ='
+ ' {files}\n'.format(
+ name=pkg_name.upper(),
+ files=' '.join(license_files)))
+ license_file_line = license_file_line.replace(' \n', '\n')
+ lines.append(license_file_line)
+ return lines
+
+
+def create_mk_requirements(pkg_name, pkg_req):
+ """
+ Create the lines referring to the dependencies of the of the
+ <package_name>.mk file
+
+ Keyword Arguments:
+ pkg_name -- name of the package
+ pkg_req -- dependencies of the package
+ """
+ lines = []
+ python_pkg_req = ['python-{name}'.format(name=pkg)
+ for pkg in pkg_req]
+ dependencies_line = ('PYTHON_{name}_DEPENDENCIES ='
+ ' {reqs}\n'.format(
+ name=pkg_name.upper(),
+ reqs=' '.join(python_pkg_req)))
+ lines.append(dependencies_line)
+ return lines
+
+
+def create_config_mk(pkg_name, version, license, url, targz,
+ tar_folder, pkg_req, package_location):
+ """
+ Create the lines corresponding to the <package_name>.mk file
+
+ Keyword Arguments:
+ pkg_name -- name of the package
+ version -- version of the package
+ license -- name of the package's license
+ url -- where to download the package
+ targz -- name of the archive when downloaded
+ tar_folder -- name of the folder where the setup.py can be found
+ pkg_req -- dependencies of the package
+ package_location -- path to the extracted package
+ """
+ lines = create_mk_header(pkg_name)
+ lines += create_mk_download_info(pkg_name, version, targz, url)
+ lines += create_mk_setup(pkg_name, tar_folder)
+ lines += create_mk_license(pkg_name, license, package_location)
+ if pkg_req:
+ lines += create_mk_requirements(pkg_name, pkg_req)
+
+ lines.append('\n')
+ lines.append('$(eval $(python-package))')
+ lines.append('\n')
+
+ return lines
+
+
+def create_hash_file(url, digest, hash_function='sha356'):
+ """
+ Create the lines corresponding to the <package_name>.hash files
+
+ Keyword Arguments:
+ url -- metadata 'url' from the pypi json api
+ digest -- digest made from the downladed archive
+ hash_function -- algorythm used for hashing
+ """
+ lines = []
+ commented_line = '# {method} calculated by scanpypi\n'.format(
+ method=hash_function)
+ lines.append(commented_line)
+ hash_line = '{method}\t{digest} {filename}\n'.format(
+ method=hash_function,
+ digest=digest,
+ filename=url['filename'])
+ lines.append(hash_line)
+ return lines
+
+
+def create_config_in(pkg_name, pkg_req, package):
+ """
+ Creates the Config.in file of a package
+
+ pkg_name -- name of the package
+ pkg_req -- dependencies of the package
+ package -- metadata of the package from pypi
+ """
+ lines = []
+ config_line = 'config BR2_PACKAGE_PYTHON_{name}\n'.format(
+ name=pkg_name.upper())
+ lines.append(config_line)
+ python_line = '\tdepends on BR2_PACKAGE_PYTHON\n'
+ lines.append(python_line)
+
+ bool_line = '\tbool "python-{name}"\n'.format(name=pkg_name)
+ lines.append(bool_line)
+ if pkg_req:
+ for dep in pkg_req:
+ dep_line = '\tselect BR2_PACKAGE_PYTHON_{req}\n'.format(
+ req=dep.upper())
+ lines.append(dep_line)
+
+ lines.append('\thelp\n')
+
+ help_lines = package['info']['summary'].split('\n')
+ help_lines.append('')
+ help_lines.append(package['info']['home_page'])
+ help_lines = ['\t {line}\n'.format(line=line)
+ for line in help_lines]
+ lines += help_lines
+ return lines
+
+
+if __name__ == "__main__":
+
+ # Building the parser
+ parser = argparse.ArgumentParser(
+ description="Creates buildroot packages from the metadata of "
+ "an existing pypi(pip) packages and include it "
+ "in menuconfig")
+ parser.add_argument("packages",
+ help="list of packages to be made",
+ nargs='+')
+ parser.add_argument("-o", "--output",
+ help="""
+ Output directory for packages
+ """,
+ default='.')
+
+ args = parser.parse_args()
+ packages = list(set(args.packages))
+
+ # tmp_path is where we'll extract the files later
+ tmp_prefix = 'scanpypi-'
+ pkg_folder = args.output
+ tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
+
+ for real_pkg_name in packages:
+ pkg_name = pkg_buildroot_name(real_pkg_name)
+ print('buildroot package name for {}:'.format(real_pkg_name),
+ pkg_name)
+ # First we download the package
+ # Most of the info we need can only be found inside the package
+ print('Package:', pkg_name)
+ print('Fetching package', real_pkg_name)
+ pkg_json, url = fetch_package_info(real_pkg_name)
+ if not pkg_json:
+ continue
+
+ pkg_dir = pkg_folder + '/python-' + pkg_name
+ package = json.loads(pkg_json)
+ used_url = ''
+ print('Downloading package {pkg}...'.format(
+ pkg=package['info']['name']))
+ download, targz, used_url, as_file = download_package(package)
+ version = package['info']['version']
+
+ if not download:
+ print('Error downloading package :', pkg_name)
+ continue
+
+ sha256_digest = hashlib.sha256(as_file.read()).hexdigest()
+
+ # extract the tarball
+ tar_folder, tmp_extract = extract_package(pkg_name, as_file, tmp_path)
+
+ # Loading the package install info from the package
+ sys.path.append(tmp_extract)
+ print(tmp_extract)
+ import setup
+ setup = reload(setup)
+ sys.path.remove(tmp_extract)
+
+ pkg_req = None
+ # Package requierement are an argument of the setup function
+ if 'install_requires' in setup_info(tar_folder):
+ pkg_req = get_requirements(tar_folder)
+ # We could stop here
+ # or ask the user if he still wants to continue
+
+ # Buildroot python packages require 3 files
+ # The first is the mk file
+ # See:
+ # http://buildroot.uclibc.org/downloads/manual/manual.html
+ print('Checking if package {name} already exists...'.format(
+ name=pkg_dir))
+ try:
+ os.makedirs(pkg_dir)
+ except OSError as exception:
+ if exception.errno != errno.EEXIST:
+ print("ERROR: ", exception.message, file=sys.stderr)
+ continue
+ print('Error: Package {name} already exists'.format(name=pkg_dir))
+ del_pkg = raw_input(
+ 'Do you want to delete existing package ? [y/N]')
+ if del_pkg.lower() == 'y':
+ shutil.rmtree(pkg_dir)
+ os.makedirs(pkg_dir)
+ else:
+ continue
+ pkg_mk = 'python-{name}.mk'.format(name=pkg_name)
+ path_to_mk = '/'.join([pkg_dir, pkg_mk])
+ print('Creating {file}...'.format(file=path_to_mk))
+ config_mk_lines = create_config_mk(pkg_name, version,
+ package['info']['license'],
+ used_url, targz, tar_folder,
+ pkg_req, tmp_extract)
+ with open(path_to_mk, 'w') as mk_file:
+ mk_file.writelines(config_mk_lines)
+
+ # The second file we make is the hash file
+ # It consists of hashes of the package tarball
+ # http://buildroot.uclibc.org/downloads/manual/manual.html#adding-packages-hash
+ pkg_hash = 'python-{name}.hash'.format(name=pkg_name)
+ path_to_hash = '/'.join([pkg_dir, pkg_hash])
+ print('Creating {filename}...'.format(filename=path_to_hash))
+ hash_lines = create_hash_file(used_url, sha256_digest)
+ with open(path_to_hash, 'w') as hash_file:
+ hash_file.writelines(hash_lines)
+
+ # The Config.in is the last file we create
+ # It is used by buildroot's menuconfig, gconfig, xconfig or nconfig
+ # it is used to displayspackage info and to select requirements
+ # http://buildroot.uclibc.org/downloads/manual/manual.html#_literal_config_in_literal_file
+ path_to_config = '/'.join([pkg_dir, 'Config.in'])
+ print('Creating {file}...'.format(file=path_to_config))
+ config_in_lines = create_config_in(pkg_name, pkg_req, package)
+ with open(path_to_config, 'w') as config_file:
+ config_file.writelines(config_in_lines)
--
2.4.3
More information about the buildroot
mailing list