[Buildroot] [PATCH v4] python: rework *.pyc files generation
Samuel Martin
s.martin49 at gmail.com
Fri Dec 11 22:22:52 UTC 2015
Hi Yegor, all,
On Thu, Nov 19, 2015 at 4:52 PM, <yegorslists at googlemail.com> wrote:
> From: Yegor Yefremov <yegorslists at googlemail.com>
>
> This patch generates all *.pyc files from *.py located in
> <target>/usr/lib/python*, before stripping the rootfs.
>
> *.pyc generation on per package basis is disabled through
> "--no-compile" option.
>
> *.pyc generation for Python's internal packages is disabled
> through --disable-pyc-build configure option.
>
> This prevents modules from packages, that do not compile the *.py
> files, from disappearing.
AFAI have tested, this change works as expected, except for broken
python modules.
In such cases, 'compileall.compile_dir(...)' does not raise any
exceptions when it failed to compile the *.py file, leading to the
unfortunate consequence: the *.pyc file is not generated and the *.py
file is lost if BR2_PACKAGE_PYTHON_PYC_ONLY is enabled.
This raises the question: do we want to make the build fail if some
*.pyc generation failed?
IMHO, the answer is yes.
Digging a bit this issue, the solution [2] inspired from [1] works
fine with python2 and python3.
Note: with python2: compileall.compile_dir fails on python-pyxml
package; with python3 I have not trigger any failure but I only tested
a limited number of packages
[1] http://stackoverflow.com/questions/615632/how-to-detect-errors-from-compileall-compile-dir
[2] http://code.bulix.org/2zk0mg-89949?raw
Regards,
>
> Signed-off-by: Yegor Yefremov <yegorslists at googlemail.com>
> Cc: Samuel Martin <s.martin49 at gmail.com>
> ---
> Changes:
> v4: disable *.pyc creation for Python own modules (Thomas Petazzoni)
> v3: put pyc compilation command into own macro (Thomas Petazzoni)
> v2: move finalization code from root Makefile to python.mk and python3.mk (Thomas Petazzoni)
>
> package/pkg-python.mk | 2 +-
> package/python/python.mk | 16 +++++++++++++++-
> package/python3/python3.mk | 20 +++++++++++++++-----
> 3 files changed, 31 insertions(+), 7 deletions(-)
>
> diff --git a/package/pkg-python.mk b/package/pkg-python.mk
> index b7a702d..9725d24 100644
> --- a/package/pkg-python.mk
> +++ b/package/pkg-python.mk
> @@ -253,7 +253,7 @@ ifndef $(2)_INSTALL_TARGET_CMDS
> define $(2)_INSTALL_TARGET_CMDS
> (cd $$($$(PKG)_BUILDDIR)/; \
> $$($$(PKG)_BASE_ENV) $$($$(PKG)_ENV) \
> - $$($(2)_PYTHON_INTERPRETER) setup.py install \
> + $$($(2)_PYTHON_INTERPRETER) setup.py install --no-compile \
> $$($$(PKG)_BASE_INSTALL_TARGET_OPTS) \
> $$($$(PKG)_INSTALL_TARGET_OPTS))
> endef
> diff --git a/package/python/python.mk b/package/python/python.mk
> index 80ffbde..5ef3589 100644
> --- a/package/python/python.mk
> +++ b/package/python/python.mk
> @@ -143,7 +143,8 @@ PYTHON_CONF_OPTS += \
> --disable-tk \
> --disable-nis \
> --disable-dbm \
> - --disable-pyo-build
> + --disable-pyo-build \
> + --disable-pyc-build
>
> # This is needed to make sure the Python build process doesn't try to
> # regenerate those files with the pgen program. Otherwise, it builds
> @@ -217,12 +218,25 @@ PYTHON_PATH = $(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/sysconfigdata/
> $(eval $(autotools-package))
> $(eval $(host-autotools-package))
>
> +define PYTHON_CREATE_PYC_FILES
> + PYTHONPATH="$(PYTHON_PATH)" \
> + $(HOST_DIR)/usr/bin/python$(PYTHON_VERSION_MAJOR) -c "import compileall; \
> + compileall.compile_dir('$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)')"
> +endef
> +
> ifeq ($(BR2_PACKAGE_PYTHON_PYC_ONLY),y)
> define PYTHON_FINALIZE_TARGET
> + $(PYTHON_CREATE_PYC_FILES)
> find $(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR) -name '*.py' -print0 | xargs -0 rm -f
> endef
> endif
>
> +ifeq ($(BR2_PACKAGE_PYTHON_PY_PYC),y)
> +define PYTHON_FINALIZE_TARGET
> + $(PYTHON_CREATE_PYC_FILES)
> +endef
> +endif
> +
> ifeq ($(BR2_PACKAGE_PYTHON_PY_ONLY),y)
> define PYTHON_FINALIZE_TARGET
> find $(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR) -name '*.pyc' -print0 | xargs -0 rm -f
> diff --git a/package/python3/python3.mk b/package/python3/python3.mk
> index fd5709a..c87092f 100644
> --- a/package/python3/python3.mk
> +++ b/package/python3/python3.mk
> @@ -79,10 +79,6 @@ ifeq ($(BR2_PACKAGE_PYTHON3_PYC_ONLY),y)
> PYTHON3_CONF_OPTS += --enable-old-stdlib-cache
> endif
>
> -ifeq ($(BR2_PACKAGE_PYTHON3_PY_ONLY),y)
> -PYTHON3_CONF_OPTS += --disable-pyc-build
> -endif
> -
> ifeq ($(BR2_PACKAGE_PYTHON3_SQLITE),y)
> PYTHON3_DEPENDENCIES += sqlite
> else
> @@ -137,7 +133,8 @@ PYTHON3_CONF_OPTS += \
> --disable-tk \
> --disable-nis \
> --disable-idle3 \
> - --disable-pyo-build
> + --disable-pyo-build \
> + --disable-pyc-build
>
> # This is needed to make sure the Python build process doesn't try to
> # regenerate those files with the pgen program. Otherwise, it builds
> @@ -208,12 +205,25 @@ PYTHON3_PATH = $(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)/sysconfigdat
> $(eval $(autotools-package))
> $(eval $(host-autotools-package))
>
> +define PYTHON3_CREATE_PYC_FILES
> + PYTHONPATH="$(PYTHON3_PATH)" \
> + $(HOST_DIR)/usr/bin/python$(PYTHON3_VERSION_MAJOR) -c "import compileall; \
> + compileall.compile_dir('$(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)')"
> +endef
> +
> ifeq ($(BR2_PACKAGE_PYTHON3_PYC_ONLY),y)
> define PYTHON3_FINALIZE_TARGET
> + $(PYTHON3_CREATE_PYC_FILES)
> find $(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR) -name '*.py' -print0 | xargs -0 rm -f
> endef
> endif
>
> +ifeq ($(BR2_PACKAGE_PYTHON3_PY_PYC),y)
> +define PYTHON3_FINALIZE_TARGET
> + $(PYTHON3_CREATE_PYC_FILES)
> +endef
> +endif
> +
> ifeq ($(BR2_PACKAGE_PYTHON3_PY_ONLY),y)
> define PYTHON3_FINALIZE_TARGET
> find $(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR) -name '*.pyc' -print0 | xargs -0 rm -f
> --
> 2.1.4
>
--
Samuel
More information about the buildroot
mailing list