[Buildroot] [PATCH-NEXT v2 4/5] support/download/go-post-process: implement Go vendoring support
Christian Stewart
christian at paral.in
Sun Sep 19 07:10:15 UTC 2021
From: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
This commit introduces the download post-process script
support/download/go-post-process, and hooks it into the Go package
infrastructure.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
Signed-off-by: Christian Stewart <christian at paral.in>
v1 -> v2:
- re-submitting Thomas's series with adjustments:
- run "go mod init" just before "go mod vendor"
- this fixes the case when go.mod does not exist
- use -modcacherw to fix "make clean" permissions errors
- use the mk_tar_gz helper in the post-process step
Signed-off-by: Christian Stewart <christian at paral.in>
---
package/pkg-download.mk | 1 +
package/pkg-golang.mk | 8 +++++-
support/download/dl-wrapper | 6 +++--
support/download/go-post-process | 35 ++++++++++++++++++++++++
support/download/post-process-helpers | 38 ++++++++++++---------------
5 files changed, 64 insertions(+), 24 deletions(-)
create mode 100755 support/download/go-post-process
mode change 100644 => 100755 support/download/post-process-helpers
diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 66a415cce0..378b2c66f0 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -119,6 +119,7 @@ define DOWNLOAD
-o '$($(2)_DL_DIR)/$(notdir $(1))' \
$(if $($(2)_DOWNLOAD_POST_PROCESS),-p '$($(2)_DOWNLOAD_POST_PROCESS)') \
$(if $($(2)_GIT_SUBMODULES),-r) \
+ $(if $($(2)_GOMOD),-g '$($(2)_GOMOD)') \
$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \
$(QUIET) \
-- \
diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index d07242310d..3d2c45fa09 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -42,12 +42,13 @@ define inner-golang-package
$(2)_BUILD_OPTS += \
-ldflags "$$($(2)_LDFLAGS)" \
+ -modcacherw \
-tags "$$($(2)_TAGS)" \
-trimpath \
-p $(PARALLEL_JOBS)
# Target packages need the Go compiler on the host.
-$(2)_DEPENDENCIES += host-go
+$(2)_DOWNLOAD_DEPENDENCIES += host-go
$(2)_BUILD_TARGETS ?= .
@@ -72,6 +73,11 @@ $(2)_SRC_SOFTWARE = $$(word 2,$$(subst /, ,$$(call notdomain,$$($(2)_SITE))))
# If the go.mod file does not exist, one is written with this root path.
$(2)_GOMOD ?= $$($(2)_SRC_DOMAIN)/$$($(2)_SRC_VENDOR)/$$($(2)_SRC_SOFTWARE)
+$(2)_DOWNLOAD_POST_PROCESS = go
+$(2)_DL_ENV = \
+ $(HOST_GO_COMMON_ENV) \
+ GOPROXY=direct
+
# Generate a go.mod file if it doesn't exist. Note: Go is configured
# to use the "vendor" dir and not make network calls.
define $(2)_GEN_GOMOD
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 2d74554213..2fc530f24f 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -17,7 +17,7 @@
# We want to catch any unexpected failure, and exit immediately.
set -e
-export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
+export BR_BACKEND_DL_GETOPTS=":hc:d:g:o:n:N:H:ru:qf:e"
main() {
local OPT OPTARG
@@ -25,11 +25,12 @@ main() {
local -a uris
# Parse our options; anything after '--' is for the backend
- while getopts ":c:d:D:o:n:N:H:rf:u:qp:" OPT; do
+ while getopts ":c:d:D:g:o:n:N:H:rf:u:qp:" OPT; do
case "${OPT}" in
c) cset="${OPTARG}";;
d) dl_dir="${OPTARG}";;
D) old_dl_dir="${OPTARG}";;
+ g) gomod_init="${OPTARG}";;
o) output="${OPTARG}";;
n) raw_base_name="${OPTARG}";;
N) base_name="${OPTARG}";;
@@ -138,6 +139,7 @@ main() {
if [ -n "${post_process}" ] ; then
${OLDPWD}/support/download/${post_process}-post-process \
+ -g "${gomod_init}" \
-o "${tmpf}" \
-n "${raw_base_name}"
fi
diff --git a/support/download/go-post-process b/support/download/go-post-process
new file mode 100755
index 0000000000..b7f05b37c1
--- /dev/null
+++ b/support/download/go-post-process
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+. "${0%/*}/post-process-helpers"
+
+# Parse our options
+while getopts "n:g:o:" OPT; do
+ case "${OPT}" in
+ g) gomod_init="${OPTARG}";;
+ o) output="${OPTARG}";;
+ n) base_name="${OPTARG}";;
+ :) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
+ \?) error "unknown option '%s'\n" "${OPTARG}";;
+ esac
+done
+
+# Already vendored tarball, nothing to do
+if tar tf ${output} | grep -q "^[^/]*/vendor" ; then
+ exit 0
+fi
+
+unpack ${base_name} ${output}
+
+# Do the Go vendoring
+pushd ${base_name} > /dev/null
+# modcacherw option leaves directories in the module cache at their default
+# permissions rather than making them read-only.
+if [ ! -f go.mod ] && [ -n "${gomod_init}" ]; then
+ go mod init -modcacherw ${gomod_init}
+fi
+go mod vendor -modcacherw -v
+popd > /dev/null
+
+repack $(pwd) ${base_name} ${output}
diff --git a/support/download/post-process-helpers b/support/download/post-process-helpers
old mode 100644
new mode 100755
index bed8df2577..b3231207a6
--- a/support/download/post-process-helpers
+++ b/support/download/post-process-helpers
@@ -1,30 +1,26 @@
+# Helpers for the post-process step to re-pack vendored archives.
+
+. "${0%/*}/helpers"
unpack() {
- dest="$1"
- tarball="$2"
+ dest="$1"
+ tarball="$2"
- mkdir ${dest}
- tar -C ${dest} --strip-components=1 -xf ${tarball}
+ mkdir ${dest}
+ tar -C ${dest} --strip-components=1 -xf ${tarball}
}
repack() {
- src="$1"
- tarball="$2"
-
- # Generate the archive, sort with the C locale so that it is reproducible.
- find "$(basename ${src})" -not -type d -print0 >files.list
- LC_ALL=C sort -z <files.list >files.list.sorted
+ local in_dir="${1}"
+ local base_dir="${2}"
+ local out="${3}"
- # let's use a fixed hardcoded date to be reproducible
- date="2020-02-06 01:02:03 +0000"
+ # let's use a fixed hardcoded date to be reproducible
+ date="2020-02-06 01:02:03 +0000"
- # Create GNU-format tarballs, since that's the format of the tarballs on
- # sources.buildroot.org and used in the *.hash files
- tar cf new.tar --null --verbatim-files-from --numeric-owner --format=gnu \
- --owner=0 --group=0 --mtime="${date}" -T files.list.sorted
- gzip -6 -n <new.tar >new.tar.gz
- mv "${tarball}" "${tarball}".old
- mv new.tar.gz "${tarball}"
- rm "${tarball}".old
- rm -rf ${src}
+ # use the helper to create a reproducible archive
+ mk_tar_gz "${in_dir}/${base_dir}" "${base_dir}" "${date}" "${out}"
}
+
+# Keep this line and the following as last lines in this file.
+# vim: ft=bash
--
2.33.0
More information about the buildroot
mailing list