[Buildroot] [PATCH-NEXT v2 4/5] support/download/go-post-process: implement Go vendoring support

Christian Stewart christian at paral.in
Sun Sep 19 07:10:15 UTC 2021


From: Thomas Petazzoni <thomas.petazzoni at bootlin.com>

This commit introduces the download post-process script
support/download/go-post-process, and hooks it into the Go package
infrastructure.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
Signed-off-by: Christian Stewart <christian at paral.in>

v1 -> v2:

 - re-submitting Thomas's series with adjustments:
 - run "go mod init" just before "go mod vendor"
 - this fixes the case when go.mod does not exist
 - use -modcacherw to fix "make clean" permissions errors
 - use the mk_tar_gz helper in the post-process step

Signed-off-by: Christian Stewart <christian at paral.in>
---
 package/pkg-download.mk               |  1 +
 package/pkg-golang.mk                 |  8 +++++-
 support/download/dl-wrapper           |  6 +++--
 support/download/go-post-process      | 35 ++++++++++++++++++++++++
 support/download/post-process-helpers | 38 ++++++++++++---------------
 5 files changed, 64 insertions(+), 24 deletions(-)
 create mode 100755 support/download/go-post-process
 mode change 100644 => 100755 support/download/post-process-helpers

diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 66a415cce0..378b2c66f0 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -119,6 +119,7 @@ define DOWNLOAD
 		-o '$($(2)_DL_DIR)/$(notdir $(1))' \
 		$(if $($(2)_DOWNLOAD_POST_PROCESS),-p '$($(2)_DOWNLOAD_POST_PROCESS)') \
 		$(if $($(2)_GIT_SUBMODULES),-r) \
+		$(if $($(2)_GOMOD),-g '$($(2)_GOMOD)') \
 		$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \
 		$(QUIET) \
 		-- \
diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index d07242310d..3d2c45fa09 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -42,12 +42,13 @@ define inner-golang-package
 
 $(2)_BUILD_OPTS += \
 	-ldflags "$$($(2)_LDFLAGS)" \
+	-modcacherw \
 	-tags "$$($(2)_TAGS)" \
 	-trimpath \
 	-p $(PARALLEL_JOBS)
 
 # Target packages need the Go compiler on the host.
-$(2)_DEPENDENCIES += host-go
+$(2)_DOWNLOAD_DEPENDENCIES += host-go
 
 $(2)_BUILD_TARGETS ?= .
 
@@ -72,6 +73,11 @@ $(2)_SRC_SOFTWARE = $$(word 2,$$(subst /, ,$$(call notdomain,$$($(2)_SITE))))
 # If the go.mod file does not exist, one is written with this root path.
 $(2)_GOMOD ?= $$($(2)_SRC_DOMAIN)/$$($(2)_SRC_VENDOR)/$$($(2)_SRC_SOFTWARE)
 
+$(2)_DOWNLOAD_POST_PROCESS = go
+$(2)_DL_ENV = \
+	$(HOST_GO_COMMON_ENV) \
+	GOPROXY=direct
+
 # Generate a go.mod file if it doesn't exist. Note: Go is configured
 # to use the "vendor" dir and not make network calls.
 define $(2)_GEN_GOMOD
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 2d74554213..2fc530f24f 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -17,7 +17,7 @@
 # We want to catch any unexpected failure, and exit immediately.
 set -e
 
-export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
+export BR_BACKEND_DL_GETOPTS=":hc:d:g:o:n:N:H:ru:qf:e"
 
 main() {
     local OPT OPTARG
@@ -25,11 +25,12 @@ main() {
     local -a uris
 
     # Parse our options; anything after '--' is for the backend
-    while getopts ":c:d:D:o:n:N:H:rf:u:qp:" OPT; do
+    while getopts ":c:d:D:g:o:n:N:H:rf:u:qp:" OPT; do
         case "${OPT}" in
         c)  cset="${OPTARG}";;
         d)  dl_dir="${OPTARG}";;
         D)  old_dl_dir="${OPTARG}";;
+        g)  gomod_init="${OPTARG}";;
         o)  output="${OPTARG}";;
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
@@ -138,6 +139,7 @@ main() {
 
         if [ -n "${post_process}" ] ; then
                 ${OLDPWD}/support/download/${post_process}-post-process \
+                         -g "${gomod_init}" \
                          -o "${tmpf}" \
                          -n "${raw_base_name}"
         fi
diff --git a/support/download/go-post-process b/support/download/go-post-process
new file mode 100755
index 0000000000..b7f05b37c1
--- /dev/null
+++ b/support/download/go-post-process
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+. "${0%/*}/post-process-helpers"
+
+# Parse our options
+while getopts "n:g:o:" OPT; do
+        case "${OPT}" in
+        g)  gomod_init="${OPTARG}";;
+        o)  output="${OPTARG}";;
+        n)  base_name="${OPTARG}";;
+        :)  error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
+        \?) error "unknown option '%s'\n" "${OPTARG}";;
+        esac
+done
+
+# Already vendored tarball, nothing to do
+if tar tf ${output} | grep -q "^[^/]*/vendor" ; then
+	exit 0
+fi
+
+unpack ${base_name} ${output}
+
+# Do the Go vendoring
+pushd ${base_name} > /dev/null
+# modcacherw option leaves directories in the module cache at their default
+# permissions rather than making them read-only.
+if [ ! -f go.mod ] && [ -n "${gomod_init}" ]; then
+    go mod init -modcacherw ${gomod_init}
+fi
+go mod vendor -modcacherw -v
+popd > /dev/null
+
+repack $(pwd) ${base_name} ${output}
diff --git a/support/download/post-process-helpers b/support/download/post-process-helpers
old mode 100644
new mode 100755
index bed8df2577..b3231207a6
--- a/support/download/post-process-helpers
+++ b/support/download/post-process-helpers
@@ -1,30 +1,26 @@
+# Helpers for the post-process step to re-pack vendored archives.
+
+. "${0%/*}/helpers"
 
 unpack() {
-        dest="$1"
-        tarball="$2"
+    dest="$1"
+    tarball="$2"
 
-        mkdir ${dest}
-        tar -C ${dest} --strip-components=1 -xf ${tarball}
+    mkdir ${dest}
+    tar -C ${dest} --strip-components=1 -xf ${tarball}
 }
 
 repack() {
-        src="$1"
-        tarball="$2"
-
-        # Generate the archive, sort with the C locale so that it is reproducible.
-        find "$(basename ${src})" -not -type d -print0 >files.list
-        LC_ALL=C sort -z <files.list >files.list.sorted
+    local in_dir="${1}"
+    local base_dir="${2}"
+    local out="${3}"
 
-        # let's use a fixed hardcoded date to be reproducible
-        date="2020-02-06 01:02:03 +0000"
+    # let's use a fixed hardcoded date to be reproducible
+    date="2020-02-06 01:02:03 +0000"
 
-        # Create GNU-format tarballs, since that's the format of the tarballs on
-        # sources.buildroot.org and used in the *.hash files
-        tar cf new.tar --null --verbatim-files-from --numeric-owner --format=gnu \
-            --owner=0 --group=0 --mtime="${date}" -T files.list.sorted
-        gzip -6 -n <new.tar >new.tar.gz
-        mv "${tarball}" "${tarball}".old
-        mv new.tar.gz "${tarball}"
-        rm "${tarball}".old
-        rm -rf ${src}
+    # use the helper to create a reproducible archive
+    mk_tar_gz "${in_dir}/${base_dir}" "${base_dir}" "${date}" "${out}"
 }
+
+# Keep this line and the following as last lines in this file.
+# vim: ft=bash
-- 
2.33.0



More information about the buildroot mailing list