diff --git a/.gitignore b/.gitignore index 92d6b778..e370ceda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +aci-builder/files/dgr/usr/bin/tar* + dist/ target/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 3a262c00..bec76df2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,8 @@ install: - sudo rm -rf /var/lib/apt/lists/* - sudo add-apt-repository ppa:duggan/bats --yes - sudo apt-get update -qq - - sudo apt-get install -qq upx + - sudo apt-get install -qq upx coreutils + - sudo apt-get install -qq realpath || true - sudo ./scripts/travis-install-rkt.sh - sudo apt-get install -qq bats diff --git a/aci-builder/bin-run/builder.go b/aci-builder/bin-run/builder.go index 56455238..4740e2d1 100644 --- a/aci-builder/bin-run/builder.go +++ b/aci-builder/bin-run/builder.go @@ -136,39 +136,36 @@ func (b *Builder) tarAci() error { } upperPath := b.pod.Root + PATH_OVERLAY + "/" + upperId + PATH_UPPER - upperNamedRootfs := upperPath + "/" + manifestApp(b.pod).Name.String() - upperRootfs := upperPath + common.PathRootfs - - if err := os.RemoveAll(upperNamedRootfs + PATH_TMP); err != nil { - logs.WithEF(err, b.fields.WithField("path", upperNamedRootfs+PATH_TMP)).Warn("Failed to clean tmp directory") - } - - if err := os.Rename(upperNamedRootfs, upperRootfs); err != nil { // TODO this is dirty and can probably be renamed during tar - return errs.WithEF(err, b.fields.WithField("path", upperNamedRootfs), "Failed to rename rootfs") - } - defer os.Rename(upperRootfs, upperNamedRootfs) - - dir, err := os.Getwd() - if err != nil { - return errs.WithEF(err, b.fields, "Failed to get current working directory") - } - defer func() { - if err := os.Chdir(dir); err != nil { - logs.WithEF(err, b.fields.WithField("path", dir)).Warn("Failed to chdir back") + rootfsAlias := manifestApp(b.pod).Name.String() + destination := b.aciTargetPath + common.PathImageAci // absolute dir, outside upperPath (think: /tmp/…) + + params := []string{"--sort=name", "--numeric-owner", "--exclude", rootfsAlias + PATH_TMP + "/*"} + params = append(params, "-C", upperPath, "--transform", "s@^"+rootfsAlias+"@rootfs@") + params = append(params, "-cf", destination, common.PathManifest[1:], rootfsAlias) + + logs.WithF(b.fields).Debug("Calling tar to collect all files") + if err := common.ExecCmd("tar", params...); err != nil { + // In case the host's tar is too old, try the builder's if it exists. + stage1Tar := rktcommon.Stage1RootfsPath(b.pod.Root) + var buildersTar string + for _, t := range []string{"/dgr/usr/bin/tar", "/bin/tar", "/usr/bin/tar"} { + buildersTar = filepath.Join(stage1Tar, t) + if _, err := os.Stat(buildersTar); err == nil { + break + } } - }() - if err := os.Chdir(upperPath); err != nil { - return errs.WithEF(err, b.fields.WithField("path", upperPath), "Failed to chdir to upper base path") - } - if err := common.Tar(b.aciTargetPath+common.PathImageAci, common.PathManifest[1:], common.PathRootfs[1:]+"/"); err != nil { - return errs.WithEF(err, b.fields, "Failed to tar aci") + if err2 := common.ExecCmd(buildersTar, params...); err2 != nil { + // If that failed, output the original error nevertheless. + logs.WithFields(b.fields).WithField("params", params).Debug("Parameters to 'tar' within the builder") + return errs.WithEF(err, b.fields, "Failed to tar aci") + } + logs.WithF(b.fields).Debug("Had to resort to 'tar' from the builder to create the aci file") } // common.ExecCmd sometimes silently fails, hence the redundant check. - if _, err := os.Stat(b.aciTargetPath + common.PathImageAci); os.IsNotExist(err) { + if _, err := os.Stat(destination); os.IsNotExist(err) { return errs.WithEF(err, b.fields, "Expected aci has not been created") } - logs.WithField("path", dir).Debug("chdir") return nil } diff --git a/aci-builder/build-tar.sh b/aci-builder/build-tar.sh new file mode 100755 index 00000000..c39eafc1 --- /dev/null +++ b/aci-builder/build-tar.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -euo pipefail +if [[ ! -z ${debug+x} ]]; then + set -x +fi + +start=$(date +%s) +dir="$(dirname $0)" +mkdir -p "${dir}/files/dgr/usr/bin" +: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"} + +echo -e "\033[0;32mBuilding tar\033[0m\n" + +# 'core2' is very old, but without setting an option some GCC versions +# will pick built-in defaults – which could be as "recent" as 'silvermont', +# thus excluding CPUs without SSE4 (like some pre-2007 AMDs still in the wild). +: ${CPU_SETTING:="-march=core2 -mtune=intel"} +if [[ "$(uname -m)" != "x86_64" ]]; then + CPU_SETTING="" +fi + +WORKDIR="$(mktemp -d -t aci-builder-tar.XXXXXX)" +pushd . &>/dev/null +cd ${WORKDIR} +curl --silent --show-error --fail --location \ + -H "accept: application/x-xz, application/x-tar, application/tar+xz" \ + https://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz \ +| tar --strip-components=1 --xz -x + +if ! ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \ + CFLAGS="-Os ${CPU_SETTING} -ffunction-sections -fdata-sections -fstack-protector-strong -fpie -fpic" \ + LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--as-needed -Wl,--strip-all -Wl,--gc-sections" >/dev/null; then + + # Old compiler. Most probably in a Travis-CI VM. Resort to conservative settings. + echo "== First run of configure failed. Trying fallback options:" + ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \ + CFLAGS="-Os ${CPU_SETTING%% *} -ffunction-sections -fdata-sections" \ + LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--strip-all -Wl,--gc-sections" >/dev/null +fi +make -j$(nproc) >/dev/null + +popd &>/dev/null +mv ${WORKDIR}/src/tar "${dir}/files/dgr/usr/bin/tar" +rm -r ${WORKDIR} + +echo -e "\033[0;32mBuilding tar took: $[ $(date +%s) - ${start} ]s\033[0m\n" + +exit 0 diff --git a/aci-builder/build.sh b/aci-builder/build.sh index 5e3e5c9b..0c980608 100755 --- a/aci-builder/build.sh +++ b/aci-builder/build.sh @@ -16,26 +16,28 @@ mkdir -p ${rootfs}/dgr ${rootfs}/usr/bin GOOS=linux GOARCH=amd64 go build --ldflags '-s -w -extldflags "-static"' -o ${rootfs}/dgr/builder/stage1/run ${dir}/bin-run upx ${rootfs}/dgr/builder/stage1/run -sudo tar xf ${dir}/rootfs.tar.xz -C ${rootfs}/dgr/ +: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"} + +sudo "${tar}" \ + --transform "s:usr/sbin/haveged:usr/bin/haveged:" \ + --exclude "./etc/udev" \ + --exclude "./usr/share/locale" \ + --exclude "./usr/libexec" \ + --exclude "./usr/lib/systemd" \ + --exclude "./usr/lib/udev" \ + --exclude "./usr/sbin" \ + -C ${rootfs}/dgr/ \ + -xf ${dir}/rootfs.tar.xz sudo cp -R ${dir}/files/. ${rootfs} sudo chown root: ${rootfs} cp ${dir}/manifest.json ${target}/manifest sudo cp --no-preserve=ownership ${dist}/templater ${rootfs}/dgr/usr/bin/ -# some cleanup -sudo rm -Rf ${rootfs}/dgr/etc/udev -sudo rm -Rf ${rootfs}/dgr/usr/share/locale -sudo rm -Rf ${rootfs}/dgr/usr/libexec -sudo rm -Rf ${rootfs}/dgr/usr/lib/systemd -sudo rm -Rf ${rootfs}/dgr/usr/lib/udev - - -sudo mv ${rootfs}/dgr/usr/sbin/haveged ${rootfs}/dgr/usr/bin/haveged -sudo rm -Rf ${rootfs}/dgr/usr/sbin/ sudo bash -c "cd ${rootfs}/dgr/usr && ln -s bin sbin && cd -" cd ${target} -sudo tar cpfz ../bindata/aci-builder.aci rootfs manifest +sudo "${tar}" --sort=name --numeric-owner \ + -cpzf ../bindata/aci-builder.aci manifest rootfs sudo chown ${USER}: ../bindata/aci-builder.aci sudo rm -Rf rootfs/ cd - diff --git a/aci-tester/build.sh b/aci-tester/build.sh index c779d028..29bfe4ed 100755 --- a/aci-tester/build.sh +++ b/aci-tester/build.sh @@ -30,6 +30,9 @@ curl --fail --silent --show-error --location --remote-time --compressed --create chmod +x ${rootfs}/dgr/usr/bin/* +: ${tar:="$(realpath "${dir}/../aci-builder/files/dgr/usr/bin/tar")"} cd ${target} -tar cpfz ../bindata/aci-tester.aci rootfs manifest +"${tar}" --sort=name --numeric-owner \ + --owner=0 --group=0 \ + -cpzf ../bindata/aci-tester.aci manifest rootfs cd - diff --git a/dgr/common/tar.go b/dgr/common/tar.go deleted file mode 100644 index 1da0af8a..00000000 --- a/dgr/common/tar.go +++ /dev/null @@ -1,11 +0,0 @@ -package common - -func Tar(destination string, source ...string) error { // remove zip - source = append(source, "") - source = append(source, "") - copy(source[2:], source[0:]) - source[0] = "cpf" - source[1] = destination - - return ExecCmd("tar", source...) -} diff --git a/gomake.cfg b/gomake.cfg index d81d3150..03fb9576 100644 --- a/gomake.cfg +++ b/gomake.cfg @@ -4,6 +4,10 @@ release_osarchi=linux-amd64 upx=true pre-build() { + local tar="$(realpath "$(dirname $0)/aci-builder/files/dgr/usr/bin/tar")" + if [[ ! -x "${tar}" ]] || ! "${tar}" --help | grep -q -F sort; then + ${work_path}/aci-builder/build-tar.sh + fi [ -f ${work_path}/${target_name}/templater ] || ${work_path}/bin-templater/build.sh [ -f ${work_path}/${target_name}/bindata/aci-tester.aci ] || ${work_path}/aci-tester/build.sh [ -f ${work_path}/${target_name}/bindata/aci-builder.aci ] || ${work_path}/aci-builder/build.sh