diff --git a/Dockerfile b/Dockerfile index 811699e..343a52f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ COPY --from=datefudge /usr/bin/datefudge /usr/bin/datefudge COPY --from=resizefat32 /usr/bin/resizefat32 /usr/bin/resizefat32 RUN curl "https://github.com/gardenlinux/aws-kms-pkcs11/releases/download/latest/aws_kms_pkcs11-$(dpkg --print-architecture).so" -sLo "/usr/lib/$(uname -m)-linux-gnu/pkcs11/aws_kms_pkcs11.so" COPY builder /builder +RUN python3 -m pip install --break-system-packages -r "/builder/requirements.txt" --root-user-action ignore RUN mkdir /builder/cert COPY setup_namespace /usr/sbin/setup_namespace RUN curl -sSLf https://github.com/gardenlinux/seccomp_fake_xattr/releases/download/latest/seccomp_fake_xattr-$(uname -m).tar.gz \ diff --git a/build b/build index aebd69b..7f95e92 100755 --- a/build +++ b/build @@ -97,12 +97,9 @@ commit="$(./get_commit)" timestamp="$(./get_timestamp)" default_version="$(./get_version)" - if [ "$resolve_cname" = 1 ]; then arch="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" dpkg --print-architecture)" - cname="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" /builder/parse_features --feature-dir /builder/features --default-arch "$arch" --default-version "$default_version" --cname "$1")" - short_commit="$(head -c 8 <<< "$commit")" - echo "$cname-$short_commit" >&3 + "$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" gl-cname --feature-dir /builder/features --arch "$arch" --version "${default_version}-${commit}" "$1" exit 0 fi diff --git a/builder/Makefile b/builder/Makefile index e37705f..ecd00de 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -10,10 +10,11 @@ export BASH_ENV := make_bash_env MAKEFLAGS += --no-builtin-rules -lastword = $(word $(words $1),$1) -prelastword = $(word $(words $1),_ $1) -cname_version = $(call lastword,$(subst -, ,$1)) -cname_arch = $(call prelastword,$(subst -, ,$1)) +cname_parts = $(subst _, , $(subst -, , $1)) +cname_gl_commit = $(lastword $(call cname_parts,$1)) +gl_version = $(lastword $(filter-out $(call cname_gl_commit,$1), $(call cname_parts,$1))) +cname_gl_version = $(call gl_version,$1)-$(call cname_gl_commit,$1) +cname_arch = $(lastword $(filter-out $(call gl_version,$1) $(call cname_gl_commit,$1), $(call cname_parts,$1))) define require_var = ifndef $1 @@ -24,11 +25,13 @@ endef required_vars := REPO COMMIT TIMESTAMP DEFAULT_VERSION TEMPFS_SIZE $(foreach var,$(required_vars),$(eval $(call require_var,$(var)))) -PARSE_FEATURES_ARGS := +GL_ALLOW_FRANKENSTEIN := ifdef ALLOW_FRANKENSTEIN -PARSE_FEATURES_ARGS := --allow-frankenstein +GL_ALLOW_FRANKENSTEIN := true endif +export GL_ALLOW_FRANKENSTEIN + SHORT_COMMIT := $(shell head -c 8 <<< '$(COMMIT)') DEFAULT_ARCH := $(shell dpkg --print-architecture) @@ -43,22 +46,22 @@ clean: .build/%.sentinel: true -.build/bootstrap-%-$(SHORT_COMMIT).tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call cname_version,$$*)) +.build/bootstrap-%.tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call gl_version,$$*)) target '$@' - info 'bootstrapping $*-$(SHORT_COMMIT)' + info 'bootstrapping $*' arch='$(call cname_arch,$*)' - version='$(call cname_version,$*)' + version='$(call gl_version,$*)' ./bootstrap "$$arch" "$$version" '$(REPO)' keyring.gpg '$@' -.build/%-$(SHORT_COMMIT).tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_version,$$*)-$(SHORT_COMMIT).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert) +.build/%.tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_gl_version,$$*).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert) target '$@' '$<' info 'configuring rootfs $*-$(SHORT_COMMIT)' - features="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' features)" - features_platforms="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' platforms)" - features_elements="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' elements)" - features_flags="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' flags)" + features="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*' features)" + features_platforms="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'platforms)" + features_elements="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'elements)" + features_flags="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'flags)" BUILDER_CNAME='$*' - BUILDER_VERSION='$(call cname_version,$*)' + BUILDER_VERSION='$(call gl_version,$*)' BUILDER_ARCH='$(call cname_arch,$*)' BUILDER_TIMESTAMP='$(TIMESTAMP)' BUILDER_COMMIT='$(COMMIT)' @@ -70,17 +73,17 @@ clean: ./configure '$(word 1,$^)' '$@' define artifact_template = -.build/%-$(SHORT_COMMIT).$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert) +.build/%.$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert) script='$$(word 1,$$^)' input='$$(word 2,$$^)' target '$$@' "$$$$input" info 'building $1 image $$*' - features="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' features)" - features_platforms="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' platforms)" - features_elements="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' elements)" - features_flags="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' flags)" + features="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' features)" + features_platforms="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' platforms)" + features_elements="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' elements)" + features_flags="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' flags)" BUILDER_CNAME='$$*' - BUILDER_VERSION='$$(call cname_version,$$*)' + BUILDER_VERSION='$$(call gl_version,$$*)' BUILDER_ARCH='$$(call cname_arch,$$*)' BUILDER_TIMESTAMP='$$(TIMESTAMP)' BUILDER_COMMIT='$$(COMMIT)' @@ -94,7 +97,7 @@ endef $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifact_template,$(artifact_rule)))) -.build/%-$(SHORT_COMMIT).artifacts: $$(shell COMMIT=$(SHORT_COMMIT) ./make_list_build_artifacts '$$*') +.build/%.artifacts: $$(shell COMMIT=$(SHORT_COMMIT) DEFAULT_VERSION=$(DEFAULT_VERSION) NATIVE_ARCH=$(NATIVE_ARCH) ./make_list_build_artifacts '$$*') target '$@' echo -n > '$@' for f in $^; do @@ -102,7 +105,7 @@ $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifac echo "$$(basename "$$f").log" | tee -a '$@' done -%: .build/$$(shell ./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)' --cname '$$*')-$(SHORT_COMMIT).artifacts +%: .build/$$(shell gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)-$$(SHORT_COMMIT)' --cname '$$*').artifacts ln -f -s -r '$<' '.build/$*' # prevents match anything rule from applying to Makefile and image/convert scripts diff --git a/builder/make_get_image_dependencies b/builder/make_get_image_dependencies index f8789a0..f90edaf 100755 --- a/builder/make_get_image_dependencies +++ b/builder/make_get_image_dependencies @@ -7,7 +7,7 @@ exec 1>&2 # get longest chain of extensions, but not extensions starting with a number to prevent parsing minor version as extension extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_]*)*$' <<< "$1")" -artifact_base="${1%"-$COMMIT$extension"}" +artifact_base="${1%"$extension"}" cname="$(basename "$artifact_base")" [ "$extension" != ".raw" ] || extension= @@ -18,15 +18,15 @@ input= if [ -f "image$extension" ]; then script="image$extension" - input="$artifact_base-$COMMIT.tar" + input="$artifact_base.tar" fi if [ -f "convert$extension" ]; then script="convert$extension" - input="$artifact_base-$COMMIT.raw" + input="$artifact_base.raw" fi -IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features) +IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --cname "$cname" features) for feature in "${features[@]}"; do if [ -s "features/$feature/image$extension" ]; then @@ -36,7 +36,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="features/$feature/image$extension" - input="$artifact_base-$COMMIT.tar" + input="$artifact_base.tar" fi if [ -s "features/$feature/convert$extension" ]; then @@ -46,7 +46,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="features/$feature/convert$extension" - input="$artifact_base-$COMMIT.raw" + input="$artifact_base.raw" fi # temporarily enable file globbing (+f) @@ -68,7 +68,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="$i" - input="$artifact_base-$COMMIT.${i##*~}" + input="$artifact_base.${i##*~}" done done diff --git a/builder/make_list_build_artifacts b/builder/make_list_build_artifacts index 02b3bd1..89cb0de 100755 --- a/builder/make_list_build_artifacts +++ b/builder/make_list_build_artifacts @@ -5,21 +5,20 @@ shopt -s nullglob cname="$1" -IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features) - -artifacts=(".build/$cname-$COMMIT.tar" ".build/$cname-$COMMIT.release" ".build/$cname-$COMMIT.manifest" ".build/$cname-$COMMIT.requirements") +IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" features) +artifacts=(".build/$cname.tar" ".build/$cname.release" ".build/$cname.manifest" ".build/$cname.requirements") for feature in "${features[@]}"; do for i in "features/$feature/"{image,convert}.*; do # get target artifact file extension, usually this is the image/convert script extension # except if the script extension is of the form filename.extA~extB in which case the artifact extension is .extA only extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_~]*)*$' <<< "$i")" - artifacts+=(".build/$cname-$COMMIT${extension%~*}") + artifacts+=(".build/$cname${extension%~*}") done done -if [ "${#artifacts[@]}" = 4 ] && [ -n "$(./parse_features --allow-frankenstein --feature-dir "features" --cname "$cname" platforms)" ]; then - artifacts+=(".build/$cname-$COMMIT.raw") +if [ "${#artifacts[@]}" = 4 ] && [ -n "$(gl-features-parse --feature-dir "features" --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" platforms)" ]; then + artifacts+=(".build/$cname.raw") fi echo "${artifacts[@]}" diff --git a/builder/requirements.txt b/builder/requirements.txt new file mode 100644 index 0000000..5569fe7 --- /dev/null +++ b/builder/requirements.txt @@ -0,0 +1,3 @@ +# Basic Python requirements for Garden Linux + +gardenlinux @ git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.11.0 diff --git a/pkg.list b/pkg.list index 288c8a7..eafaaab 100644 --- a/pkg.list +++ b/pkg.list @@ -23,10 +23,8 @@ openssl ostree ostree-boot python3 -python3-mako -python3-networkx -python3-pefile -python3-yaml +python3-setuptools +python3-pip qemu-utils squashfs-tools systemd diff --git a/setup_namespace b/setup_namespace index 1085abf..06bb0aa 100755 --- a/setup_namespace +++ b/setup_namespace @@ -4,6 +4,7 @@ set -eufo pipefail if [ "${1-}" = --second-stage ]; then shift + mount -t tmpfs -o size=4G tmpfs /tmp cleanup_permissions () { if [ -d /builder/.build ]; then