Unify python-gardenlinux-lib based feature and cname parsing

Dockerfile

@@ -26,6 +26,7 @@ COPY --from=datefudge /usr/bin/datefudge /usr/bin/datefudge
 COPY --from=resizefat32 /usr/bin/resizefat32 /usr/bin/resizefat32
 RUN curl "https://github.com/gardenlinux/aws-kms-pkcs11/releases/download/latest/aws_kms_pkcs11-$(dpkg --print-architecture).so" -sLo "/usr/lib/$(uname -m)-linux-gnu/pkcs11/aws_kms_pkcs11.so"
 COPY builder /builder
+RUN python3 -m pip install --break-system-packages -r "/builder/requirements.txt" --root-user-action ignore
 RUN mkdir /builder/cert
 COPY setup_namespace /usr/sbin/setup_namespace
 RUN curl -sSLf https://github.com/gardenlinux/seccomp_fake_xattr/releases/download/latest/seccomp_fake_xattr-$(uname -m).tar.gz \

build

@@ -97,12 +97,9 @@ commit="$(./get_commit)"
 timestamp="$(./get_timestamp)"
 default_version="$(./get_version)"
 
-
 if [ "$resolve_cname" = 1 ]; then
 	arch="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" dpkg --print-architecture)"
-	cname="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" /builder/parse_features --feature-dir /builder/features --default-arch "$arch" --default-version "$default_version" --cname "$1")"
-	short_commit="$(head -c 8 <<< "$commit")"
-	echo "$cname-$short_commit" >&3
+	"$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" gl-cname --feature-dir /builder/features --arch "$arch" --version "${default_version}-${commit}" "$1"
 	exit 0
 fi
 

builder/Makefile

@@ -10,10 +10,11 @@ export BASH_ENV := make_bash_env
 
 MAKEFLAGS += --no-builtin-rules
 
-lastword = $(word $(words $1),$1)
-prelastword = $(word $(words $1),_ $1)
-cname_version = $(call lastword,$(subst -, ,$1))
-cname_arch = $(call prelastword,$(subst -, ,$1))
+cname_parts = $(subst _,  , $(subst -,  , $1))
+cname_gl_commit = $(lastword $(call cname_parts,$1))
+gl_version = $(lastword $(filter-out $(call cname_gl_commit,$1), $(call cname_parts,$1)))
+cname_gl_version = $(call gl_version,$1)-$(call cname_gl_commit,$1)
+cname_arch = $(lastword $(filter-out $(call gl_version,$1) $(call cname_gl_commit,$1), $(call cname_parts,$1)))
 
 define require_var =
 ifndef $1
@@ -24,11 +25,13 @@ endef
 required_vars := REPO COMMIT TIMESTAMP DEFAULT_VERSION TEMPFS_SIZE
 $(foreach var,$(required_vars),$(eval $(call require_var,$(var))))
 
-PARSE_FEATURES_ARGS :=
+GL_ALLOW_FRANKENSTEIN :=
 ifdef ALLOW_FRANKENSTEIN
-PARSE_FEATURES_ARGS := --allow-frankenstein
+GL_ALLOW_FRANKENSTEIN := true
 endif
 
+export GL_ALLOW_FRANKENSTEIN
+
 SHORT_COMMIT := $(shell head -c 8 <<< '$(COMMIT)')
 
 DEFAULT_ARCH := $(shell dpkg --print-architecture)
@@ -43,22 +46,22 @@ clean:
 .build/%.sentinel:
 	true
 
-.build/bootstrap-%-$(SHORT_COMMIT).tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call cname_version,$$*))
+.build/bootstrap-%.tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call gl_version,$$*))
 	target '$@'
-	info 'bootstrapping $*-$(SHORT_COMMIT)'
+	info 'bootstrapping $*'
 	arch='$(call cname_arch,$*)'
-	version='$(call cname_version,$*)'
+	version='$(call gl_version,$*)'
 	./bootstrap "$$arch" "$$version" '$(REPO)' keyring.gpg '$@'
 
-.build/%-$(SHORT_COMMIT).tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_version,$$*)-$(SHORT_COMMIT).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert)
+.build/%.tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_gl_version,$$*).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert)
 	target '$@' '$<'
 	info 'configuring rootfs $*-$(SHORT_COMMIT)'
-	features="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' features)"
-	features_platforms="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' platforms)"
-	features_elements="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' elements)"
-	features_flags="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' flags)"
+	features="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*' features)"
+	features_platforms="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'platforms)"
+	features_elements="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'elements)"
+	features_flags="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'flags)"
 	BUILDER_CNAME='$*'
-	BUILDER_VERSION='$(call cname_version,$*)'
+	BUILDER_VERSION='$(call gl_version,$*)'
 	BUILDER_ARCH='$(call cname_arch,$*)'
 	BUILDER_TIMESTAMP='$(TIMESTAMP)'
 	BUILDER_COMMIT='$(COMMIT)'
@@ -70,17 +73,17 @@ clean:
 	./configure '$(word 1,$^)' '$@'
 
 define artifact_template =
-.build/%-$(SHORT_COMMIT).$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert)
+.build/%.$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert)
 	script='$$(word 1,$$^)'
 	input='$$(word 2,$$^)'
 	target '$$@' "$$$$input"
 	info 'building $1 image $$*'
-	features="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' features)"
-	features_platforms="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' platforms)"
-	features_elements="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' elements)"
-	features_flags="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' flags)"
+	features="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' features)"
+	features_platforms="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' platforms)"
+	features_elements="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' elements)"
+	features_flags="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' flags)"
 	BUILDER_CNAME='$$*'
-	BUILDER_VERSION='$$(call cname_version,$$*)'
+	BUILDER_VERSION='$$(call gl_version,$$*)'
 	BUILDER_ARCH='$$(call cname_arch,$$*)'
 	BUILDER_TIMESTAMP='$$(TIMESTAMP)'
 	BUILDER_COMMIT='$$(COMMIT)'
@@ -94,15 +97,15 @@ endef
 
 $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifact_template,$(artifact_rule))))
 
-.build/%-$(SHORT_COMMIT).artifacts: $$(shell COMMIT=$(SHORT_COMMIT) ./make_list_build_artifacts '$$*')
+.build/%.artifacts: $$(shell COMMIT=$(SHORT_COMMIT) DEFAULT_VERSION=$(DEFAULT_VERSION) NATIVE_ARCH=$(NATIVE_ARCH) ./make_list_build_artifacts '$$*')
 	target '$@'
 	echo -n > '$@'
 	for f in $^; do
 		basename "$$f" | tee -a '$@'
 		echo "$$(basename "$$f").log" | tee -a '$@'
 	done
 
-%: .build/$$(shell ./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)' --cname '$$*')-$(SHORT_COMMIT).artifacts
+%: .build/$$(shell gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)-$$(SHORT_COMMIT)' --cname '$$*').artifacts
 	ln -f -s -r '$<' '.build/$*'
 
 # prevents match anything rule from applying to Makefile and image/convert scripts

builder/make_get_image_dependencies

@@ -7,7 +7,7 @@ exec 1>&2
 
 # get longest chain of extensions, but not extensions starting with a number to prevent parsing minor version as extension
 extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_]*)*$' <<< "$1")"
-artifact_base="${1%"-$COMMIT$extension"}"
+artifact_base="${1%"$extension"}"
 cname="$(basename "$artifact_base")"
 
 [ "$extension" != ".raw" ] || extension=
@@ -18,15 +18,15 @@ input=
 
 if [ -f "image$extension" ]; then
 	script="image$extension"
-	input="$artifact_base-$COMMIT.tar"
+	input="$artifact_base.tar"
 fi
 
 if [ -f "convert$extension" ]; then
 	script="convert$extension"
-	input="$artifact_base-$COMMIT.raw"
+	input="$artifact_base.raw"
 fi
 
-IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features)
+IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --cname "$cname" features)
 
 for feature in "${features[@]}"; do
 	if [ -s "features/$feature/image$extension" ]; then
@@ -36,7 +36,7 @@ for feature in "${features[@]}"; do
 		fi
 		is_feature_script=1
 		script="features/$feature/image$extension"
-		input="$artifact_base-$COMMIT.tar"
+		input="$artifact_base.tar"
 	fi
 
 	if [ -s "features/$feature/convert$extension" ]; then
@@ -46,7 +46,7 @@ for feature in "${features[@]}"; do
 		fi
 		is_feature_script=1
 		script="features/$feature/convert$extension"
-		input="$artifact_base-$COMMIT.raw"
+		input="$artifact_base.raw"
 	fi
 
 	# temporarily enable file globbing (+f)
@@ -68,7 +68,7 @@ for feature in "${features[@]}"; do
 		fi
 		is_feature_script=1
 		script="$i"
-		input="$artifact_base-$COMMIT.${i##*~}"
+		input="$artifact_base.${i##*~}"
 	done
 done
 

builder/make_list_build_artifacts

@@ -5,21 +5,20 @@ shopt -s nullglob
 
 cname="$1"
 
-IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features)
-
-artifacts=(".build/$cname-$COMMIT.tar" ".build/$cname-$COMMIT.release" ".build/$cname-$COMMIT.manifest" ".build/$cname-$COMMIT.requirements")
+IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" features)
+artifacts=(".build/$cname.tar" ".build/$cname.release" ".build/$cname.manifest" ".build/$cname.requirements")
 
 for feature in "${features[@]}"; do
 	for i in "features/$feature/"{image,convert}.*; do
 		# get target artifact file extension, usually this is the image/convert script extension
 		# except if the script extension is of the form filename.extA~extB in which case the artifact extension is .extA only
 		extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_~]*)*$' <<< "$i")"
-		artifacts+=(".build/$cname-$COMMIT${extension%~*}")
+		artifacts+=(".build/$cname${extension%~*}")
 	done
 done
 
-if [ "${#artifacts[@]}" = 4 ] && [ -n "$(./parse_features --allow-frankenstein --feature-dir "features" --cname "$cname" platforms)" ]; then
-	artifacts+=(".build/$cname-$COMMIT.raw")
+if [ "${#artifacts[@]}" = 4 ] && [ -n "$(gl-features-parse --feature-dir "features" --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" platforms)" ]; then
+	artifacts+=(".build/$cname.raw")
 fi
 
 echo "${artifacts[@]}"

builder/requirements.txt

@@ -0,0 +1,3 @@
+# Basic Python requirements for Garden Linux
+
+gardenlinux @ git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.11.0

pkg.list

@@ -23,10 +23,8 @@ openssl
 ostree
 ostree-boot
 python3
-python3-mako
-python3-networkx
-python3-pefile
-python3-yaml
+python3-setuptools
+python3-pip
 qemu-utils
 squashfs-tools
 systemd

setup_namespace

@@ -4,6 +4,7 @@ set -eufo pipefail
 
 if [ "${1-}" = --second-stage ]; then
 	shift
+
 	mount -t tmpfs -o size=4G tmpfs /tmp
 	cleanup_permissions () {
 		if [ -d /builder/.build ]; then