Group tests by chart for clarity, rename outputs for when searching templates

Author: beekhof

Makefile

@@ -29,10 +29,7 @@ show: ## show the starting template without installing it
 
 CHARTS=$(shell find . -type f -iname 'Chart.yaml' -exec dirname "{}"  \; | sed -e 's/.\///')
 test: ## run helm tests
-# Test that all values used by the chart are in values.yaml with the same defaults as the pattern
-	@for t in $(CHARTS); do common/scripts/test.sh $$t naked ""; if [ $$? != 0 ]; then exit 1; fi; done
-# Test the charts as the pattern would drive them
-	@for t in $(CHARTS); do common/scripts/test.sh $$t normal "$(TEST_OPTS) $(PATTERN_OPTS)"; if [ $$? != 0 ]; then exit 1; fi; done
+	@for t in $(CHARTS); do common/scripts/test.sh $$t all "$(TEST_OPTS) $(PATTERN_OPTS)"; if [ $$? != 0 ]; then exit 1; fi; done
 
 helmlint: ## run helm lint
 	@for t in $(CHARTS); do helm lint $(TEST_OPTS) $(PATTERN_OPTS) $$t; if [ $$? != 0 ]; then exit 1; fi; done

scripts/test.sh

@@ -7,48 +7,93 @@ unset KUBECONFIG
 
 target=$1
 name=$(echo $1 | sed -e s@/@-@g -e s@charts-@@)
-TEST_VARIANT="$2"
-CHART_OPTS="$3"
-
-TESTDIR=tests
-REFERENCE=${TESTDIR}/${name}-${TEST_VARIANT}.expected.yaml
-OUTPUT=${TESTDIR}/.${name}-${TEST_VARIANT}.expected.yaml
-#REFERENCE=${TESTDIR}/${name}.expected.yaml
-#OUTPUT=${TESTDIR}/.${name}.expected.yaml
-
-echo "Testing $1 chart (${TEST_VARIANT})" >&2
-helm template --kubeconfig /tmp/doesnotexistever $target --name-template $name ${CHART_OPTS} > ${OUTPUT}
-rc=$?
-if [ $rc -ne 0 ]; then
-    echo "FAIL on helm template $target --name-template $name ${CHART_OPTS}"
-    exit 1
-fi
-if [ ! -e ${REFERENCE} ]; then
-    cp ${OUTPUT} ${REFERENCE}
-fi
-diff -u ${REFERENCE}  ${OUTPUT}
-rc=$?
 
-if [ $TEST_VARIANT = normal -a $rc = 0 ]; then
+function doTest() {
+    TEST_VARIANT=$1
+    CHART_OPTS="$2"
+    TESTDIR=tests
+    TEST=${name}-${TEST_VARIANT}
+    FILENAME=${TEST}.expected.yml
+    OUTPUT=${TESTDIR}/.${FILENAME}
+    REFERENCE=${TESTDIR}/${FILENAME}
+
+    echo "Testing $name chart (${TEST_VARIANT})" >&2
+    helm template --kubeconfig /tmp/doesnotexistever $target --name-template $name ${CHART_OPTS} > ${OUTPUT}
+    rc=$?
+    if [ $rc -ne 0 ]; then
+	echo "FAIL on helm template $target --name-template $name ${CHART_OPTS}"
+	exit 1
+    fi
+    if [ ! -e ${REFERENCE} ]; then
+	cp ${OUTPUT} ${REFERENCE}
+	echo -e "\n\n#### Created test output\007\n#### Now add ${REFERENCE} to Git\n\n\007"
+	exit 2
+    fi
+    diff -u ${REFERENCE} ${OUTPUT}
+    rc=$?
+    if [ $rc = 0 ]; then
+	rm -f ${TESTDIR}/.${name}.*
+	echo -e "PASS on $name $TEST_VARIANT with opts [$CHART_OPTS]\n"
+    else
+	echo -e "FAIL on $name $TEST_VARIANT with opts [$CHART_OPTS]\n"
+	exit $rc
+    fi
+}
+
+function doTestCompare() {
+    TEST_VARIANT="differences"
+    TESTDIR=tests
+    TEST=${name}
+    FILENAME=${TEST}.expected.yml
+    OUTPUT=${TESTDIR}/.${FILENAME}
+    REFERENCE=${TESTDIR}/${FILENAME}
+
+    echo "Testing $name chart (${TEST_VARIANT})" >&2
     # Another method of finding variables missing from values.yaml, eg.
     # -    name: -datacenter
     # +    name: pattern-name-datacenter
-    diff -u ${TESTDIR}/${name}-naked.expected.yaml ${TESTDIR}/${name}-normal.expected.yaml | sed 's/20[0-9][0-9]-[09][0-9].*//' > ${OUTPUT}.variant
 
-    if [ ! -e ${REFERENCE}.variant ]; then
-	cp ${OUTPUT}.variant ${REFERENCE}.variant
+    TEST=${name}
+    FILENAME=${TEST}.expected.diff
+    OUTPUT=${TESTDIR}/.${FILENAME}
+    REFERENCE=${TESTDIR}/${FILENAME}
+
+    # Drop the date from the diff output, it will not be stable
+    diff -u ${TESTDIR}/${name}-naked.expected.yml ${TESTDIR}/${name}-normal.expected.yml | sed 's/\.yml.*20[0-9][0-9].*/.yml/g' > ${OUTPUT}
+
+    if [ ! -e ${REFERENCE} ]; then	
+	cp ${OUTPUT} ${REFERENCE}
+	echo -e "\n\n#### Created test output\007\n#### Now add ${REFERENCE} to Git\n\n\007"
+	exit 2
     fi
 
-    diff -u ${REFERENCE}.variant  ${OUTPUT}.variant
+    diff -u ${REFERENCE} ${OUTPUT}
     rc=$?
-fi
 
-if [ $rc = 0 ]; then
-    rm -f ${OUTPUT}
-    rm -f ${OUTPUT}.variant
-    echo "PASS on $target $TEST_VARIANT with opts [$CHART_OPTS]"
+    if [ $rc = 0 ]; then
+	rm -f ${TESTDIR}/.${name}.*
+	echo -e "PASS on $name $TEST_VARIANT with opts [$CHART_OPTS]\n"
+    else
+	echo -e "FAIL on $name $TEST_VARIANT with opts [$CHART_OPTS]\n"
+	exit $rc
+    fi
+}
+
+if [ $2 = "all" ]; then
+    echo -e "\n#####################"
+    echo -e "### ${name}"
+    echo -e "#####################\n"
+
+    # Test that all values used by the chart are in values.yaml with the same defaults as the pattern
+    doTest naked
+
+    # Test the charts as the pattern would drive them
+    doTest normal "$3"
+
+    # Ensure the differences between the two results are also stable
+    doTestCompare
 else
-    echo "FAIL on $target $TEST_VARIANT with opts [$CHART_OPTS]"
+    doTest $2 "$3"
 fi
 
-exit $rc
+exit 0

tests/acm-naked.expected.yaml tests/acm-naked.expected.ymltests/acm-naked.expected.yaml renamed to tests/acm-naked.expected.yml

@@ -1,5 +1,5 @@
---- tests/acm-naked.expected.yaml	
-+++ tests/acm-normal.expected.yaml	2022-10-03 11:27:20.418172850 +1100
+--- tests/acm-naked.expected.yml
++++ tests/acm-normal.expected.yml
 @@ -1,6 +1,386 @@
  ---
 -# Source: acm/templates/policies/application-policies.yaml

tests/acm-normal.expected.yaml tests/acm-normal.expected.ymltests/acm-normal.expected.yaml renamed to tests/acm-normal.expected.yml

@@ -1,5 +1,5 @@
---- tests/clustergroup-naked.expected.yaml	2022-10-03 11:27:20.418172850 +1100
-+++ tests/clustergroup-normal.expected.yaml	2022-10-03 11:27:20.418172850 +1100
+--- tests/clustergroup-naked.expected.yml
++++ tests/clustergroup-normal.expected.yml
 @@ -1,17 +1,243 @@
  ---
 +# Source: pattern-clustergroup/templates/core/namespaces.yaml