fix(linux): include missing artifacts in source tarball 🗜️

linux/scripts/dist.sh

@@ -39,8 +39,9 @@ dch keyman --newversion "${KEYMAN_VERSION}" --force-bad-version --nomultimaint
 
 # Create the tarball
 
-# files and folders to include in the tarball
-# shellcheck disable=2034  # to_exclude appears to be unused
+# We always include these files which are the minimum files and
+# folder required for Ubuntu/Debian packaging
+# shellcheck disable=2034  # to_include appears to be unused
 to_include=(
   common/build.sh \
   common/cpp \
@@ -60,18 +61,61 @@ to_include=(
 
 # files and subfolders to exclude from paths included in 'to_include',
 # i.e. the exceptions to 'to_include'.
+
 # shellcheck disable=2034  # to_exclude appears to be unused
 to_exclude=(
+  build \
   common/test/keyboards/baseline/kmcomp-*.zip \
-  core/build \
-  linux/build \
   linux/builddebs \
   linux/docs/help \
   linux/keyman-config/keyman_config/version.py \
   linux/keyman-config/buildtools/build-langtags.py \
-  linux/keyman-system-service/build
 )
 
+if [[ -z "${create_origdist+x}" ]]; then
+  # If we build a full source tarball we include additional files
+  # so that it's possible to run `${KEYMAN_ROOT}/build.sh` on Linux
+
+  # shellcheck disable=2034  # to_include appears to be unused
+  to_include+=(
+    common/tools/hextobin \
+    common/web/keyman-version \
+    common/web/langtags \
+    common/web/types \
+    common/windows/cpp \
+    common/windows/include \
+    developer/src/common/include \
+    developer/src/common/web \
+    developer/src/ext/json \
+    developer/src/kmc \
+    developer/src/kmc-analyze \
+    developer/src/kmc-copy \
+    developer/src/kmc-generate \
+    developer/src/kmc-keyboard-info \
+    developer/src/kmc-kmn \
+    developer/src/kmc-ldml \
+    developer/src/kmc-model \
+    developer/src/kmc-model-info \
+    developer/src/kmc-package \
+    developer/src/kmcmplib \
+    docs/minimum-versions.md.in
+    resources/build \
+    resources/standards-data \
+  )
+
+  # additional files and subfolders to exclude from paths included in 'to_include',
+  # i.e. the exceptions to 'to_include'.
+  # shellcheck disable=2034  # to_exclude appears to be unused
+  to_exclude+=(
+    *.exe \
+    resources/build/history \
+    resources/build/l10n \
+    resources/build/mac \
+    resources/build/win \
+    resources/build/*.lua \
+  )
+fi
+
 # array to store list of --tar-ignore parameters generated from to_include and to_exclude.
 ignored_files=()
 
@@ -109,7 +153,7 @@ cd "${BASEDIR}"
 
 # create orig.tar.gz
 if [[ ! -z "${create_origdist+x}" ]]; then
-    cd dist
+    cd "${KEYMAN_ROOT}/linux/dist"
     pkgvers="keyman-${KEYMAN_VERSION}"
     tar xfz keyman-"${KEYMAN_VERSION}".tar.gz
     mv -v keyman "${pkgvers}" 2>/dev/null || mv -v "$(find . -mindepth 1 -maxdepth 1 -type d)" "${pkgvers}"

linux/scripts/package-build.inc.sh

@@ -91,78 +91,135 @@ function generate_tar_ignore_list() {
   local list_var="$4"
   local prefix="$5"
   local includes_array="${includes_var}[@]"
+  # shellcheck disable=SC2034
   local includes=("${!includes_array}")
   local excludes_array="${excludes_var}[@]"
   local excludes=("${!excludes_array}")
-  local dir all_dirs found_match inc
-
-  mapfile -t all_dirs < <(find "${directory}" -mindepth 1 -maxdepth 1 -type d | sort)
-  for dir in "${all_dirs[@]}"; do
-    found_match=false
-    for inc in "${includes[@]}"; do
-      if [[ "./${inc}" =~ ^${dir} ]]; then
-        found_match=true
-        if [[ "./${inc}" != "${dir}" ]]; then
-          # check subdirectories
-          generate_tar_ignore_list "${dir}" "${includes_var}" "${excludes_var}" "${list_var}" "${prefix}"
-        fi
-        # check if files/subdir in $dir are in excludes list
-        _generate_excludes_for_dir "${dir}" "${includes_var}" "${excludes_var}" "${list_var}"
-        break
-      fi
-    done
-    if ! ${found_match}; then
-      _add_to_list "${list_var}" "${dir}"
+
+  # Loop through excludes and put all without path in single_excludes
+  local single_excludes=()
+  for item in "${excludes[@]}"; do
+    if [[ ${item} != */* ]]; then
+        single_excludes+=("${item}")
     fi
   done
+
+  local ignore_list=()
+  _process_directory "${directory}" false
+
+  for item in "${ignore_list[@]}"; do
+    eval "${list_var}+=(\"--tar-ignore=${item}\")"
+  done
 }
 
-function _generate_excludes_for_dir() {
+function _process_directory() {
   local directory="$1"
-  local includes_var="$2"
-  local excludes_var="$3"
-  local list_var="$4"
-  local includes_array="${includes_var}[@]"
-  local includes=("${!includes_array}")
-  local excludes_array="${excludes_var}[@]"
-  local excludes=("${!excludes_array}")
-  local file all_files excluded included is_match
-
-  mapfile -t all_files < <(find "${directory}" -mindepth 1 -maxdepth 1 | sort)
-  is_match=false
-  for file in "${all_files[@]}"; do
-    for included in "${includes[@]}"; do
-      if [[ "${file}" == ./${included} ]]; then
-        is_match=true
-        break
+  local isParentIncluded="$2"
+  local all_items item
+
+  mapfile -t all_items < <(find "${directory}" -mindepth 1 -maxdepth 1 | sort)
+  for item in "${all_items[@]}"; do
+    debug "Checking item: ${item}"
+    if _is_exact_match "includes" "${item}"; then
+      debug "  Including (full match): ${item}"
+      if [[ -d "${item}" ]]; then
+        _process_directory "${item}" true
       fi
-    done
-    if ${is_match} ; then
-      if [[ "${file}" != ./${included} ]] && [[ -f "${file}" ]]; then
-      _add_to_list "${list_var}" "${file}"
+    elif _starts_with "includes" "${item}"; then
+      debug "  Including (partial match): ${item}"
+      if [[ -d "${item}" ]]; then
+        _process_directory "${item}" "${isParentIncluded}"
       fi
+    elif _is_exact_match "excludes" "${item}"; then
+      debug "  Excluding (full match): ${item}"
+      _add_to_list "${item}"
+    elif _ends_with "single_excludes" "${item}" || _is_wildcard_match "single_excludes" "${item}"; then
+      debug "  Excluding (single exclude): ${item}"
+      _add_to_list "${item}"
+    elif [[ "${isParentIncluded}" == "false" ]]; then
+      debug "  Excluding (not included): ${item}"
+      _add_to_list "${item}"
     else
-      for excluded in "${excludes[@]}"; do
-        if [[ "${file}" == ./${excluded} ]]; then
-          _add_to_list "${list_var}" "${file}"
-          break
-        elif [[ "./${excluded}" =~ ^${file} ]]; then
-          # check subdirectories
-          _generate_excludes_for_dir "${file}" "${includes_var}" "${excludes_var}" "${list_var}"
-          break
-        fi
-      done
+      debug "  Including (parent included): ${item}"
+      if [[ -d "${item}" ]]; then
+        _process_directory "${item}" "${isParentIncluded}"
+      fi
     fi
   done
 }
 
 function _add_to_list() {
-  local list_var="$1"
-  local filename="$2"
-
-  # Note: the files end up in subdirectories under `keyman` (or rather
-  # the directory name of $KEYMAN_ROOT), so we can
-  # include that when matching files and directories to ignore.
-  # shellcheck disable=SC2154
-  eval "${list_var}+=(\"--tar-ignore=${prefix}/${filename#./}\")"
+  local item="$1"
+  ignore_list+=("${prefix}/${item#./}")
+}
+
+# Returns true if one of the values in the $1 array equals ${file} ($2)
+# Example: will return true for array=(path1/path2) file=./path1/path2
+function _is_exact_match() {
+  local array_var="$1"
+  local file="$2"
+  local array_name="${array_var}[@]"
+  local haystack=("${!array_name}")
+  local item
+  for item in "${haystack[@]}"; do
+    if [[ "./${item#./}" == "${file}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Returns true if one of the values in the $1 array starts with ${file} ($2)
+# Example: will return true for array=(path1/path2) file=./path1
+function _starts_with() {
+  local array_var="$1"
+  local file="$2"
+  local array_name="${array_var}[@]"
+  local array_values=("${!array_name}")
+  local array_item
+  for array_item in "${array_values[@]}"; do
+    if [[ "./${array_item#./}" == ${file}* ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Returns true if ${file} ($2) ends with one of the values in the $1 array
+# Example: will return true for array=(path2) file=./path1/path2
+function _ends_with() {
+  local array_var="$1"
+  local file="$2"
+  local array_name="${array_var}[@]"
+  local array_values=("${!array_name}")
+  local array_item
+  for array_item in "${array_values[@]}"; do
+    if [[ "${file}" == */${array_item#./} ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Returns true if ${file} ($2) matches one of the values of the $1 array.
+# These values may contain wildcards.
+# Example: will return true for array=(*.sh) file=./path1/build.sh
+function _is_wildcard_match() {
+  local array_var="$1"
+  local file="$2"
+  local array_name="${array_var}[@]"
+  local array_values=("${!array_name}")
+  local array_item
+  for array_item in "${array_values[@]}"; do
+    array_item=${array_item/./\\.}
+    if [[ "${file}" =~ /${array_item/\*/.\*}$ ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function debug() {
+  # echo "$@"
+  return 0
 }