Skip to content

Commit e776a2c

Browse files
ermshipereteermshiperete
authored
Merge pull request #15566 from keymanapp/fix/linux/15544_sourcepkg
fix(linux): include missing artifacts in source tarball 🗜️ With this change it's possible to build Keyman from the source tarball by running `./build.sh` from the Keyman root directory. When building Keyman as part of the Ubuntu/Debian packaging, we don't use the toplevel `build.sh` file but instead call the build scripts in the subdirectories. This omits building some components that are currently not needed for running Keyman for Linux. The `dist.sh` script only includes the necessary files in this case, distinguished by the `origdist` parameter. This results in a significantly smaller tarball (2.5MB vs 51MB). This change also enhances the `generate_tar_ignore_list` function so that it is now possible to add `build` to the exclude list to ignore all `build` subdirectories. Fixes: #15544
2 parents bdeede1 + 9322a8a commit e776a2c

5 files changed

Lines changed: 689 additions & 77 deletions

File tree

‎linux/scripts/dist.sh‎

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@ dch keyman --newversion "${KEYMAN_VERSION}" --force-bad-version --nomultimaint
3939

4040
# Create the tarball
4141

42-
# files and folders to include in the tarball
43-
# shellcheck disable=2034 # to_exclude appears to be unused
42+
# We always include these files which are the minimum files and
43+
# folder required for Ubuntu/Debian packaging
44+
# shellcheck disable=2034 # to_include appears to be unused
4445
to_include=(
4546
common/build.sh \
4647
common/cpp \
@@ -60,18 +61,61 @@ to_include=(
6061

6162
# files and subfolders to exclude from paths included in 'to_include',
6263
# i.e. the exceptions to 'to_include'.
64+
6365
# shellcheck disable=2034 # to_exclude appears to be unused
6466
to_exclude=(
67+
build \
6568
common/test/keyboards/baseline/kmcomp-*.zip \
66-
core/build \
67-
linux/build \
6869
linux/builddebs \
6970
linux/docs/help \
7071
linux/keyman-config/keyman_config/version.py \
7172
linux/keyman-config/buildtools/build-langtags.py \
72-
linux/keyman-system-service/build
7373
)
7474

75+
if [[ -z "${create_origdist+x}" ]]; then
76+
# If we build a full source tarball we include additional files
77+
# so that it's possible to run `${KEYMAN_ROOT}/build.sh` on Linux
78+
79+
# shellcheck disable=2034 # to_include appears to be unused
80+
to_include+=(
81+
common/tools/hextobin \
82+
common/web/keyman-version \
83+
common/web/langtags \
84+
common/web/types \
85+
common/windows/cpp \
86+
common/windows/include \
87+
developer/src/common/include \
88+
developer/src/common/web \
89+
developer/src/ext/json \
90+
developer/src/kmc \
91+
developer/src/kmc-analyze \
92+
developer/src/kmc-copy \
93+
developer/src/kmc-generate \
94+
developer/src/kmc-keyboard-info \
95+
developer/src/kmc-kmn \
96+
developer/src/kmc-ldml \
97+
developer/src/kmc-model \
98+
developer/src/kmc-model-info \
99+
developer/src/kmc-package \
100+
developer/src/kmcmplib \
101+
docs/minimum-versions.md.in
102+
resources/build \
103+
resources/standards-data \
104+
)
105+
106+
# additional files and subfolders to exclude from paths included in 'to_include',
107+
# i.e. the exceptions to 'to_include'.
108+
# shellcheck disable=2034 # to_exclude appears to be unused
109+
to_exclude+=(
110+
*.exe \
111+
resources/build/history \
112+
resources/build/l10n \
113+
resources/build/mac \
114+
resources/build/win \
115+
resources/build/*.lua \
116+
)
117+
fi
118+
75119
# array to store list of --tar-ignore parameters generated from to_include and to_exclude.
76120
ignored_files=()
77121

@@ -109,7 +153,7 @@ cd "${BASEDIR}"
109153

110154
# create orig.tar.gz
111155
if [[ ! -z "${create_origdist+x}" ]]; then
112-
cd dist
156+
cd "${KEYMAN_ROOT}/linux/dist"
113157
pkgvers="keyman-${KEYMAN_VERSION}"
114158
tar xfz keyman-"${KEYMAN_VERSION}".tar.gz
115159
mv -v keyman "${pkgvers}" 2>/dev/null || mv -v "$(find . -mindepth 1 -maxdepth 1 -type d)" "${pkgvers}"

‎linux/scripts/package-build.inc.sh‎

Lines changed: 115 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -91,78 +91,135 @@ function generate_tar_ignore_list() {
9191
local list_var="$4"
9292
local prefix="$5"
9393
local includes_array="${includes_var}[@]"
94+
# shellcheck disable=SC2034
9495
local includes=("${!includes_array}")
9596
local excludes_array="${excludes_var}[@]"
9697
local excludes=("${!excludes_array}")
97-
local dir all_dirs found_match inc
98-
99-
mapfile -t all_dirs < <(find "${directory}" -mindepth 1 -maxdepth 1 -type d | sort)
100-
for dir in "${all_dirs[@]}"; do
101-
found_match=false
102-
for inc in "${includes[@]}"; do
103-
if [[ "./${inc}" =~ ^${dir} ]]; then
104-
found_match=true
105-
if [[ "./${inc}" != "${dir}" ]]; then
106-
# check subdirectories
107-
generate_tar_ignore_list "${dir}" "${includes_var}" "${excludes_var}" "${list_var}" "${prefix}"
108-
fi
109-
# check if files/subdir in $dir are in excludes list
110-
_generate_excludes_for_dir "${dir}" "${includes_var}" "${excludes_var}" "${list_var}"
111-
break
112-
fi
113-
done
114-
if ! ${found_match}; then
115-
_add_to_list "${list_var}" "${dir}"
98+
99+
# Loop through excludes and put all without path in single_excludes
100+
local single_excludes=()
101+
for item in "${excludes[@]}"; do
102+
if [[ ${item} != */* ]]; then
103+
single_excludes+=("${item}")
116104
fi
117105
done
106+
107+
local ignore_list=()
108+
_process_directory "${directory}" false
109+
110+
for item in "${ignore_list[@]}"; do
111+
eval "${list_var}+=(\"--tar-ignore=${item}\")"
112+
done
118113
}
119114

120-
function _generate_excludes_for_dir() {
115+
function _process_directory() {
121116
local directory="$1"
122-
local includes_var="$2"
123-
local excludes_var="$3"
124-
local list_var="$4"
125-
local includes_array="${includes_var}[@]"
126-
local includes=("${!includes_array}")
127-
local excludes_array="${excludes_var}[@]"
128-
local excludes=("${!excludes_array}")
129-
local file all_files excluded included is_match
130-
131-
mapfile -t all_files < <(find "${directory}" -mindepth 1 -maxdepth 1 | sort)
132-
is_match=false
133-
for file in "${all_files[@]}"; do
134-
for included in "${includes[@]}"; do
135-
if [[ "${file}" == ./${included} ]]; then
136-
is_match=true
137-
break
117+
local isParentIncluded="$2"
118+
local all_items item
119+
120+
mapfile -t all_items < <(find "${directory}" -mindepth 1 -maxdepth 1 | sort)
121+
for item in "${all_items[@]}"; do
122+
debug "Checking item: ${item}"
123+
if _is_exact_match "includes" "${item}"; then
124+
debug " Including (full match): ${item}"
125+
if [[ -d "${item}" ]]; then
126+
_process_directory "${item}" true
138127
fi
139-
done
140-
if ${is_match} ; then
141-
if [[ "${file}" != ./${included} ]] && [[ -f "${file}" ]]; then
142-
_add_to_list "${list_var}" "${file}"
128+
elif _starts_with "includes" "${item}"; then
129+
debug " Including (partial match): ${item}"
130+
if [[ -d "${item}" ]]; then
131+
_process_directory "${item}" "${isParentIncluded}"
143132
fi
133+
elif _is_exact_match "excludes" "${item}"; then
134+
debug " Excluding (full match): ${item}"
135+
_add_to_list "${item}"
136+
elif _ends_with "single_excludes" "${item}" || _is_wildcard_match "single_excludes" "${item}"; then
137+
debug " Excluding (single exclude): ${item}"
138+
_add_to_list "${item}"
139+
elif [[ "${isParentIncluded}" == "false" ]]; then
140+
debug " Excluding (not included): ${item}"
141+
_add_to_list "${item}"
144142
else
145-
for excluded in "${excludes[@]}"; do
146-
if [[ "${file}" == ./${excluded} ]]; then
147-
_add_to_list "${list_var}" "${file}"
148-
break
149-
elif [[ "./${excluded}" =~ ^${file} ]]; then
150-
# check subdirectories
151-
_generate_excludes_for_dir "${file}" "${includes_var}" "${excludes_var}" "${list_var}"
152-
break
153-
fi
154-
done
143+
debug " Including (parent included): ${item}"
144+
if [[ -d "${item}" ]]; then
145+
_process_directory "${item}" "${isParentIncluded}"
146+
fi
155147
fi
156148
done
157149
}
158150

159151
function _add_to_list() {
160-
local list_var="$1"
161-
local filename="$2"
162-
163-
# Note: the files end up in subdirectories under `keyman` (or rather
164-
# the directory name of $KEYMAN_ROOT), so we can
165-
# include that when matching files and directories to ignore.
166-
# shellcheck disable=SC2154
167-
eval "${list_var}+=(\"--tar-ignore=${prefix}/${filename#./}\")"
152+
local item="$1"
153+
ignore_list+=("${prefix}/${item#./}")
154+
}
155+
156+
# Returns true if one of the values in the $1 array equals ${file} ($2)
157+
# Example: will return true for array=(path1/path2) file=./path1/path2
158+
function _is_exact_match() {
159+
local array_var="$1"
160+
local file="$2"
161+
local array_name="${array_var}[@]"
162+
local haystack=("${!array_name}")
163+
local item
164+
for item in "${haystack[@]}"; do
165+
if [[ "./${item#./}" == "${file}" ]]; then
166+
return 0
167+
fi
168+
done
169+
return 1
170+
}
171+
172+
# Returns true if one of the values in the $1 array starts with ${file} ($2)
173+
# Example: will return true for array=(path1/path2) file=./path1
174+
function _starts_with() {
175+
local array_var="$1"
176+
local file="$2"
177+
local array_name="${array_var}[@]"
178+
local array_values=("${!array_name}")
179+
local array_item
180+
for array_item in "${array_values[@]}"; do
181+
if [[ "./${array_item#./}" == ${file}* ]]; then
182+
return 0
183+
fi
184+
done
185+
return 1
186+
}
187+
188+
# Returns true if ${file} ($2) ends with one of the values in the $1 array
189+
# Example: will return true for array=(path2) file=./path1/path2
190+
function _ends_with() {
191+
local array_var="$1"
192+
local file="$2"
193+
local array_name="${array_var}[@]"
194+
local array_values=("${!array_name}")
195+
local array_item
196+
for array_item in "${array_values[@]}"; do
197+
if [[ "${file}" == */${array_item#./} ]]; then
198+
return 0
199+
fi
200+
done
201+
return 1
202+
}
203+
204+
# Returns true if ${file} ($2) matches one of the values of the $1 array.
205+
# These values may contain wildcards.
206+
# Example: will return true for array=(*.sh) file=./path1/build.sh
207+
function _is_wildcard_match() {
208+
local array_var="$1"
209+
local file="$2"
210+
local array_name="${array_var}[@]"
211+
local array_values=("${!array_name}")
212+
local array_item
213+
for array_item in "${array_values[@]}"; do
214+
array_item=${array_item/./\\.}
215+
if [[ "${file}" =~ /${array_item/\*/.\*}$ ]]; then
216+
return 0
217+
fi
218+
done
219+
return 1
220+
}
221+
222+
function debug() {
223+
# echo "$@"
224+
return 0
168225
}

0 commit comments

Comments
 (0)