Merge remote-tracking branch 'origin/6.0/release' into 6.0/patch

nealquigley · nealquigley · commit 5dc053870556 · 2022-03-10T20:35:36.000Z
diff --git a/lib/common.sh b/lib/common.sh
@@ -27,7 +27,7 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
 #
 export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
 
-export UBUNTU_DISTRIBUTION="bionic"
+export UBUNTU_DISTRIBUTION="focal"
 
 #
 # We currently support getting the linux kernel from 3 different sources:
@@ -471,21 +471,10 @@ function create_workdir() {
 function install_pkgs() {
 	for attempt in {1..3}; do
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
-			"apt-get install -y --allow-downgrades $*"
+			"apt-get install -y $*"
 
-		#
-		# We use the "--allow-downgrades" for the case of a
-		# package needing to install the "unzip" debian package
-		# that we build via the "misc-debs" linux-pkg package.
-		# The "misc-debs" based "unzip" package may have an
-		# older version than what's already installed on the
-		# system, so we need to "--allow-downgrades" in order to
-		# install that specific package; further, that specific
-		# package is required to build the "virtualization"
-		# debian packages.
-		#
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-			-y --allow-downgrades "$@" && return
+			-y "$@" && return
 
 		echo "apt-get install failed, retrying."
 		sleep 10
@@ -1241,17 +1230,6 @@ function determine_target_kernels() {
 	echo_bold "  $KERNEL_VERSIONS"
 }
 
-#
-# Install gcc 8, and make it the default
-#
-function install_gcc8() {
-	logmust install_pkgs gcc-8 g++-8
-	logmust sudo update-alternatives --install /usr/bin/gcc gcc \
-		/usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7
-	logmust sudo update-alternatives --install /usr/bin/gcc gcc \
-		/usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
-}
-
 #
 # Store git-related build info for the package after the build is done.
 # Note that some of this metadata is used by the Jenkins build so be careful
diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
@@ -14,14 +14,14 @@ delphix-rust
 delphix-sso-app
 drgn
 docker-python-image
+fluentd-gems
 gdb-python
 grub2
 host-jdks
 libkdumpfile
 make-jpkg
 makedumpfile
 masking
-misc-debs
 nfs-utils
 performance-diagnostics
 ptools
diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+#
+# This package has the same Git URL as the 'masking' package. In general we
+# probably don't want to have multiple packages with the same URL, since tools
+# like git-ab-pre-push expect that there is a 1:1 correspondence between
+# packages and URLs. However, this is OK in this case because git-ab-pre-push
+# only works with packages that are included in the appliance, which this one
+# isn't.
+#
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
+
+PACKAGE_DEPENDENCIES="adoptopenjdk"
+SKIP_COPYRIGHTS_CHECK=true
+
+function prepare() {
+	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	logmust cd "$WORKDIR/repo"
+
+	logmust ./gradlew --no-daemon --stacktrace \
+		-Porg.gradle.configureondemand=false \
+		-PenvironmentName=linuxappliance \
+		:tools:docker:packageMaskingKubernetes
+
+	logmust cp -v tools/docker/build/masking-kubernetes.zip \
+		"$WORKDIR/artifacts/"
+}
diff --git a/packages/fluentd-gems/config.sh b/packages/fluentd-gems/config.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fluentd-gems.git"
+
+function build() {
+	logmust mkdir -p "$WORKDIR/repo"
+	logmust dpkg_buildpackage_default
+}
diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh
@@ -17,19 +17,16 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2"
-PACKAGE_DEPENDENCIES="zfs"
 
 UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2
-UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates
+UPSTREAM_GIT_BRANCH="applied/ubuntu/${UBUNTU_DISTRIBUTION}-updates"
 
 SKIP_COPYRIGHTS_CHECK=true
 
 #
 # Install build dependencies for the package.
 #
 function prepare() {
-	# Install libzfs which is required to build grub
-	logmust install_pkgs "$DEPDIR"/zfs/{libnvpair1linux,libuutil1linux,libzfs2linux,libzpool2linux,libzfslinux-dev}_*.deb
 	logmust install_build_deps_from_control_file
 }
 
diff --git a/packages/linux-kernel-aws/config.delphix.sh b/packages/linux-kernel-aws/config.delphix.sh
@@ -18,7 +18,7 @@
 # shellcheck disable=SC2034
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-aws.git"
 
-UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/bionic"
+UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/${UBUNTU_DISTRIBUTION}"
 # Note: UPSTREAM_GIT_BRANCH is not used here
 UPSTREAM_GIT_BRANCH="none"
 
diff --git a/packages/linux-kernel-azure/config.delphix.sh b/packages/linux-kernel-azure/config.delphix.sh
@@ -18,7 +18,7 @@
 # shellcheck disable=SC2034
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-azure.git"
 
-UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/bionic"
+UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/${UBUNTU_DISTRIBUTION}"
 # Note: UPSTREAM_GIT_BRANCH is not used here
 UPSTREAM_GIT_BRANCH="none"
 
diff --git a/packages/linux-kernel-gcp/config.delphix.sh b/packages/linux-kernel-gcp/config.delphix.sh
@@ -18,7 +18,7 @@
 # shellcheck disable=SC2034
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-gcp.git"
 
-UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/bionic"
+UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/${UBUNTU_DISTRIBUTION}"
 # Note: UPSTREAM_GIT_BRANCH is not used here
 UPSTREAM_GIT_BRANCH="none"
 
diff --git a/packages/linux-kernel-generic/config.delphix.sh b/packages/linux-kernel-generic/config.delphix.sh
@@ -18,7 +18,7 @@
 # shellcheck disable=SC2034
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-generic.git"
 
-UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/bionic"
+UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/${UBUNTU_DISTRIBUTION}"
 # Note: UPSTREAM_GIT_BRANCH is not used here
 UPSTREAM_GIT_BRANCH="none"
 
diff --git a/packages/linux-kernel-oracle/config.delphix.sh b/packages/linux-kernel-oracle/config.delphix.sh
@@ -18,7 +18,7 @@
 # shellcheck disable=SC2034
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-oracle.git"
 
-UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/bionic"
+UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/${UBUNTU_DISTRIBUTION}"
 # Note: UPSTREAM_GIT_BRANCH is not used here
 UPSTREAM_GIT_BRANCH="none"
 
diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh
diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh
@@ -17,7 +17,7 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/dlpx-app-gate.git"
-PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish host-jdks misc-debs"
+PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish host-jdks"
 
 function prepare() {
 	logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies"
@@ -26,8 +26,7 @@ function prepare() {
 	logmust install_pkgs \
 		"$DEPDIR"/adoptopenjdk/*.deb \
 		"$DEPDIR"/crypt-blowfish/*.deb \
-		"$DEPDIR"/host-jdks/*.deb \
-		"$DEPDIR"/misc-debs/unzip_6.0-21ubuntu1_amd64.deb
+		"$DEPDIR"/host-jdks/*.deb
 }
 
 function build() {
@@ -60,7 +59,7 @@ function build() {
 	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
 		logmust ant -Ddockerize=true -DbuildJni=true \
 			-DhotfixGenDlpxVersion="$DELPHIX_RELEASE_VERSION" \
-			all package
+			-Dbuild.legacy.resources.war=true all package
 	else
 		logmust ant -Ddockerize=true -DbuildJni=true all package
 	fi
diff --git a/setup.sh b/setup.sh
@@ -22,20 +22,6 @@ logmust check_running_system
 
 logmust determine_default_git_branch
 
-#
-# In order to ship vSDK 4.0.0 with 6.0.12.0, we need python3.8 (and the
-# relevant distutils release). The following line gets the deadsnakes
-# (https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) repo from
-# which we install python3.8 in packages/virtualization/config.sh.
-# This line will be removed via DLPX-78481 when 6.0/stage begins
-# tracking 6.0.13.0.
-#
-function install_python_38() {
-	logmust sudo add-apt-repository ppa:deadsnakes/ppa -y
-	logmust sudo apt-get update
-	logmust install_pkgs python 3.8
-}
-
 #
 # Update the sources.list file to point to our internal package mirror. If no
 # mirror url is passed in, then the latest mirror snapshot is used.
@@ -114,7 +100,6 @@ function add_swap() {
 	fi
 }
 
-logmust install_python_38
 logmust configure_apt_sources
 logmust sudo apt-get update
 
@@ -139,13 +124,6 @@ logmust install_pkgs \
 
 logmust install_shfmt
 
-#
-# Starting with kernel 5.4, gcc 7 can no longer compile kernel modules, so
-# install gcc 8
-# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1849348
-#
-logmust install_gcc8
-
 logmust add_swap
 
 logmust git config --global user.email "eng@delphix.com"

Original file line number	Diff line number	Diff line change
`@@ -17,19 +17,16 @@`
`17`	`17`	`# shellcheck disable=SC2034`
`18`	`18`
`19`	`19`	`DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2"`
`20`		`-PACKAGE_DEPENDENCIES="zfs"`
`21`	`20`
`22`	`21`	`UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2`
`23`		`-UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates`
	`22`	`+UPSTREAM_GIT_BRANCH="applied/ubuntu/${UBUNTU_DISTRIBUTION}-updates"`
`24`	`23`
`25`	`24`	`SKIP_COPYRIGHTS_CHECK=true`
`26`	`25`
`27`	`26`	`#`
`28`	`27`	`# Install build dependencies for the package.`
`29`	`28`	`#`
`30`	`29`	`function prepare() {`
`31`		`- # Install libzfs which is required to build grub`
`32`		`- logmust install_pkgs "$DEPDIR"/zfs/{libnvpair1linux,libuutil1linux,libzfs2linux,libzpool2linux,libzfslinux-dev}_*.deb`
`33`	`30`	`logmust install_build_deps_from_control_file`
`34`	`31`	`}`
`35`	`32`