DLPX-88225 Automated hotfix process leads to hotfix branch engine version always being set as latest engine version TOOL-23229 Improvements to masking/containerized-masking/virtualization linux-pkg handling (#304)

PR URL: https://www.github.com/delphix/linux-pkg/pull/304

Author: sonamkindy

lib/common.sh

@@ -28,7 +28,7 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
 #
 export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
 
-export UBUNTU_DISTRIBUTION="noble"
+export UBUNTU_DISTRIBUTION="focal"
 
 #
 # We currently support getting the linux kernel from 3 different sources:
@@ -559,7 +559,7 @@ function install_shfmt() {
 			arch="arm"
 		fi
 		logmust sudo wget -nv -O /usr/local/bin/shfmt \
-			"https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_$arch"
+			https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_$arch
 		logmust sudo chmod +x /usr/local/bin/shfmt
 	fi
 	echo "shfmt version $(shfmt -version) is installed."
@@ -597,47 +597,6 @@ function install_kernel_headers() {
 	done
 }
 
-#
-# Install kernel dbgsym packages for all target kernels.
-# The kernel packages are fetched from S3.
-#
-function install_kernel_dbgsyms() {
-	logmust determine_target_kernels
-	check_env KERNEL_VERSIONS DEPDIR
-
-	logmust list_linux_kernel_packages
-	# Note: linux packages returned in _RET_LIST
-
-	local pkg
-	for pkg in "${_RET_LIST[@]}"; do
-		logmust install_pkgs "$DEPDIR/$pkg/"linux-image-*dbgsym*.ddeb
-	done
-
-	#
-	# Verify that headers are installed for all kernel versions
-	# stored in KERNEL_VERSIONS
-	#
-	local kernel
-	for kernel in $KERNEL_VERSIONS; do
-		logmust dpkg-query -l "linux-image-$kernel-dbgsym*" >/dev/null
-	done
-}
-
-function install_kernel_headers_and_dbgsyms() {
-	logmust install_kernel_headers
-	logmust install_kernel_dbgsyms
-
-	#
-	# Additionally, we add these symlinks so that kernel module builds will
-	# be able to generate BTF information, as they look for the "vmlinux" file
-	# in the kernel header directory.
-	#
-	local kernel
-	for kernel in $KERNEL_VERSIONS; do
-		logmust sudo ln -sf "/usr/lib/debug/boot/vmlinux-$kernel" "/usr/src/linux-headers-$kernel/vmlinux"
-	done
-}
-
 function delphix_revision() {
 	#
 	# We use "delphix" in the default revision to make it easy to find all
@@ -657,10 +616,6 @@ function delphix_revision() {
 	echo "delphix.$(date '+%Y.%m.%d.%H.%M')"
 }
 
-function compare_versions() {
-	dpkg --compare-versions "$@"
-}
-
 function determine_dependencies_base_url() {
 	[[ -n "$DEPENDENCIES_BASE_URL" ]] && return
 
@@ -791,24 +746,6 @@ function git_fetch_helper() {
 	git fetch "$git_url" "$@" || die "git fetch failed"
 }
 
-#
-# Run git fetch with the passed arguments. Git url must be passed as first
-# argument. If FETCH_GIT_TOKEN is set and this is a github repository
-# then pass-in the token when fetching.
-#
-function git_branch_exists() {
-	local git_url="$1"
-	local label=''
-	shift
-
-	if [[ -n "$FETCH_GIT_TOKEN" ]] &&
-		[[ "$git_url" == https://github.com/* ]]; then
-		git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
-		label='[token passed]'
-	fi
-	git fetch "$git_url" "$@" --no-tags --depth=1 || return 1
-}
-
 #
 # Fetch package repository into $WORKDIR/repo
 #
@@ -832,25 +769,6 @@ function fetch_repo_from_git() {
 		logmust git show-ref repo-HEAD
 		logmust git show-ref upstream-HEAD
 	else
-
-		#
-		# The "os-upgrade" branch is used for Ubuntu version upgrade projects. As such, the following
-		# block of code will only be executed if the `PACKAGE_GIT_BRANCH` is set to "os-upgrade". Not
-		# all product repositories are required to have an "os-upgrade" branch, because some packages
-		# will successfully build on the new Ubuntu version as well as the old without any changes.
-		# To allow engineering to create the "os-upgrade" branch only in repositories that require
-		# changes for the new Ubuntu version, this block of code will check if the "os-upgrade" branch
-		# exists in the repository and if it does not, it will build off of "develop" instead.
-		#
-		if [[ "$PACKAGE_GIT_BRANCH" == "os-upgrade" ]]; then
-			git_branch_exists "$PACKAGE_GIT_URL" "$PACKAGE_GIT_BRANCH"
-			branch_exists=$?
-			if [[ $branch_exists -ne 0 ]]; then
-				echo "NOTE: The branch 'os-upgrade' does not exist for this repo. Using 'develop' instead."
-				PACKAGE_GIT_BRANCH="develop"
-			fi
-		fi
-
 		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
 			"+$PACKAGE_GIT_BRANCH:repo-HEAD" --depth=1
 		logmust git show-ref repo-HEAD
@@ -1087,24 +1005,9 @@ function push_to_remote() {
 #
 function set_changelog() {
 	check_env PACKAGE_REVISION
+	local src_package="${1:-$PACKAGE}"
 	local final_version
 
-	#
-	# If the name of the source package isn't passed in as a parameter,
-	# then deduce it. If there's a debian/control file that specifies that
-	# package name, then use it. Otherwise, default to the name of the
-	# linux-pkg directory name. This can't always be the default because
-	# for some packages, those are different.  For example, the
-	# challenge-response linux-pkg directory generates the
-	# pam-challenge-response debian package.
-	#
-	if [[ -n $1 ]]; then
-		src_package=$1
-	elif [[ -f debian/control ]]; then
-		src_package=$(awk '/^Source:/ { print $2 }' debian/control)
-	fi
-	src_package=${src_package:-$PACKAGE}
-
 	#
 	# If PACKAGE_VERSION hasn't been set already, then retrieve it from
 	# The changelog file. If the changelog file doesn't exist, which
@@ -1169,7 +1072,7 @@ function get_kernel_version_for_platform_from_apt() {
 	# image for that particular platform. For instance, Ubuntu has a
 	# meta-package for AWS called 'linux-image-aws', which depends on
 	# package 'linux-image-4.15.0-1027-aws'. The latter is the linux image
-	# for kernel version '4.15.0-1027-aws'. We use this dependency to figure
+	# for kernel version '4.15.0-1027-aws'. We use this depenency to figure
 	# out the default kernel version for a given platform.
 	#
 	# The "generic" platform is a special case, since we want to use the
@@ -1178,10 +1081,11 @@ function get_kernel_version_for_platform_from_apt() {
 	# Note that while the default kernel is usually also the latest
 	# available, it is not always the case.
 	#
-	if [[ "$platform" == generic ]]; then
-		package="linux-image-${platform}-hwe-24.04"
-	else
+
+	if [[ "$platform" != generic ]] && [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then
 		package="linux-image-${platform}"
+	else
+		package="linux-image-${platform}-hwe-20.04"
 	fi
 
 	if [[ "$(apt-cache show --no-all-versions "$package" \
@@ -1414,87 +1318,3 @@ function set_secret_build_args() {
 		_SECRET_BUILD_ARGS+=("-DSECRET_DB_AWS_REGION=$SECRET_DB_AWS_REGION")
 	fi
 }
-
-#
-# Secure boot variables and functions
-#
-# S3 bucket containing keys and certs
-# ./db subdirectory contains the db key and various certs:
-#   .der is for signing modules like ZFS and connstat
-#   .crt is for signing vmlinuz
-#   signing_key.pem is the format expected by kernel build for signing its modules
-#
-# ./pub contains the auth files, secure boot enrollment certs.
-#
-S3_KEYS_URL="s3://secure-boot-keys-prod/release"
-#
-# The kernel build expects the signing_key.pem in this directory, i.e.
-# CONFIG_MODULE_SIG_KEY is set to /var/tmp/sbkeys/signing_key.pem in
-# resources/delphix_kernel_annotations
-#
-SB_KEYS_DIR="/var/tmp/sbkeys"
-SBSIGN_KEY="$SB_KEYS_DIR/db.key"
-SBSIGN_DER="$SB_KEYS_DIR/db.der"
-
-function download_keys() {
-	logmust mkdir -p $SB_KEYS_DIR
-	logmust aws s3 cp --recursive "$S3_KEYS_URL/db/" $SB_KEYS_DIR
-}
-
-function delete_keys() {
-	logmust rm -r $SB_KEYS_DIR
-}
-
-# Update DEBIAN/md5sum for package directory after
-# some files were updated, i.e. secure-boot signed.
-#
-function update_md5sums() {
-	pkg_dir=$1
-	echo_bold "Updating md5sums for $pkg_dir"
-
-	(
-		cd "$pkg_dir" || exit
-		: >DEBIAN/md5sums
-		# print paths relative to root of package
-		while IFS= read -r -d '' f; do
-			rel="${f#./}"
-			md5sum "$rel" >>DEBIAN/md5sums
-		done < <(find . -type f ! -path './DEBIAN/*' ! -path './etc/depmod*' -print0)
-	)
-}
-
-function repack_deb() {
-	deb_name=$1
-	deb_dir=$2
-	temp_deb=$(mktemp /tmp/deb.XXXXXX)
-
-	logmust fakeroot dpkg-deb -b "$deb_dir" "$temp_deb"
-	logmust mv "$temp_deb" "$deb_name"
-}
-
-#
-# Sign .ko files in the module list
-#
-function sign_modules() {
-	deb_pkgs="$1"
-	echo_bold "Signing $deb_pkgs"
-	download_keys
-
-	while IFS= read -r pkg; do
-		echo_bold "Processing $pkg"
-		temp_dir=$(mktemp -d -p "/var/tmp/")
-		logmust fakeroot dpkg-deb -R "$pkg" "$temp_dir"
-
-		# Find and sign all .ko files in package
-		find "$temp_dir" -type f -name "*.ko" -print0 |
-			while IFS= read -r -d '' kernel_mod; do
-				logmust kmodsign sha256 "$SBSIGN_KEY" "$SBSIGN_DER" "$kernel_mod" "$kernel_mod.signed"
-				logmust mv "$kernel_mod.signed" "$kernel_mod"
-				logmust modinfo -F signer "$kernel_mod"
-			done
-		# Repack the .deb"
-		update_md5sums "$temp_dir"
-		repack_deb "$pkg" "$temp_dir"
-	done <<<"$deb_pkgs"
-	delete_keys
-}