BCF-6450: Rear Recover hung on last stage for hours

usr/share/rear/conf/default.conf

@@ -4429,3 +4429,14 @@ PYTHON_MINIMAL=false
 ####
 # Location of file created just before layout code is executed
 LAYOUT_CODE_STARTED="$TMPDIR/cove_rear_layout_code_started"
+
+####
+# Use the --noprofile option for the login shell during recovery process in
+# calls such as 'chroot $TARGET_FS_ROOT /bin/bash --login -c <cmd>'.
+#
+# By default, automatically detect whether the login shell can be used as is.
+# If USE_NOPROFILE_FOR_LOGIN_SHELL=1, skip the check and force the use of
+# the -noprofile option.
+# If USE_NOPROFILE_FOR_LOGIN_SHELL=0, skip the check use the login shell
+# without the -noprofile option.
+USE_NOPROFILE_FOR_LOGIN_SHELL="${USE_NOPROFILE_FOR_LOGIN_SHELL:-}"

usr/share/rear/finalize/Debian/i386/550_rebuild_initramfs.sh

@@ -46,7 +46,7 @@ echo "$INITRD_MODULES" >$TARGET_FS_ROOT/etc/initramfs-tools/modules
 
 # Handle mdadm.conf Debian style:
 if [ -r /proc/mdstat -a -r $TARGET_FS_ROOT/etc/mdadm/mdadm.conf -a -x $TARGET_FS_ROOT/usr/share/mdadm/mkconf ] ; then
-    if chroot $TARGET_FS_ROOT /bin/bash --login -c "/usr/share/mdadm/mkconf >/etc/mdadm/mdadm.conf" ; then
+    if run_in_chroot "/usr/share/mdadm/mkconf >/etc/mdadm/mdadm.conf" ; then
         LogPrint "Updated '/etc/mdadm/mdadm.conf' before recreating initramfs"
     else
         LogPrint "WARNING:

usr/share/rear/finalize/Linux-i386/660_install_grub2.sh

@@ -61,7 +61,7 @@ function bios_grub_install ()
     if is_true $USING_UEFI_BOOTLOADER ; then
         # If running under UEFI, we need to specify the target explicitly, otherwise grub-install thinks
         # that we are installing the EFI bootloader.
-        if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-install --target=i386-pc $grub2_install_device" ; then
+        if ! run_in_chroot "$grub_name-install --target=i386-pc $grub2_install_device" ; then
             LogPrintError "Failed to install GRUB2 for BIOS boot (target i386-pc) on $bootdisk"
             # purely informational test that may help to explain the reason for the error
             if ! test -d "$TARGET_FS_ROOT/boot/$grub_name/i386-pc" ; then
@@ -70,7 +70,7 @@ function bios_grub_install ()
             return 1
         fi
     else
-        if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-install $grub2_install_device" ; then
+        if ! run_in_chroot "$grub_name-install $grub2_install_device" ; then
             LogPrintError "Failed to install GRUB2 on $grub2_install_device"
             return 1
         fi
@@ -109,7 +109,7 @@ if ! test -d "$TARGET_FS_ROOT/boot/$grub_name" ; then
 fi
 
 # Generate GRUB configuration file anew to be on the safe side (this could be even mandatory in MIGRATION_MODE):
-if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-mkconfig -o /boot/$grub_name/grub.cfg" ; then
+if ! run_in_chroot "$grub_name-mkconfig -o /boot/$grub_name/grub.cfg" ; then
     LogPrintError "Failed to generate boot/$grub_name/grub.cfg in $TARGET_FS_ROOT - trying to install GRUB2 nevertheless"
 fi
 

usr/share/rear/finalize/Linux-ppc64le/660_install_grub2.sh

@@ -86,7 +86,7 @@ fi
 
 # Generate GRUB configuration file anew to be on the safe side (this could be even mandatory in MIGRATION_MODE):
 local grub2_config_generated="yes"
-if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-mkconfig -o /boot/$grub_name/grub.cfg" ; then
+if ! run_in_chroot "$grub_name-mkconfig -o /boot/$grub_name/grub.cfg" ; then
     grub2_config_generated="no"
     # TODO: We should make this fatal.  Outdated/incomplete/just wrong grub2.cfg may result into an unbootable system.
     LogPrintError "Failed to generate boot/$grub_name/grub.cfg in $TARGET_FS_ROOT - trying to install GRUB2 nevertheless"
@@ -159,7 +159,7 @@ if test "$GRUB2_INSTALL_DEVICES" ; then
         else
             LogPrint "Installing GRUB2 on $grub2_install_device (specified in GRUB2_INSTALL_DEVICES)"
         fi
-        if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-install $grub2_no_nvram_option $grub2_install_device" ; then
+        if ! run_in_chroot "$grub_name-install $grub2_no_nvram_option $grub2_install_device" ; then
             LogPrintError "Failed to install GRUB2 on $grub2_install_device"
             grub2_install_failed="yes"
         fi
@@ -210,7 +210,7 @@ for part in $part_list ; do
         # Zero out the PPC PReP boot partition
         # cf. https://github.com/rear/rear/pull/673
         dd if=/dev/zero of=$part
-        if chroot $TARGET_FS_ROOT /bin/bash --login -c "$grub_name-install $grub2_no_nvram_option $part" ; then
+        if run_in_chroot "$grub_name-install $grub2_no_nvram_option $part" ; then
             # In contrast to the above behaviour when GRUB2_INSTALL_DEVICES is specified
             # consider it here as a successful bootloader installation when GRUB2
             # got installed on at least one PPC PReP boot partition:

usr/share/rear/finalize/SUSE_LINUX/s390/660_install_grub2_and_zipl.sh

@@ -43,7 +43,7 @@ type -p grub-probe || type -p grub2-probe || return 0
 
 LogPrint "Installing GRUB2 boot loader plus ZIPL..."
 
-chroot $TARGET_FS_ROOT /bin/bash --login -c "update-bootloader --reinit" && NOBOOTLOADER=''
+run_in_chroot "update-bootloader --reinit" && NOBOOTLOADER=''
 
 is_true $NOBOOTLOADER || return 0
 LogPrintError "Failed to install GRUB2 plus ZIPL - you may have to manually install it"

usr/share/rear/lib/linux-functions.sh

@@ -255,3 +255,52 @@ function filesystem_name () {
     fi
 }
 
+# Run the given command as 'chroot $TARGET_FS_ROOT /bin/bash --login -c <cmd>'.
+# Detect whether the login shell can be used as is or the --noprofile option
+# is required.
+function run_in_chroot() {
+    if [ -z "$USE_NOPROFILE_FOR_LOGIN_SHELL" ]; then
+        chroot "$TARGET_FS_ROOT" /bin/bash --login -c true 0<&6 &
+        local pid=$!
+
+        for _ in {1..10}; do
+            sleep 0.5
+            if ! kill -0 "$pid" 2>/dev/null; then
+                if wait "$pid"; then
+                    USE_NOPROFILE_FOR_LOGIN_SHELL=0
+                    LogPrint "The login shell works without any issues."
+                else
+                    USE_NOPROFILE_FOR_LOGIN_SHELL=1
+                fi
+                break
+            fi
+        done
+
+        # If login shell is blocked, e.g., because /bin/bash is
+        # called inside /root/.profile, need to kill the process
+        if [ -z "$USE_NOPROFILE_FOR_LOGIN_SHELL" ]; then
+            kill -9 "$pid"
+            wait "$pid"
+            USE_NOPROFILE_FOR_LOGIN_SHELL=1
+        fi
+
+        if [ "$USE_NOPROFILE_FOR_LOGIN_SHELL" = "1" ]; then
+            if ! chroot "$TARGET_FS_ROOT" /bin/bash --login --noprofile -c true; then
+                LogPrintError "Cannot chroot into '$TARGET_FS_ROOT'."
+                LogPrintError "The restored file system might be broken."
+            else
+                LogPrintError "Cannot use the login shell inside the restored file system."
+                LogPrintError "The login shell will be used with the '--noprofile' option."
+            fi
+        fi
+    fi
+
+    local command="$1"
+
+    local noprofile=""
+    if [ "$USE_NOPROFILE_FOR_LOGIN_SHELL" = "1" ]; then
+        noprofile="--noprofile"
+    fi
+
+    chroot "$TARGET_FS_ROOT" /bin/bash --login $noprofile -c "$command"
+}

usr/share/rear/restore/YUM/default/950_grub2_mkconfig.sh

@@ -31,10 +31,10 @@ mount -o bind /dev $TARGET_FS_ROOT/dev || true
 #chroot $TARGET_FS_ROOT /sbin/mkinitrd -v
 
 # Run grub2-mkconfig in the target system.
-# A login shell in between is needed when shell scripts are called insinde 'chroot'
+# A login shell in between is needed when shell scripts are called inside 'chroot'
 # cf. https://github.com/rear/rear/issues/862#issuecomment-282039428
 # In particular grub2-mkconfig is a shell script that calls other shell scripts:
-chroot $TARGET_FS_ROOT /bin/bash --login -c '/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg'
+run_in_chroot '/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg'
 
 # FIXME: This should not be needed here but work via finalize/Linux-i386/660_install_grub2.sh
 # Install bootloader in the target system:

usr/share/rear/restore/YUM/default/970_set_root_password.sh

@@ -33,7 +33,7 @@ set -e -u -o pipefail
 # the commands inside 'chroot' as one would type them in a normal working shell.
 # In particular one can call programs (like 'passwd') by their basename without path
 # cf. https://github.com/rear/rear/issues/862#issuecomment-274068914
-{ chroot $TARGET_FS_ROOT /bin/bash --login -c "echo -e '$root_password\n$root_password' | passwd root" ; } 2>>/dev/$SECRET_OUTPUT_DEV
+{ run_in_chroot "echo -e '$root_password\n$root_password' | passwd root" ; } 2>>/dev/$SECRET_OUTPUT_DEV
 
 # Restore the ReaR default bash flags and options (see usr/sbin/rear):
 apply_bash_flags_and_options_commands "$DEFAULT_BASH_FLAGS_AND_OPTIONS_COMMANDS"

usr/share/rear/restore/YUM/default/980_initial_network_setup.sh

@@ -36,22 +36,22 @@ for network_setup_command in "${YUM_NETWORK_SETUP_COMMANDS[@]}" ; do
             # plus automated response to all requested user input via yes '' (i.e. only plain [Enter] as user input)
             # and ignore non zero exit codes from YaST to avoid that "rear recover" aborts here:
             LogPrint "Initial network setup in the target system via 'yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp'"
-            chroot $TARGET_FS_ROOT /bin/bash --login -c "yes '' | TERM=dumb yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp" || true
+            run_in_chroot "yes '' | TERM=dumb yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp" || true
             ;;
         (NETWORKING_PREPARATION_COMMANDS)
             LogPrint "Initial network setup in the target system as specified in NETWORKING_PREPARATION_COMMANDS"
             for networking_preparation_command in "${NETWORKING_PREPARATION_COMMANDS[@]}" ; do
                 if test "$networking_preparation_command" ; then
                     # Only report errors to avoid that "rear recover" aborts here:
-                    chroot $TARGET_FS_ROOT /bin/bash --login -c "$networking_preparation_command" || LogPrint "Command failed: $networking_preparation_command"
+                    run_in_chroot "$networking_preparation_command" || LogPrint "Command failed: $networking_preparation_command"
                 fi
             done
             ;;
         (*)
             if test "$network_setup_command" ; then
                 LogPrint "Initial network setup in the target system via $network_setup_command"
                 # Only report errors to avoid that "rear recover" aborts here:
-                chroot $TARGET_FS_ROOT /bin/bash --login -c "$network_setup_command" || LogPrint "Command failed: $network_setup_command"
+                run_in_chroot "$network_setup_command" || LogPrint "Command failed: $network_setup_command"
             fi
             ;;
     esac

usr/share/rear/restore/ZYPPER/default/950_grub2_mkconfig.sh

@@ -28,10 +28,10 @@ mount -o bind /dev $TARGET_FS_ROOT/dev || true
 chroot $TARGET_FS_ROOT /sbin/mkinitrd -v
 
 # Run grub2-mkconfig in the target system.
-# A login shell in between is needed when shell scripts are called insinde 'chroot'
+# A login shell in between is needed when shell scripts are called inside 'chroot'
 # cf. https://github.com/rear/rear/issues/862#issuecomment-282039428
 # In particular grub2-mkconfig is a shell script that calls other shell scripts:
-chroot $TARGET_FS_ROOT /bin/bash --login -c '/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg'
+run_in_chroot '/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg'
 
 # FIXME: This should not be needed here but work via finalize/Linux-i386/660_install_grub2.sh
 # Install bootloader in the target system:

usr/share/rear/restore/ZYPPER/default/970_set_root_password.sh

@@ -32,7 +32,7 @@ set -e -u -o pipefail
 # the commands inside 'chroot' as one would type them in a normal working shell.
 # In particular one can call programs (like 'passwd') by their basename without path
 # cf. https://github.com/rear/rear/issues/862#issuecomment-274068914
-{ chroot $TARGET_FS_ROOT /bin/bash --login -c "echo -e '$root_password\n$root_password' | passwd root" ; } 2>>/dev/$SECRET_OUTPUT_DEV
+{ run_in_chroot "echo -e '$root_password\n$root_password' | passwd root" ; } 2>>/dev/$SECRET_OUTPUT_DEV
 
 # Restore the ReaR default bash flags and options (see usr/sbin/rear):
 apply_bash_flags_and_options_commands "$DEFAULT_BASH_FLAGS_AND_OPTIONS_COMMANDS"

usr/share/rear/restore/ZYPPER/default/980_initial_network_setup.sh

@@ -38,22 +38,22 @@ for network_setup_command in "${ZYPPER_NETWORK_SETUP_COMMANDS[@]}" ; do
             # plus automated response to all requested user input via yes '' (i.e. only plain [Enter] as user input)
             # and ignore non zero exit codes from YaST to avoid that "rear recover" aborts here:
             LogPrint "Initial network setup in the target system via 'yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp'"
-            chroot $TARGET_FS_ROOT /bin/bash --login -c "yes '' | TERM=dumb yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp" || true
+            run_in_chroot "yes '' | TERM=dumb yast2 --ncurses lan add name=eth0 ethdevice=eth0 bootproto=dhcp" || true
             ;;
         (NETWORKING_PREPARATION_COMMANDS)
             LogPrint "Initial network setup in the target system as specified in NETWORKING_PREPARATION_COMMANDS"
             for networking_preparation_command in "${NETWORKING_PREPARATION_COMMANDS[@]}" ; do
                 if test "$networking_preparation_command" ; then
                     # Only report errors to avoid that "rear recover" aborts here:
-                    chroot $TARGET_FS_ROOT /bin/bash --login -c "$networking_preparation_command" || LogPrint "Command failed: $networking_preparation_command"
+                    run_in_chroot "$networking_preparation_command" || LogPrint "Command failed: $networking_preparation_command"
                 fi
             done
             ;;
         (*)
             if test "$network_setup_command" ; then
                 LogPrint "Initial network setup in the target system via $network_setup_command"
                 # Only report errors to avoid that "rear recover" aborts here:
-                chroot $TARGET_FS_ROOT /bin/bash --login -c "$network_setup_command" || LogPrint "Command failed: $network_setup_command"
+                run_in_chroot "$network_setup_command" || LogPrint "Command failed: $network_setup_command"
             fi
             ;;
     esac

usr/share/rear/restore/default/900_create_missing_directories.sh

@@ -75,7 +75,7 @@ if test -f "$directories_permissions_owner_group_file" ; then
                     # the commands inside 'chroot' as one would type them in a normal working shell.
                     # In particular one can call programs (like 'chown') by their basename without path
                     # cf. https://github.com/rear/rear/issues/862#issuecomment-274068914
-                    if ! chroot $TARGET_FS_ROOT /bin/bash --login -c "chown $v $owner:$group $directory" 1>&2 ; then
+                    if ! run_in_chroot "chown $v $owner:$group $directory" 1>&2 ; then
                         LogPrintError "Failed to 'chown $owner:$group $directory' "
                     fi
                 fi