Fix x86 trap padding semantics and robust unhook restore

Author: YinMo19

examples/instrument_unhook_restore/main.rs

@@ -72,14 +72,19 @@ extern "C" fn init() {
             }
         };
 
-        let _ = instrument(target_address, on_hit_should_not_run);
+        if let Err(err) = instrument(target_address, on_hit_should_not_run) {
+            println!("instrument failed: {err}");
+            return;
+        }
 
         let hooked = calc_fn(3, 4);
         println!("hooked_calc(3, 4) = {hooked}");
         if hooked != 123 {
             return;
         }
 
-        let _ = unhook(target_address);
+        if let Err(err) = unhook(target_address) {
+            println!("unhook failed: {err}");
+        }
     }
 }

src/lib.rs

@@ -291,13 +291,9 @@ fn instrument_internal(
             let original_bytes = memory::read_bytes(address, step_len as usize)?;
             let original4 = memory::read_bytes(address, 4)?;
 
-            if step_len == 1 {
-                let _ = memory::patch_u8(address, memory::int3_opcode())?;
-            } else {
-                let mut trap_patch = vec![0x90u8; step_len as usize];
-                trap_patch[0] = memory::int3_opcode();
-                let _ = memory::patch_bytes_public(address, &trap_patch)?;
-            }
+            let mut trap_patch = vec![0x90u8; step_len as usize];
+            trap_patch[0] = memory::int3_opcode();
+            let _ = memory::patch_bytes_public(address, &trap_patch)?;
 
             let mut opcode = [0u8; 4];
             opcode.copy_from_slice(&original4);
@@ -371,7 +367,7 @@ pub fn inline_hook(addr: u64, replace_fn: u64) -> Result<u32, SigHookError> {
             Err(err) => return Err(err),
         };
 
-        let original = memory::read_bytes(addr, patch.len())?;
+        let original = memory::read_bytes(addr, 16)?;
         let inserted = unsafe { state::cache_inline_patch(addr, &original)? };
         if let Err(err) = memory::patch_bytes_public(addr, &patch) {
             if inserted {
@@ -403,7 +399,7 @@ pub fn inline_hook(addr: u64, replace_fn: u64) -> Result<u32, SigHookError> {
             memory::encode_absolute_jump(replace_fn).to_vec()
         };
 
-        let original = memory::read_bytes(addr, patch.len())?;
+        let original = memory::read_bytes(addr, memory::encode_absolute_jump(0).len())?;
         let inserted = unsafe { state::cache_inline_patch(addr, &original)? };
         if let Err(err) = memory::patch_bytes_public(addr, &patch) {
             if inserted {

src/memory.rs

@@ -248,12 +248,6 @@ fn patch_bytes(address: u64, bytes: &[u8]) -> Result<Vec<u8>, SigHookError> {
     Ok(original)
 }
 
-#[cfg(all(target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))]
-pub(crate) fn patch_u8(address: u64, new_opcode: u8) -> Result<u8, SigHookError> {
-    let original = patch_bytes(address, &[new_opcode])?;
-    Ok(original[0])
-}
-
 pub(crate) fn read_bytes(address: u64, len: usize) -> Result<Vec<u8>, SigHookError> {
     if address == 0 || len == 0 {
         return Err(SigHookError::InvalidAddress);