Re: [patch i386, 2/2] skip insns generated by -fstack-protector

[Yao Qi <yao at codesourcery dot com>]

On 12/24/2010 04:21 PM, Yao Qi wrote:
> Patch 2 is about handling i386 stack protector insns during prologue
> analysis.  Without patch 1, patch 2 doesn't work in some cases.

Here is a prologue generated by GCC, instructions on [1] are for stack 
protector.

     push   %ebp
     mov    %esp,%ebp

     and    $0xfffffff0,%esp
     add    $0xffffff80,%esp

     mov    %gs:0x14,%eax   // <---- [1]
     mov    %eax,0x7c(%esp) // <---- [1]
     xor    %eax,%eax       // <---- [1]

Compared with instructions for arm stack protector, i386's counterpart 
is relatively simpler.  This patch is to handle them in prologue 
parsing.  Comments are welcome.

--
Yao Qi

gdb/

	* i386-tdep.c (i386_skip_stack_protector) New.
	(i386_analyze_prologue): Chain i386_skip_stack_protector.
	
diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index 8c6f896..ee40603 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -1455,6 +1455,35 @@ i386_analyze_register_saves (CORE_ADDR pc, CORE_ADDR current_pc,
   return pc;
 }
 
+/* Check whether PC points at code that for stack protector, which
+   is usually a sequence of three instructions,
+
+   mov    %gs:0x14,%eax
+   mov    %eax,0x7c(%esp)
+   xor    %eax,%eax
+
+   If so, returns the address of the first instruction after the
+   stack protector code or CURRENT_PC, whichever is smaller.
+   Otherwise, return PC.  */
+
+static CORE_ADDR
+i386_skip_stack_protector (CORE_ADDR pc, CORE_ADDR current_pc)
+{
+  gdb_byte buf[12];
+  if (target_read_memory (pc, buf, sizeof buf))
+    return pc;
+
+  /* Instruction `mov %gs:0x14,%eax' can be regarded as `fingerprint' of a
+   sequence of code for stack protector, since it is unique and can't be
+   found elsewhere.  */
+  if (/* mov %gs:0x14,%eax.  */
+      buf[0] != 0x65 && buf[1] != 0xa1 && buf[2] != 14
+      && buf[10] != 0x31 /* xor    %eax,%eax.  */)
+    return pc;
+
+  return min (pc + 12, current_pc);
+}
+
 /* Do a full analysis of the prologue at PC and update CACHE
    accordingly.  Bail out early if CURRENT_PC is reached.  Return the
    address where the analysis stopped.
@@ -1493,7 +1522,9 @@ i386_analyze_prologue (struct gdbarch *gdbarch,
   pc = i386_skip_probe (pc);
   pc = i386_analyze_stack_align (pc, current_pc, cache);
   pc = i386_analyze_frame_setup (gdbarch, pc, current_pc, cache);
-  return i386_analyze_register_saves (pc, current_pc, cache);
+  pc = i386_analyze_register_saves (pc, current_pc, cache);
+  pc = i386_skip_stack_protector (pc, current_pc);
+  return pc;
 }
 
 /* Return PC of first real instruction.  */