[RFC][PATCH 3/4][kprobe](djprobe) djprobe for i386 architecture code

[Masami Hiramatsu <masami dot hiramatsu dot pt at hitachi dot com>]

Hi,

This patch is i386 architecture dependent part of the djprobe.
It is completely merged to arch/i386/kprobes.c and
include/asm-i386/kprobes.h.
And I modified the template of the stub code to use fast call.

Thanks,

-- 
Masami HIRAMATSU
Linux Technology Center
Hitachi, Ltd., Systems Development Laboratory
E-mail: masami.hiramatsu.pt@hitachi.com

---
 arch/i386/kernel/kprobes.c |  233 ++++++++++++++++++++++++++++++++++++++++-----
 include/asm-i386/kprobes.h |   11 ++
 2 files changed, 222 insertions(+), 22 deletions(-)

Index: linux-2.6.19-rc5-mm2/arch/i386/kernel/kprobes.c
===================================================================
--- linux-2.6.19-rc5-mm2.orig/arch/i386/kernel/kprobes.c
+++ linux-2.6.19-rc5-mm2/arch/i386/kernel/kprobes.c
@@ -26,6 +26,9 @@
  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  *		<prasanna@in.ibm.com> added function-return probes.
+ * 2006-Nov	Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> and
+ *		Satoshi Oshima <soshima@redhat.com> added direct-jump
+ *		optimization (a.k.a. djprobe).
  */

 #include <linux/kprobes.h>
@@ -40,9 +43,51 @@ void jprobe_return_end(void);

 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
+/* Instruction pages for djprobe's stub code */
+static struct kprobe_insn_page_list djprobe_insn_pages = {
+	.list = HLIST_HEAD_INIT,
+	.insn_size = 0,
+	.nr_garbage = 0
+};
+
+/* Used in djprobe_template_holder and kretprobe_trampoline_holder  */
+#define SAVE_REGS_STRING	\
+	"	pushl %gs\n"	\
+	"	pushl %es\n"	\
+	"	pushl %ds\n"	\
+	"	pushl %eax\n"	\
+	"	pushl %ebp\n"	\
+	"	pushl %edi\n"	\
+	"	pushl %esi\n"	\
+	"	pushl %edx\n"	\
+	"	pushl %ecx\n"	\
+	"	pushl %ebx\n"
+#define RESTORE_REGS_STRING	\
+	"	popl %ebx\n"	\
+	"	popl %ecx\n"	\
+	"	popl %edx\n"	\
+	"	popl %esi\n"	\
+	"	popl %edi\n"	\
+	"	popl %ebp\n"	\
+	"	popl %eax\n"

-/* insert a jmp code */
-static __always_inline void set_jmp_op(void *from, void *to)
+/*
+ * On pentium series, Unsynchronized cross-modifying code
+ * operations can cause unexpected instruction execution results.
+ * So after code modified, we should synchronize it on each processor.
+ */
+static void __local_serialize_cpu(void *info)
+{
+	sync_core();
+}
+
+void arch_serialize_cpus(void)
+{
+	on_each_cpu(__local_serialize_cpu, NULL, 1, 1);
+}
+
+/* Insert a jmp code */
+static __always_inline void set_jmp_op(void *from, void *to, int call)
 {
 	struct __arch_jmp_op {
 		char op;
@@ -50,7 +95,11 @@ static __always_inline void set_jmp_op(v
 	} __attribute__((packed)) *jop;
 	jop = (struct __arch_jmp_op *)from;
 	jop->raddr = (long)(to) - ((long)(from) + 5);
-	jop->op = RELATIVEJUMP_INSTRUCTION;
+	if (call) {
+		jop->op = RELATIVECALL_INSTRUCTION;
+	} else {
+		jop->op = RELATIVEJUMP_INSTRUCTION;
+	}
 }

 /*
@@ -363,16 +412,7 @@ no_kprobe:
 			"	pushf\n"
 			/* skip cs, eip, orig_eax */
 			"	subl $12, %esp\n"
-			"	pushl %gs\n"
-			"	pushl %ds\n"
-			"	pushl %es\n"
-			"	pushl %eax\n"
-			"	pushl %ebp\n"
-			"	pushl %edi\n"
-			"	pushl %esi\n"
-			"	pushl %edx\n"
-			"	pushl %ecx\n"
-			"	pushl %ebx\n"
+			SAVE_REGS_STRING
 			"	movl %esp, %eax\n"
 			"	call trampoline_handler\n"
 			/* move eflags to cs */
@@ -380,14 +420,8 @@ no_kprobe:
 			"	movl %edx, 48(%esp)\n"
 			/* save true return address on eflags */
 			"	movl %eax, 52(%esp)\n"
-			"	popl %ebx\n"
-			"	popl %ecx\n"
-			"	popl %edx\n"
-			"	popl %esi\n"
-			"	popl %edi\n"
-			"	popl %ebp\n"
-			"	popl %eax\n"
-			/* skip eip, orig_eax, es, ds, gs */
+			RESTORE_REGS_STRING
+			/* skip gs, ds, es, orig_eax, eip */
 			"	addl $20, %esp\n"
 			"	popf\n"
 			"	ret\n");
@@ -539,7 +573,7 @@ static void __kprobes resume_execution(s
 			 * jumps back to correct address.
 			 */
 			set_jmp_op((void *)regs->eip,
-				   (void *)orig_eip + (regs->eip - copy_eip));
+				   (void *)orig_eip + (regs->eip - copy_eip), 0);
 			p->ainsn.boostable = 1;
 		} else {
 			p->ainsn.boostable = -1;
@@ -753,7 +787,162 @@ int __kprobes longjmp_break_handler(stru
 	return 0;
 }

+#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PM)
+/* Functions for Direct Jump Optimization (djprobe) */
+/* stub template addresses */
+ void __kprobes djprobe_template_holder(void)
+ {
+	asm volatile ( ".global arch_tmpl_stub_entry\n"
+			"arch_tmpl_stub_entry: \n"
+		       "	pushf\n"
+		       /* skip cs, eip, orig_eax */
+		       "	subl $12, %esp\n"
+		       SAVE_REGS_STRING
+		       "	movl %esp, %edx\n"
+		       ".global arch_tmpl_stub_val\n"
+		       "arch_tmpl_stub_val: \n"
+		       "	movl $0xffffffff, %eax\n"
+		       ".global arch_tmpl_stub_call\n"
+		       "arch_tmpl_stub_call: \n"
+		       ASM_NOP5
+		       RESTORE_REGS_STRING
+			/* skip gs, ds, es, orig_eax, eip, cs */
+			"	addl $24, %esp\n"
+			"	popf\n"
+		       ".global arch_tmpl_stub_end\n"
+		       "arch_tmpl_stub_end: \n");
+ }
+
+/* djprobe call back function: called from stub code */
+fastcall static void djprobe_callback(struct djprobe_instance *djpi,
+				      struct pt_regs *regs)
+{
+	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+
+	preempt_disable();
+	if (kprobe_running()) {
+		kprobes_inc_nmissed_count(&djpi->kp);
+	} else {
+		/* save skipped registers */
+		regs->xcs = __KERNEL_CS;
+		regs->eip = (long)djpi->kp.addr + sizeof(kprobe_opcode_t);
+		regs->orig_eax = 0xffffffff;
+
+		__get_cpu_var(current_kprobe) = &djpi->kp;
+		kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+		aggr_pre_handler(&djpi->kp, regs);
+		__get_cpu_var(current_kprobe) = NULL;
+	}
+	preempt_enable_no_resched();
+}
+
+extern kprobe_opcode_t arch_tmpl_stub_entry;
+extern kprobe_opcode_t arch_tmpl_stub_val;
+extern kprobe_opcode_t arch_tmpl_stub_call;
+extern kprobe_opcode_t arch_tmpl_stub_end;
+
+#define STUB_VAL_IDX \
+	((long)&arch_tmpl_stub_val - (long)&arch_tmpl_stub_entry + 1)
+#define STUB_CALL_IDX \
+	((long)&arch_tmpl_stub_call - (long)&arch_tmpl_stub_entry)
+#define STUB_END_IDX \
+	((long)&arch_tmpl_stub_end - (long)&arch_tmpl_stub_entry)
+
+#define INT3_SIZE 1
+#define JUMP_SIZE 5
+#define ADDR_SIZE 4
+#define STUB_SIZE \
+	(STUB_END_IDX + MAX_KPROBE_LENGTH + JUMP_SIZE)
+
+static __always_inline void __codecopy(void *dest, const void *src, size_t size)
+{
+	memcpy(dest, src, size);
+	flush_icache_range((unsigned long)dest, (unsigned long)dest + size);
+}
+
+/*
+ * Copy post processing instructions
+ * Target instructions MUST be relocatable.
+ */
+int __kprobes arch_prepare_djprobe_instance(struct djprobe_instance *djpi)
+{
+	char *stub;
+
+	djpi->stub.insn = __get_insn_slot(&djprobe_insn_pages);
+	if (djpi->stub.insn == NULL) {
+		return -ENOMEM;
+	}
+	stub = (char *)djpi->stub.insn;
+
+	/* copy arch-dep-instance from template */
+	memcpy(stub, &arch_tmpl_stub_entry, STUB_END_IDX);
+
+	/* set probe information */
+	*((long *)(stub + STUB_VAL_IDX)) = (long)djpi;
+	/* set probe function call */
+	set_jmp_op(stub + STUB_CALL_IDX, djprobe_callback, 1);
+
+	/* copy instructions into the out-of-line buffer */
+	memcpy(stub + STUB_END_IDX, djpi->kp.addr, djpi->kp.length);
+
+	/* set returning jmp instruction at the tail of out-of-line buffer */
+	set_jmp_op(stub + STUB_END_IDX + djpi->kp.length,
+		   (char *)djpi->kp.addr + djpi->kp.length, 0);
+
+	flush_icache_range((unsigned long) stub,
+			   (unsigned long) stub + STUB_END_IDX +
+			   djpi->kp.length + JUMP_SIZE);
+	return 0;
+}
+
+void __kprobes arch_release_djprobe_instance(struct djprobe_instance *djpi)
+{
+	if (djpi->stub.insn)
+		__free_insn_slot(&djprobe_insn_pages, djpi->stub.insn, 0);
+}
+
+void __kprobes arch_preoptimize_djprobe_instance(struct djprobe_instance *djpi)
+{
+	long rel =
+	    (long)(djpi->stub.insn) - ((long)(djpi->kp.addr) + JUMP_SIZE);
+	/* insert the destination address only */
+	__codecopy((void *)((char *)djpi->kp.addr + INT3_SIZE), &rel,
+		   ADDR_SIZE);
+}
+
+void __kprobes arch_optimize_djprobe_instance(struct djprobe_instance *djpi)
+{
+	kprobe_opcode_t op = RELATIVEJUMP_INSTRUCTION;
+	__codecopy(djpi->kp.addr, &op, sizeof(kprobe_opcode_t));
+}
+
+void __kprobes arch_unoptimize_djprobe_instance(struct djprobe_instance *djpi)
+{
+	/* change (the 1st byte of) jump to int3. */
+	arch_arm_kprobe(&djpi->kp);
+	arch_serialize_cpus();
+	/*
+	 * recover the instructions covered by the destination address.
+	 * the int3 will be removed by arch_disarm_kprobe()
+	 */
+	__codecopy((void *)((long)djpi->kp.addr + INT3_SIZE),
+		   (void *)((long)djpi->stub.insn + STUB_END_IDX + INT3_SIZE),
+		   ADDR_SIZE);
+}
+
+/* djprobe handler : switch to a bypass code */
+int __kprobes arch_switch_to_stub(struct djprobe_instance *djpi,
+				  struct pt_regs *regs)
+{
+	regs->eip = (unsigned long)djpi->stub.insn;
+	reset_current_kprobe();
+	preempt_enable_no_resched();
+	return 1;		/* already prepared */
+}
+#endif
+
 int __init arch_init_kprobes(void)
 {
+	djprobe_insn_pages.insn_size = STUB_SIZE;
 	return 0;
 }
Index: linux-2.6.19-rc5-mm2/include/asm-i386/kprobes.h
===================================================================
--- linux-2.6.19-rc5-mm2.orig/include/asm-i386/kprobes.h
+++ linux-2.6.19-rc5-mm2/include/asm-i386/kprobes.h
@@ -35,6 +35,7 @@ struct pt_regs;
 typedef u8 kprobe_opcode_t;
 #define BREAKPOINT_INSTRUCTION	0xcc
 #define RELATIVEJUMP_INSTRUCTION 0xe9
+#define RELATIVECALL_INSTRUCTION 0xe8
 #define MAX_INSN_SIZE 16
 #define MAX_STACK_SIZE 64
 #define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \
@@ -44,9 +45,15 @@ typedef u8 kprobe_opcode_t;

 #define JPROBE_ENTRY(pentry)	(kprobe_opcode_t *)pentry
 #define ARCH_SUPPORTS_KRETPROBES
+#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PM)
+#define ARCH_SUPPORTS_DJPROBES
+#endif
 #define  ARCH_INACTIVE_KPROBE_COUNT 0
 #define flush_insn_slot(p)	do { } while (0)

+#define MAX_KPROBE_LENGTH (5 + MAX_INSN_SIZE - 1)
+#define MIN_KPROBE_LENGTH 5
+
 void arch_remove_kprobe(struct kprobe *p);
 void kretprobe_trampoline(void);

@@ -61,6 +68,10 @@ struct arch_specific_insn {
 	int boostable;
 };

+struct arch_djprobe_stub {
+	kprobe_opcode_t *insn;
+};
+
 struct prev_kprobe {
 	struct kprobe *kp;
 	unsigned long status;