This is the mail archive of the systemtap@sourceware.org mailing list for the systemtap project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[RFC][PATCH 2/4][kprobe](djprobe) Direct jump optimized kprobes core patch


Hi,

This patch is the architecture independent part of the djprobe.

In this patch, both of jump inserting and buffer releasing are
executed by the commit_kprobes() batch function. This batch
function freezes processes while inserting jumps and releasing
buffers on the preemptible kernel. So we can do it safely.
(This safety check routine is provided by the previous patch.
http://sources.redhat.com/ml/systemtap/2006-q4/msg00453.html)
In this patch, commit_kprobes() doesn't do anything to
unoptimized kprobes. But I think this method can be used for
speeding up unregistration of kprobes.

Definitely, the handlers of kprobes whose length member is set
are activated/deactivated right after invoked register/
unregister_kprobe() functions. Before invoking commit_kprobes(),
these handlers are invoked by normal kprobes.

Thanks,

-- 
Masami HIRAMATSU
Linux Technology Center
Hitachi, Ltd., Systems Development Laboratory
E-mail: masami.hiramatsu.pt@hitachi.com

---
 include/linux/kprobes.h |   44 ++++++++
 kernel/kprobes.c        |  263 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 290 insertions(+), 17 deletions(-)
Index: linux-2.6.19-rc5-mm2/include/linux/kprobes.h
===================================================================
--- linux-2.6.19-rc5-mm2.orig/include/linux/kprobes.h
+++ linux-2.6.19-rc5-mm2/include/linux/kprobes.h
@@ -28,6 +28,9 @@
  * 2005-May	Hien Nguyen <hien@us.ibm.com> and Jim Keniston
  *		<jkenisto@us.ibm.com>  and Prasanna S Panchamukhi
  *		<prasanna@in.ibm.com> added function-return probes.
+ * 2006-Nov	Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> and
+ *		Satoshi Oshima <soshima@redhat.com> added direct-jump
+ *		optimization (a.k.a. djprobe).
  */
 #include <linux/list.h>
 #include <linux/notifier.h>
@@ -83,6 +86,9 @@ struct kprobe {
 	/* Offset into the symbol */
 	unsigned int offset;

+	/* Length of the replaced instructions for direct jump optimization */
+	unsigned int length;
+
 	/* Called before addr is executed. */
 	kprobe_pre_handler_t pre_handler;

@@ -173,6 +179,39 @@ extern void show_registers(struct pt_reg
 extern kprobe_opcode_t *get_insn_slot(void);
 extern void free_insn_slot(kprobe_opcode_t *slot, int dirty);
 extern void kprobes_inc_nmissed_count(struct kprobe *p);
+extern kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_page_list *pages);
+extern void __free_insn_slot(struct kprobe_insn_page_list *pages,
+			     kprobe_opcode_t * slot, int dirty);
+extern int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
+
+#ifdef ARCH_SUPPORTS_DJPROBES
+/*
+ * Direct jump optimized probe
+ * Note:
+ * User can optimize his kprobe to reduce "break" overhead. He must
+ * analyze target instructions, count its length and confirm those
+ * instructions are relocatable. After that, set the length member of
+ * the kprobe to that length. If unsure, clear length member.
+ */
+struct djprobe_instance {
+	struct kprobe kp;
+	struct list_head list;	/* list for commitment */
+	struct arch_djprobe_stub stub;
+};
+
+/* architecture dependent functions for direct jump optimization */
+extern int arch_prepare_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_release_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_preoptimize_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_optimize_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_unoptimize_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_serialize_cpus(void);
+extern int arch_switch_to_stub(struct djprobe_instance *djpi,
+			       struct pt_regs * regs);
+#else
+#define MIN_KPROBE_LENGTH 0
+#define MAX_KPROBE_LENGTH 0
+#endif

 /* Get the kprobe at this addr (if any) - called with preemption disabled */
 struct kprobe *get_kprobe(void *addr);
@@ -196,6 +235,7 @@ static inline struct kprobe_ctlblk *get_

 int register_kprobe(struct kprobe *p);
 void unregister_kprobe(struct kprobe *p);
+int commit_kprobes(void);
 int setjmp_pre_handler(struct kprobe *, struct pt_regs *);
 int longjmp_break_handler(struct kprobe *, struct pt_regs *);
 int register_jprobe(struct jprobe *p);
@@ -226,6 +266,10 @@ static inline int register_kprobe(struct
 static inline void unregister_kprobe(struct kprobe *p)
 {
 }
+static inline int commit_kprobes(void)
+{
+	return -ENOSYS;
+}
 static inline int register_jprobe(struct jprobe *p)
 {
 	return -ENOSYS;
Index: linux-2.6.19-rc5-mm2/kernel/kprobes.c
===================================================================
--- linux-2.6.19-rc5-mm2.orig/kernel/kprobes.c
+++ linux-2.6.19-rc5-mm2/kernel/kprobes.c
@@ -30,6 +30,9 @@
  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  *		<prasanna@in.ibm.com> added function-return probes.
+ * 2006-Nov	Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> and
+ *		Satoshi Oshima <soshima@redhat.com> added direct-jump
+ *		optimization (a.k.a. djprobe).
  */
 #include <linux/kprobes.h>
 #include <linux/hash.h>
@@ -304,7 +307,7 @@ struct kprobe __kprobes *get_kprobe(void
  * Aggregate handlers for multiple kprobes support - these handlers
  * take care of invoking the individual kprobe handlers on p->list
  */
-static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
+int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 {
 	struct kprobe *kp;

@@ -363,11 +366,26 @@ static int __kprobes aggr_break_handler(
 	return ret;
 }

+static int __kprobes djprobe_pre_handler(struct kprobe *kp,
+					 struct pt_regs *regs);
+
+/* return true if the kprobe is an aggregator */
+static inline int is_aggr_kprobe(struct kprobe *p)
+{
+	return p->pre_handler == aggr_pre_handler ||
+		p->pre_handler == djprobe_pre_handler;
+}
+/* return true if the kprobe is a djprobe */
+static inline int is_djprobe(struct kprobe *p)
+{
+	return p->pre_handler == djprobe_pre_handler;
+}
+
 /* Walks the list and increments nmissed count for multiprobe case */
 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
 {
 	struct kprobe *kp;
-	if (p->pre_handler != aggr_pre_handler) {
+	if (!is_aggr_kprobe(p)) {
 		p->nmissed++;
 	} else {
 		list_for_each_entry_rcu(kp, &p->list, list)
@@ -482,6 +500,7 @@ static inline void copy_kprobe(struct kp
 {
 	memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 	memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
+	p->length = old_p->length;
 }

 /*
@@ -534,7 +553,10 @@ static int __kprobes register_aggr_kprob
 	int ret = 0;
 	struct kprobe *ap;

-	if (old_p->pre_handler == aggr_pre_handler) {
+	if (is_aggr_kprobe(old_p)) {
+		if (is_djprobe(old_p) &&
+		    (p->break_handler || p->post_handler))
+			return -EEXIST;
 		copy_kprobe(old_p, p);
 		ret = add_new_kprobe(old_p, p);
 	} else {
@@ -556,6 +578,209 @@ static int __kprobes in_kprobes_function
 	return 0;
 }

+/* Called with kprobe_mutex held */
+static int __kprobes __register_kprobe_core(struct kprobe *p)
+{
+	int ret;
+	if ((ret = arch_prepare_kprobe(p)) != 0)
+		return ret;
+
+	INIT_HLIST_NODE(&p->hlist);
+	hlist_add_head_rcu(&p->hlist,
+			   &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
+
+	if (atomic_add_return(1, &kprobe_count) == \
+				(ARCH_INACTIVE_KPROBE_COUNT + 1))
+		register_page_fault_notifier(&kprobe_page_fault_nb);
+
+	arch_arm_kprobe(p);
+	return 0;
+}
+
+#ifdef ARCH_SUPPORTS_DJPROBES
+static LIST_HEAD(registering_list);
+static LIST_HEAD(unregistering_list);
+
+/* Switch to stub buffer : this handler is invoked before inserting a jump */
+static int __kprobes djprobe_pre_handler(struct kprobe *kp,
+					 struct pt_regs *regs)
+{
+	struct djprobe_instance *djpi;
+	djpi = container_of(kp, struct djprobe_instance, kp);
+	return arch_switch_to_stub(djpi, regs);
+}
+
+static int __kprobes detect_probe_collision(struct kprobe *p)
+{
+	int i;
+	struct kprobe *list_p;
+	/* check collision with other optimized kprobes */
+	for (i = 1; i < MAX_KPROBE_LENGTH; i++) {
+		list_p = get_kprobe((char *)p->addr - i);
+		if (list_p) {
+			if (list_p->length > i) {
+				/* other djprobes partially covered */
+				return -EEXIST;
+			}
+			break;
+		}
+	}
+	/* check collision with other kprobes */
+	for (i = 1; i < p->length; i++) {
+		if (get_kprobe((char *)p->addr + i)) {
+			p->length = 0;	/* not optimizable */
+			break;
+		}
+	}
+	return 0;
+}
+
+static int __kprobes register_djprobe(struct kprobe *p)
+{
+	int ret = 0;
+	struct djprobe_instance *djpi;
+
+	if (p->break_handler || p->post_handler)
+		return -EINVAL;
+
+	/* allocate a new instance */
+	djpi = kzalloc(sizeof(struct djprobe_instance), GFP_KERNEL);
+	if (djpi == NULL) {
+		return -ENOMEM;
+	}
+	djpi->kp.addr = p->addr;
+	djpi->kp.length = p->length;
+	djpi->kp.pre_handler = djprobe_pre_handler;
+	INIT_LIST_HEAD(&djpi->list);
+	INIT_LIST_HEAD(&djpi->kp.list);
+
+	/* allocate and initialize stub */
+	if ((ret = arch_prepare_djprobe_instance(djpi)) < 0) {
+		kfree(djpi);
+		goto out;
+	}
+
+	/* register as a kprobe */
+	if ((ret = __register_kprobe_core(&djpi->kp)) != 0) {
+		arch_release_djprobe_instance(djpi);
+		kfree(djpi);
+		goto out;
+	}
+	list_add(&djpi->list, &registering_list);
+
+	register_aggr_kprobe(&djpi->kp, p);
+out:
+	return ret;
+}
+
+static void __kprobes unoptimize_djprobe(struct kprobe *p)
+{
+	struct djprobe_instance *djpi;
+	djpi = container_of(p, struct djprobe_instance, kp);
+	if (!list_empty(&djpi->list)) {
+		/* not committed yet */
+		list_del_init(&djpi->list);
+	} else {
+		/* unoptimize probe point */
+		arch_unoptimize_djprobe_instance(djpi);
+	}
+}
+
+static void __kprobes unregister_djprobe(struct kprobe *p)
+{
+	/* Djprobes will be freed by
+	 * commit_kprobes.
+	 */
+	struct djprobe_instance *djpi;
+	djpi = container_of(p, struct djprobe_instance, kp);
+	list_add(&djpi->list, &unregistering_list);
+}
+
+/* Called with kprobe_mutex held */
+static int __kprobes __commit_djprobes(void)
+{
+	struct djprobe_instance *djpi;
+
+	if (list_empty(&registering_list) &&
+	    list_empty(&unregistering_list)) {
+		return 0;
+	}
+
+	/* ensure safety */
+	if (check_safety() != 0) return -EAGAIN;
+
+	/* optimize probes */
+	if (!list_empty(&registering_list)) {
+		list_for_each_entry(djpi, &registering_list, list) {
+			arch_preoptimize_djprobe_instance(djpi);
+		}
+		arch_serialize_cpus();
+		while (!list_empty(&registering_list)) {
+			djpi = list_entry(registering_list.next,
+					  struct djprobe_instance, list);
+			list_del_init(&djpi->list);
+			arch_optimize_djprobe_instance(djpi);
+		}
+	}
+	/* release code buffer */
+	while (!list_empty(&unregistering_list)) {
+		djpi = list_entry(unregistering_list.next,
+				  struct djprobe_instance, list);
+		list_del(&djpi->list);
+		arch_release_djprobe_instance(djpi);
+		kfree(djpi);
+	}
+	return 0;
+}
+
+/*
+ * Commit to optimize kprobes and remove optimized kprobes.
+ */
+int __kprobes commit_kprobes(void)
+{
+	int ret = 0;
+	mutex_lock(&kprobe_mutex);
+	ret = __commit_djprobes();
+	mutex_unlock(&kprobe_mutex);
+	return ret;
+}
+
+#else	/* ARCH_SUPPORTS_DJPROBES */
+
+static int __kprobes djprobe_pre_handler(struct kprobe *kp,
+					 struct pt_regs *regs)
+{
+	return 0;
+}
+
+static inline int detect_probe_collision(struct kprobe *p)
+{
+	return 0;
+}
+
+int __kprobes commit_kprobes(void)
+{
+	return 0;
+}
+
+static inline int register_djprobe(struct kprobe *p)
+{
+	p->length = 0;	/* Disable direct jump optimization */
+	return __register_kprobe_core(p);
+}
+
+static inline void unoptimize_djprobe(struct kprobe *p)
+{
+	return;
+}
+
+static inline void unregister_djprobe(struct kprobe *p)
+{
+	kfree(p);
+}
+
+#endif	/* ARCH_SUPPORTS_DJPROBES */
+
 static int __kprobes __register_kprobe(struct kprobe *p,
 	unsigned long called_from)
 {
@@ -574,7 +799,8 @@ static int __kprobes __register_kprobe(s
 		kprobe_lookup_name(p->symbol_name, p->addr);
 	}

-	if (!p->addr)
+	if (!p->addr || (p->length &&
+	    (p->length < MIN_KPROBE_LENGTH || p->length > MAX_KPROBE_LENGTH)))
 		return -EINVAL;
 	p->addr = (kprobe_opcode_t *)(((char *)p->addr)+ p->offset);

@@ -608,19 +834,14 @@ static int __kprobes __register_kprobe(s
 		goto out;
 	}

-	if ((ret = arch_prepare_kprobe(p)) != 0)
+	if ((ret = detect_probe_collision(p)) != 0)
 		goto out;

-	INIT_HLIST_NODE(&p->hlist);
-	hlist_add_head_rcu(&p->hlist,
-		       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
-
-	if (atomic_add_return(1, &kprobe_count) == \
-				(ARCH_INACTIVE_KPROBE_COUNT + 1))
-		register_page_fault_notifier(&kprobe_page_fault_nb);
-
-	arch_arm_kprobe(p);
-
+	if (p->length) {
+		ret = register_djprobe(p);
+		goto out;
+	}
+	ret = __register_kprobe_core(p);
 out:
 	mutex_unlock(&kprobe_mutex);

@@ -656,10 +877,13 @@ void __kprobes unregister_kprobe(struct
 		return;
 	}
 valid_p:
-	if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
+	if ((old_p == p) || (is_aggr_kprobe(old_p) &&
 		(p->list.next == &old_p->list) &&
 		(p->list.prev == &old_p->list))) {
 		/* Only probe on the hash list */
+		if (is_djprobe(old_p)) {
+			unoptimize_djprobe(old_p);
+		}
 		arch_disarm_kprobe(p);
 		hlist_del_rcu(&old_p->hlist);
 		cleanup_p = 1;
@@ -678,7 +902,11 @@ valid_p:
 	if (cleanup_p) {
 		if (p != old_p) {
 			list_del_rcu(&p->list);
-			kfree(old_p);
+			if (is_djprobe(old_p)) {
+				unregister_djprobe(old_p);
+			} else {
+				kfree(old_p);
+			}
 		}
 		arch_remove_kprobe(p);
 	} else {
@@ -836,6 +1064,7 @@ __initcall(init_kprobes);

 EXPORT_SYMBOL_GPL(register_kprobe);
 EXPORT_SYMBOL_GPL(unregister_kprobe);
+EXPORT_SYMBOL_GPL(commit_kprobes);
 EXPORT_SYMBOL_GPL(register_jprobe);
 EXPORT_SYMBOL_GPL(unregister_jprobe);
 EXPORT_SYMBOL_GPL(jprobe_return);


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]