From: Mathieu Desnoyers Date: Sun, 29 May 2011 15:33:53 +0000 (-0400) Subject: RCU search fix X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=230dd2881e8e46ec8d008961f446b724a60203bc;p=userspace-rcu.git RCU search fix Fix removal by using the decay scheme for the whole branch from z->right to y (inclusively) when transplanting y up to z. Signed-off-by: Mathieu Desnoyers --- diff --git a/tests/test_urcu_rbtree.c b/tests/test_urcu_rbtree.c index 587db7b..757b940 100644 --- a/tests/test_urcu_rbtree.c +++ b/tests/test_urcu_rbtree.c @@ -228,17 +228,18 @@ void *thr_reader(void *_count) cmm_smp_mb(); for (;;) { - rcu_read_lock(); for (i = 0; i < global_items; i++) { - node = rcu_rbtree_search(&rbtree, rbtree.root, + rcu_read_lock(); + node = rcu_rbtree_search(&rbtree, + rcu_dereference(rbtree.root), global_key[i]); assert(!rcu_rbtree_is_nil(node)); + rcu_read_unlock(); } debug_yield_read(); if (unlikely(rduration)) loop_sleep(rduration); - rcu_read_unlock(); nr_reads++; if (unlikely(!test_duration_read())) break; @@ -278,18 +279,21 @@ void *thr_writer(void *_count) for (;;) { rcu_copy_mutex_lock(); - rcu_read_lock(); for (i = 0; i < NR_RAND; i++) { node = rbtree_alloc(); key[i] = (void *)(unsigned long)(rand() % 2048); node->key = key[i]; + rcu_read_lock(); rcu_rbtree_insert(&rbtree, node); + rcu_read_unlock(); } + rcu_copy_mutex_unlock(); if (unlikely(wduration)) loop_sleep(wduration); + rcu_copy_mutex_lock(); for (i = 0; i < NR_RAND; i++) { #if 0 node = rcu_rbtree_min(rbtree, rbtree->root); @@ -306,13 +310,14 @@ void *thr_writer(void *_count) } printf("\n"); #endif + rcu_read_lock(); node = rcu_rbtree_search(&rbtree, rbtree.root, key[i]); assert(!rcu_rbtree_is_nil(node)); rcu_rbtree_remove(&rbtree, node); + rcu_read_unlock(); call_rcu(&node->head, rbtree_free); } - rcu_read_unlock(); rcu_copy_mutex_unlock(); nr_writes++; if (unlikely(!test_duration_write())) diff --git a/urcu-rbtree.c b/urcu-rbtree.c index 7e1b333..b6e2e13 100644 --- a/urcu-rbtree.c +++ b/urcu-rbtree.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,35 @@ #define RBTREE_RCU_SUPPORT_ROTATE_RIGHT #define RBTREE_RCU_SUPPORT_TRANSPLANT +#ifdef EXTRA_DEBUG +static pthread_mutex_t test_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t outer_mutex = PTHREAD_MUTEX_INITIALIZER; + +static +void lock_outer_mutex(void) +{ + pthread_mutex_lock(&outer_mutex); +} + +static +void unlock_outer_mutex(void) +{ + pthread_mutex_unlock(&outer_mutex); +} + +static +void lock_test_mutex(void) +{ + pthread_mutex_lock(&test_mutex); +} + +static +void unlock_test_mutex(void) +{ + pthread_mutex_unlock(&test_mutex); +} +#endif + static void set_decay(struct rcu_rbtree_node *x, struct rcu_rbtree_node *xc) { @@ -139,6 +169,7 @@ struct rcu_rbtree_node* rcu_rbtree_search(struct rcu_rbtree *rbtree, x = rcu_dereference(x); while (!rcu_rbtree_is_nil(x) && k != x->key) { + usleep(10); if (rbtree->comp(k, x->key) < 0) x = rcu_dereference(x->left); else @@ -147,6 +178,52 @@ struct rcu_rbtree_node* rcu_rbtree_search(struct rcu_rbtree *rbtree, return x; } +static +struct rcu_rbtree_node *rcu_rbtree_min_dup_decay(struct rcu_rbtree *rbtree, + struct rcu_rbtree_node *x, + struct rcu_rbtree_node **zr) +{ + struct rcu_rbtree_node *xl; + + x = rcu_dereference(x); + + if (rcu_rbtree_is_nil(x)) { + *zr = x; + return x; + } else + *zr = x = dup_decay_node(rbtree, x); + + while (!rcu_rbtree_is_nil(xl = rcu_dereference(x->left))) { + x = dup_decay_node(rbtree, xl); + x->p = get_decay(x->p); + x->p->left = get_decay(x->p->left); + } + return x; +} + +static +struct rcu_rbtree_node *rcu_rbtree_min_update_decay(struct rcu_rbtree *rbtree, + struct rcu_rbtree_node *x) +{ + struct rcu_rbtree_node *xl; + + x = rcu_dereference(x); + + if (rcu_rbtree_is_nil(x)) + return x; + else { + x->right->p = get_decay(x->right->p); + x->left->p = get_decay(x->left->p); + } + + while (!rcu_rbtree_is_nil(xl = rcu_dereference(x->left))) { + x = xl; + xl->right->p = get_decay(xl->right->p); + xl->left->p = get_decay(xl->left->p); + } + return x; +} + struct rcu_rbtree_node *rcu_rbtree_min(struct rcu_rbtree *rbtree, struct rcu_rbtree_node *x) { @@ -317,6 +394,7 @@ void left_rotate(struct rcu_rbtree *rbtree, { struct rcu_rbtree_node *y; + lock_test_mutex(); y = x->right; x->right = y->left; if (!rcu_rbtree_is_nil(y->left)) { @@ -336,6 +414,7 @@ void left_rotate(struct rcu_rbtree *rbtree, y->left = x; x->pos = IS_LEFT; x->p = y; + unlock_test_mutex(); } #endif @@ -414,6 +493,7 @@ void right_rotate(struct rcu_rbtree *rbtree, { struct rcu_rbtree_node *y; + lock_test_mutex(); y = x->left; x->left = y->right; if (!rcu_rbtree_is_nil(y->right)) { @@ -433,6 +513,7 @@ void right_rotate(struct rcu_rbtree *rbtree, y->right = x; x->pos = IS_RIGHT; x->p = y; + unlock_test_mutex(); } #endif @@ -604,6 +685,7 @@ void rcu_rbtree_transplant(struct rcu_rbtree *rbtree, { dbg_printf("transplant %p\n", v->key); + lock_test_mutex(); if (rcu_rbtree_is_nil(u->p)) rbtree->root = v; else if (u == u->p->left) { @@ -614,6 +696,7 @@ void rcu_rbtree_transplant(struct rcu_rbtree *rbtree, v->pos = IS_RIGHT; } v->p = u->p; + unlock_test_mutex(); } #endif @@ -720,23 +803,39 @@ void rcu_rbtree_remove_nonil(struct rcu_rbtree *rbtree, assert(!is_decay(y->p)); x = y->right; assert(!is_decay(x)); - if (y->p == z) + if (y->p == z) { + y = dup_decay_node(rbtree, y); x->p = y; - else { - rcu_rbtree_transplant(rbtree, y, y->right); + y->left = z->left; + rcu_rbtree_transplant(rbtree, z, y); + } else { + struct rcu_rbtree_node *oy_right, *z_right; + + /* + * Need to make sure y is always visible by readers. + */ + y = rcu_rbtree_min_dup_decay(rbtree, z->right, &z_right); assert(!is_decay(y)); assert(!is_decay(z)); - assert(!is_decay(z->right)); - y->right = z->right; + oy_right = y->right; + y->right = z_right; y->right->p = y; + assert(!is_decay(z->left)); + y->left = z->left; + assert(!is_decay(oy_right)); + rcu_rbtree_transplant(rbtree, y, oy_right); + rcu_rbtree_transplant(rbtree, z, y); + /* Update children */ + (void) rcu_rbtree_min_update_decay(rbtree, y->right); } - rcu_rbtree_transplant(rbtree, z, y); y = get_decay(y); assert(!is_decay(z)); assert(!is_decay(z->left)); - y->left = z->left; - y->left->p = y; y->color = z->color; + y->left->p = y; + y->right->p = get_decay(y->right->p); + assert(!is_decay(y->left)); + assert(!is_decay(y->right)); } int rcu_rbtree_remove(struct rcu_rbtree *rbtree,