rcuja: implement ja_node_ptr fast path

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/rcuja/rcuja-internal.h b/rcuja/rcuja-internal.h
index 1a3346bbbe96aa4b0ae44896bf51ad476404481f..202f168a62fadcc50d50ca93ecbfcfe4ab724429 100644 (file)
--- a/rcuja/rcuja-internal.h
+++ b/rcuja/rcuja-internal.h
@@ -140,11 +140,36 @@ void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits)
        bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
 }
 
-__attribute__((visibility("protected")))
-unsigned long ja_node_type(struct cds_ja_inode_flag *node);
+/* Hardcoded pool indexes for fast path */
+#define RCU_JA_POOL_IDX_5      5
+#define RCU_JA_POOL_IDX_6      6
+static inline
+struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node)
+{
+       unsigned long v, type_idx;
+
+       if (!node)
+               return NULL;    /* RCU_JA_NULL */
+       v = (unsigned long) node;
+       type_idx = v & JA_TYPE_MASK;
+
+       switch (type_idx) {
+       case RCU_JA_POOL_IDX_5:
+               v &= ~(JA_POOL_1D_MASK | JA_TYPE_MASK);
+               break;
+       case RCU_JA_POOL_IDX_6:
+               v &= ~(JA_POOL_2D_MASK | JA_POOL_1D_MASK | JA_TYPE_MASK);
+               break;
+       default:
+               /* RCU_JA_LINEAR or RCU_JA_PIGEON */
+               v &= JA_PTR_MASK;
+               break;
+       }
+       return (struct cds_ja_node *) v;
+}
 
 __attribute__((visibility("protected")))
-struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node);
+unsigned long ja_node_type(struct cds_ja_inode_flag *node);
 
 __attribute__((visibility("protected")))
 void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,

diff --git a/rcuja/rcuja.c b/rcuja/rcuja.c
index 29b7d8b488d0abdad71fcef77adacb596fe6e65f..c6095956ab14f23e20227f8f05b8b317ce65646f 100644 (file)
--- a/rcuja/rcuja.c
+++ b/rcuja/rcuja.c
@@ -120,7 +120,9 @@ const struct cds_ja_type ja_types[] = {
        { .type_class = RCU_JA_LINEAR, .min_child = 10, .max_child = ja_type_4_max_child, .max_linear_child = ja_type_4_max_linear_child, .order = 7, },
 
        /* Pools may fill sooner than max_child */
+       /* This pool is hardcoded at index 5. See ja_node_ptr(). */
        { .type_class = RCU_JA_POOL, .min_child = 20, .max_child = ja_type_5_max_child, .max_linear_child = ja_type_5_max_linear_child, .order = 8, .nr_pool_order = ja_type_5_nr_pool_order, .pool_size_order = 7, },
+       /* This pool is hardcoded at index 6. See ja_node_ptr(). */
        { .type_class = RCU_JA_POOL, .min_child = 45, .max_child = ja_type_6_max_child, .max_linear_child = ja_type_6_max_linear_child, .order = 9, .nr_pool_order = ja_type_6_nr_pool_order, .pool_size_order = 7, },
 
        /*
@@ -168,7 +170,9 @@ const struct cds_ja_type ja_types[] = {
        { .type_class = RCU_JA_LINEAR, .min_child = 10, .max_child = ja_type_4_max_child, .max_linear_child = ja_type_4_max_linear_child, .order = 8, },
 
        /* Pools may fill sooner than max_child. */
+       /* This pool is hardcoded at index 5. See ja_node_ptr(). */
        { .type_class = RCU_JA_POOL, .min_child = 22, .max_child = ja_type_5_max_child, .max_linear_child = ja_type_5_max_linear_child, .order = 9, .nr_pool_order = ja_type_5_nr_pool_order, .pool_size_order = 8, },
+       /* This pool is hardcoded at index 6. See ja_node_ptr(). */
        { .type_class = RCU_JA_POOL, .min_child = 51, .max_child = ja_type_6_max_child, .max_linear_child = ja_type_6_max_linear_child, .order = 10, .nr_pool_order = ja_type_6_nr_pool_order, .pool_size_order = 8, },
 
        /*
@@ -261,30 +265,6 @@ unsigned long ja_node_type(struct cds_ja_inode_flag *node)
        return type;
 }
 
-struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node)
-{
-       unsigned long type_index = ja_node_type(node);
-       const struct cds_ja_type *type;
-
-       type = &ja_types[type_index];
-       switch (type->type_class) {
-       case RCU_JA_LINEAR:
-       case RCU_JA_PIGEON:     /* fall-through */
-       case RCU_JA_NULL:       /* fall-through */
-       default:                /* fall-through */
-               return _ja_node_mask_ptr(node);
-       case RCU_JA_POOL:
-               switch (type->nr_pool_order) {
-               case 1:
-                       return (struct cds_ja_inode *) (((unsigned long) node) & ~(JA_POOL_1D_MASK | JA_TYPE_MASK));
-               case 2:
-                       return (struct cds_ja_inode *) (((unsigned long) node) & ~(JA_POOL_2D_MASK | JA_POOL_1D_MASK | JA_TYPE_MASK));
-               default:
-                       assert(0);
-               }
-       }
-}
-
 static
 struct cds_ja_inode *alloc_cds_ja_node(struct cds_ja *ja,
                const struct cds_ja_type *ja_type)