| 1 | #ifndef _URCU_RCUJA_INTERNAL_H |
| 2 | #define _URCU_RCUJA_INTERNAL_H |
| 3 | |
| 4 | /* |
| 5 | * rcuja/rcuja-internal.h |
| 6 | * |
| 7 | * Userspace RCU library - RCU Judy Array Internal Header |
| 8 | * |
| 9 | * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
| 10 | * |
| 11 | * This library is free software; you can redistribute it and/or |
| 12 | * modify it under the terms of the GNU Lesser General Public |
| 13 | * License as published by the Free Software Foundation; either |
| 14 | * version 2.1 of the License, or (at your option) any later version. |
| 15 | * |
| 16 | * This library is distributed in the hope that it will be useful, |
| 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 19 | * Lesser General Public License for more details. |
| 20 | * |
| 21 | * You should have received a copy of the GNU Lesser General Public |
| 22 | * License along with this library; if not, write to the Free Software |
| 23 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 24 | */ |
| 25 | |
| 26 | #define _GNU_SOURCE |
| 27 | #include <pthread.h> |
| 28 | #include <stdio.h> |
| 29 | #include <inttypes.h> |
| 30 | #include <unistd.h> |
| 31 | #include <urcu/rculfhash.h> |
| 32 | |
| 33 | /* |
| 34 | * Number of least significant pointer bits reserved to represent the |
| 35 | * child type. |
| 36 | */ |
| 37 | #define JA_TYPE_BITS 3 |
| 38 | #define JA_TYPE_MAX_NR (1UL << JA_TYPE_BITS) |
| 39 | #define JA_TYPE_MASK (JA_TYPE_MAX_NR - 1) |
| 40 | #define JA_PTR_MASK (~JA_TYPE_MASK) |
| 41 | |
| 42 | #define JA_ENTRY_PER_NODE 256UL |
| 43 | #define JA_LOG2_BITS_PER_BYTE 3U |
| 44 | #define JA_BITS_PER_BYTE (1U << JA_LOG2_BITS_PER_BYTE) |
| 45 | |
| 46 | #define JA_POOL_1D_MASK ((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS) |
| 47 | #define JA_POOL_2D_MASK (JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE) |
| 48 | |
| 49 | #define JA_MAX_DEPTH 9 /* Maximum depth, including leafs */ |
| 50 | |
| 51 | /* |
| 52 | * Entry for NULL node is at index 8 of the table. It is never encoded |
| 53 | * in flags. |
| 54 | */ |
| 55 | #define NODE_INDEX_NULL 8 |
| 56 | |
| 57 | /* |
| 58 | * Number of removals needed on a fallback node before we try to shrink |
| 59 | * it. |
| 60 | */ |
| 61 | #define JA_FALLBACK_REMOVAL_COUNT 8 |
| 62 | |
| 63 | /* Never declared. Opaque type used to store flagged node pointers. */ |
| 64 | struct cds_ja_inode_flag; |
| 65 | struct cds_ja_inode; |
| 66 | |
| 67 | /* |
| 68 | * Shadow node contains mutex and call_rcu head associated with a node. |
| 69 | */ |
| 70 | struct cds_ja_shadow_node { |
| 71 | struct cds_lfht_node ht_node; /* hash table node */ |
| 72 | struct cds_ja_inode_flag *node_flag; /* reverse mapping and hash table key */ |
| 73 | /* |
| 74 | * mutual exclusion on all nodes belonging to the same tree |
| 75 | * position (e.g. both nodes before and after recompaction |
| 76 | * use the same lock). |
| 77 | */ |
| 78 | pthread_mutex_t *lock; |
| 79 | unsigned int nr_child; /* number of children in node */ |
| 80 | struct rcu_head head; /* for deferred node and shadow node reclaim */ |
| 81 | int fallback_removal_count; /* removals left keeping fallback */ |
| 82 | int level; /* level in the tree */ |
| 83 | struct cds_ja *ja; /* toplevel judy array */ |
| 84 | }; |
| 85 | |
| 86 | struct cds_ja { |
| 87 | struct cds_ja_inode_flag *root; |
| 88 | unsigned int tree_depth; |
| 89 | uint64_t key_max; |
| 90 | /* |
| 91 | * We use a hash table to associate node keys to their |
| 92 | * respective shadow node. This helps reducing lookup hot path |
| 93 | * cache footprint, especially for very small nodes. |
| 94 | */ |
| 95 | struct cds_lfht *ht; |
| 96 | unsigned long nr_fallback; /* Number of fallback nodes used */ |
| 97 | }; |
| 98 | |
| 99 | static inline |
| 100 | struct cds_ja_inode_flag *ja_node_flag(struct cds_ja_inode *node, |
| 101 | unsigned long type) |
| 102 | { |
| 103 | assert(type < (1UL << JA_TYPE_BITS)); |
| 104 | return (struct cds_ja_inode_flag *) (((unsigned long) node) | type); |
| 105 | } |
| 106 | |
| 107 | static inline |
| 108 | struct cds_ja_inode_flag *ja_node_flag_pool_1d(struct cds_ja_inode *node, |
| 109 | unsigned long type, unsigned long bitsel) |
| 110 | { |
| 111 | assert(type < (1UL << JA_TYPE_BITS)); |
| 112 | assert(bitsel < JA_BITS_PER_BYTE); |
| 113 | return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel << JA_TYPE_BITS) | type); |
| 114 | } |
| 115 | |
| 116 | static inline |
| 117 | unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node) |
| 118 | { |
| 119 | return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS; |
| 120 | } |
| 121 | |
| 122 | static inline |
| 123 | void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits) |
| 124 | { |
| 125 | bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE); |
| 126 | bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS; |
| 127 | } |
| 128 | |
| 129 | __attribute__((visibility("protected"))) |
| 130 | unsigned long ja_node_type(struct cds_ja_inode_flag *node); |
| 131 | |
| 132 | __attribute__((visibility("protected"))) |
| 133 | struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node); |
| 134 | |
| 135 | __attribute__((visibility("protected"))) |
| 136 | void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node, |
| 137 | struct cds_ja_inode_flag *node_flag, |
| 138 | void (*free_node_cb)(struct rcu_head *head)); |
| 139 | |
| 140 | __attribute__((visibility("protected"))) |
| 141 | struct cds_ja_shadow_node *rcuja_shadow_lookup_lock(struct cds_lfht *ht, |
| 142 | struct cds_ja_inode_flag *node_flag); |
| 143 | |
| 144 | __attribute__((visibility("protected"))) |
| 145 | void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node); |
| 146 | |
| 147 | __attribute__((visibility("protected"))) |
| 148 | struct cds_ja_shadow_node *rcuja_shadow_set(struct cds_lfht *ht, |
| 149 | struct cds_ja_inode_flag *new_node_flag, |
| 150 | struct cds_ja_shadow_node *inherit_from, |
| 151 | struct cds_ja *ja); |
| 152 | |
| 153 | /* rcuja_shadow_clear flags */ |
| 154 | enum { |
| 155 | RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0), |
| 156 | RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1), |
| 157 | }; |
| 158 | |
| 159 | __attribute__((visibility("protected"))) |
| 160 | int rcuja_shadow_clear(struct cds_lfht *ht, |
| 161 | struct cds_ja_inode_flag *node_flag, |
| 162 | struct cds_ja_shadow_node *shadow_node, |
| 163 | unsigned int flags); |
| 164 | |
| 165 | __attribute__((visibility("protected"))) |
| 166 | void rcuja_shadow_prune(struct cds_lfht *ht, |
| 167 | unsigned int flags, |
| 168 | void (*free_node_cb)(struct rcu_head *head)); |
| 169 | |
| 170 | __attribute__((visibility("protected"))) |
| 171 | struct cds_lfht *rcuja_create_ht(const struct rcu_flavor_struct *flavor); |
| 172 | |
| 173 | __attribute__((visibility("protected"))) |
| 174 | int rcuja_delete_ht(struct cds_lfht *ht); |
| 175 | |
| 176 | //#define DEBUG |
| 177 | |
| 178 | #ifdef __linux__ |
| 179 | #include <syscall.h> |
| 180 | #endif |
| 181 | |
| 182 | #if defined(_syscall0) |
| 183 | _syscall0(pid_t, gettid) |
| 184 | #elif defined(__NR_gettid) |
| 185 | static inline pid_t gettid(void) |
| 186 | { |
| 187 | return syscall(__NR_gettid); |
| 188 | } |
| 189 | #else |
| 190 | #warning "use pid as tid" |
| 191 | static inline pid_t gettid(void) |
| 192 | { |
| 193 | return getpid(); |
| 194 | } |
| 195 | #endif |
| 196 | |
| 197 | #ifdef DEBUG |
| 198 | #define dbg_printf(fmt, args...) \ |
| 199 | fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \ |
| 200 | (unsigned long) gettid(), __func__, \ |
| 201 | __FILE__, __LINE__, ## args) |
| 202 | #else |
| 203 | #define dbg_printf(fmt, args...) \ |
| 204 | do { \ |
| 205 | /* do nothing but check printf format */ \ |
| 206 | if (0) \ |
| 207 | fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \ |
| 208 | (unsigned long) gettid(), __func__, \ |
| 209 | __FILE__, __LINE__, ## args); \ |
| 210 | } while (0) |
| 211 | #endif |
| 212 | |
| 213 | #endif /* _URCU_RCUJA_INTERNAL_H */ |