[userspace-rcu.git] / rculfhash-mm-mmap.c

/*
 * rculfhash-mm-mmap.c
 *
 * mmap/reservation based memory management for Lock-Free RCU Hash Table
 *
 * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <unistd.h>
#include <sys/mman.h>
#include "rculfhash-internal.h"

#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS		MAP_ANON
#endif

/* reserve inaccessible memory space without allocation any memory */
static void *memory_map(size_t length)
{
	void *ret = mmap(NULL, length, PROT_NONE,
			MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

	assert(ret != MAP_FAILED);
	return ret;
}

static void memory_unmap(void *ptr, size_t length)
{
	int ret __attribute__((unused));

	ret = munmap(ptr, length);

	assert(ret == 0);
}

static void memory_populate(void *ptr, size_t length)
{
	void *ret __attribute__((unused));

	ret = mmap(ptr, length, PROT_READ | PROT_WRITE,
			MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

	assert(ret == ptr);
}

/*
 * Discard garbage memory and avoid system save it when try to swap it out.
 * Make it still reserved, inaccessible.
 */
static void memory_discard(void *ptr, size_t length)
{
	void *ret __attribute__((unused));

	ret = mmap(ptr, length, PROT_NONE,
			MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

	assert(ret == ptr);
}

static
void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
{
	if (order == 0) {
		if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
			/* small table */
			ht->tbl_mmap = calloc(ht->max_nr_buckets,
					sizeof(*ht->tbl_mmap));
			assert(ht->tbl_mmap);
			return;
		}
		/* large table */
		ht->tbl_mmap = memory_map(ht->max_nr_buckets
			* sizeof(*ht->tbl_mmap));
		memory_populate(ht->tbl_mmap,
			ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
	} else if (order > ht->min_alloc_buckets_order) {
		/* large table */
		unsigned long len = 1UL << (order - 1);

		assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
		memory_populate(ht->tbl_mmap + len,
				len * sizeof(*ht->tbl_mmap));
	}
	/* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
}

/*
 * cds_lfht_free_bucket_table() should be called with decreasing order.
 * When cds_lfht_free_bucket_table(0) is called, it means the whole
 * lfht is destroyed.
 */
static
void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
{
	if (order == 0) {
		if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
			/* small table */
			poison_free(ht->tbl_mmap);
			return;
		}
		/* large table */
		memory_unmap(ht->tbl_mmap,
			ht->max_nr_buckets * sizeof(*ht->tbl_mmap));
	} else if (order > ht->min_alloc_buckets_order) {
		/* large table */
		unsigned long len = 1UL << (order - 1);

		assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
		memory_discard(ht->tbl_mmap + len, len * sizeof(*ht->tbl_mmap));
	}
	/* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
}

static
struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
{
	return &ht->tbl_mmap[index];
}

static
struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
		unsigned long max_nr_buckets)
{
	unsigned long page_bucket_size;

	page_bucket_size = getpagesize() / sizeof(struct cds_lfht_node);
	if (max_nr_buckets <= page_bucket_size) {
		/* small table */
		min_nr_alloc_buckets = max_nr_buckets;
	} else {
		/* large table */
		min_nr_alloc_buckets = max(min_nr_alloc_buckets,
					page_bucket_size);
	}

	return __default_alloc_cds_lfht(
			&cds_lfht_mm_mmap, sizeof(struct cds_lfht),
			min_nr_alloc_buckets, max_nr_buckets);
}

const struct cds_lfht_mm_type cds_lfht_mm_mmap = {
	.alloc_cds_lfht = alloc_cds_lfht,
	.alloc_bucket_table = cds_lfht_alloc_bucket_table,
	.free_bucket_table = cds_lfht_free_bucket_table,
	.bucket_at = bucket_at,
};
Commit	Line	Data
b0b55251 LJ	1	/*
	2	* rculfhash-mm-mmap.c
	3	*
	4	* mmap/reservation based memory management for Lock-Free RCU Hash Table
	5	*
	6	* Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
	7	*
	8	* This library is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License as published by the Free Software Foundation; either
	11	* version 2.1 of the License, or (at your option) any later version.
	12	*
	13	* This library is distributed in the hope that it will be useful,
	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* Lesser General Public License for more details.
	17	*
	18	* You should have received a copy of the GNU Lesser General Public
	19	* License along with this library; if not, write to the Free Software
	20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	*/
	22
	23	#include <unistd.h>
	24	#include <sys/mman.h>
	25	#include "rculfhash-internal.h"
	26
0d0cf93f MD	27	#ifndef MAP_ANONYMOUS
	28	#define MAP_ANONYMOUS MAP_ANON
	29	#endif
	30
b0b55251 LJ	31	/* reserve inaccessible memory space without allocation any memory */
	32	static void *memory_map(size_t length)
	33	{
	34	void *ret = mmap(NULL, length, PROT_NONE,
	35	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
	36
	37	assert(ret != MAP_FAILED);
	38	return ret;
	39	}
	40
	41	static void memory_unmap(void *ptr, size_t length)
	42	{
ae843a64 LJ	43	int ret __attribute__((unused));
	44
	45	ret = munmap(ptr, length);
b0b55251 LJ	46
	47	assert(ret == 0);
	48	}
	49
	50	static void memory_populate(void *ptr, size_t length)
	51	{
ae843a64 LJ	52	void *ret __attribute__((unused));
	53
	54	ret = mmap(ptr, length, PROT_READ \| PROT_WRITE,
b0b55251 LJ	55	MAP_FIXED \| MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
	56
	57	assert(ret == ptr);
	58	}
	59
	60	/*
	61	* Discard garbage memory and avoid system save it when try to swap it out.
	62	* Make it still reserved, inaccessible.
	63	*/
	64	static void memory_discard(void *ptr, size_t length)
	65	{
ae843a64 LJ	66	void *ret __attribute__((unused));
	67
	68	ret = mmap(ptr, length, PROT_NONE,
b0b55251 LJ	69	MAP_FIXED \| MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
	70
	71	assert(ret == ptr);
	72	}
	73
	74	static
	75	void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
	76	{
	77	if (order == 0) {
	78	if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
	79	/* small table */
	80	ht->tbl_mmap = calloc(ht->max_nr_buckets,
	81	sizeof(*ht->tbl_mmap));
	82	assert(ht->tbl_mmap);
	83	return;
	84	}
	85	/* large table */
	86	ht->tbl_mmap = memory_map(ht->max_nr_buckets
	87	* sizeof(*ht->tbl_mmap));
	88	memory_populate(ht->tbl_mmap,
	89	ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
	90	} else if (order > ht->min_alloc_buckets_order) {
	91	/* large table */
	92	unsigned long len = 1UL << (order - 1);
	93
	94	assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
	95	memory_populate(ht->tbl_mmap + len,
	96	len * sizeof(*ht->tbl_mmap));
	97	}
	98	/* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
	99	}
	100
	101	/*
	102	* cds_lfht_free_bucket_table() should be called with decreasing order.
	103	* When cds_lfht_free_bucket_table(0) is called, it means the whole
	104	* lfht is destroyed.
	105	*/
	106	static
	107	void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
	108	{
	109	if (order == 0) {
	110	if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
	111	/* small table */
	112	poison_free(ht->tbl_mmap);
	113	return;
	114	}
	115	/* large table */
	116	memory_unmap(ht->tbl_mmap,
	117	ht->max_nr_buckets * sizeof(*ht->tbl_mmap));
	118	} else if (order > ht->min_alloc_buckets_order) {
	119	/* large table */
	120	unsigned long len = 1UL << (order - 1);
	121
	122	assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
	123	memory_discard(ht->tbl_mmap + len, len * sizeof(*ht->tbl_mmap));
	124	}
	125	/* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
	126	}
	127
	128	static
	129	struct cds_lfht_node bucket_at(struct cds_lfht ht, unsigned long index)
	130	{
	131	return &ht->tbl_mmap[index];
	132	}
133
134	static
135	struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
136	unsigned long max_nr_buckets)
137	{
1228af1c	138	unsigned long page_bucket_size;
b0b55251	139
1228af1c	140	page_bucket_size = getpagesize() / sizeof(struct cds_lfht_node);
b0b55251 LJ	141	if (max_nr_buckets <= page_bucket_size) {
	142	/* small table */
	143	min_nr_alloc_buckets = max_nr_buckets;
	144	} else {
	145	/* large table */
	146	min_nr_alloc_buckets = max(min_nr_alloc_buckets,
	147	page_bucket_size);
	148	}
	149
1228af1c LJ	150	return __default_alloc_cds_lfht(
	151	&cds_lfht_mm_mmap, sizeof(struct cds_lfht),
	152	min_nr_alloc_buckets, max_nr_buckets);
b0b55251 LJ	153	}
	154
	155	const struct cds_lfht_mm_type cds_lfht_mm_mmap = {
	156	.alloc_cds_lfht = alloc_cds_lfht,
	157	.alloc_bucket_table = cds_lfht_alloc_bucket_table,
	158	.free_bucket_table = cds_lfht_free_bucket_table,
	159	.bucket_at = bucket_at,
	160	};