X-Git-Url: http://git.lttng.org./?a=blobdiff_plain;f=libringbuffer%2Ffrontend_internal.h;h=6e9d8be2f1c988362de1e21200a8968d462f7005;hb=f574bfb4d81b26d3cfc0243868d138e33470a950;hp=10d6c4db789bbf498e3188a1a339c5e94223653f;hpb=d2fe4771f15b6e50969bea1d7fcd74edca307de3;p=lttng-ust.git diff --git a/libringbuffer/frontend_internal.h b/libringbuffer/frontend_internal.h index 10d6c4db..6e9d8be2 100644 --- a/libringbuffer/frontend_internal.h +++ b/libringbuffer/frontend_internal.h @@ -1,42 +1,24 @@ -#ifndef _LTTNG_RING_BUFFER_FRONTEND_INTERNAL_H -#define _LTTNG_RING_BUFFER_FRONTEND_INTERNAL_H - /* - * libringbuffer/frontend_internal.h - * - * Ring Buffer Library Synchronization Header (internal helpers). + * SPDX-License-Identifier: (LGPL-2.1-only or GPL-2.0-only) * * Copyright (C) 2005-2012 Mathieu Desnoyers * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; only - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * - * Author: - * Mathieu Desnoyers + * Ring Buffer Library Synchronization Header (internal helpers). * * See ring_buffer_frontend.c for more information on wait-free algorithms. - * - * Dual LGPL v2.1/GPL v2 license. */ +#ifndef _LTTNG_RING_BUFFER_FRONTEND_INTERNAL_H +#define _LTTNG_RING_BUFFER_FRONTEND_INTERNAL_H + #include #include #include +#include #include -#include +#include +#include "ringbuffer-config.h" #include "backend_types.h" #include "frontend_types.h" #include "shm.h" @@ -45,7 +27,8 @@ /* buf_trunc mask selects only the buffer number. */ static inline -unsigned long buf_trunc(unsigned long offset, struct channel *chan) +unsigned long buf_trunc(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return offset & ~(chan->backend.buf_size - 1); @@ -53,35 +36,40 @@ unsigned long buf_trunc(unsigned long offset, struct channel *chan) /* Select the buffer number value (counter). */ static inline -unsigned long buf_trunc_val(unsigned long offset, struct channel *chan) +unsigned long buf_trunc_val(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return buf_trunc(offset, chan) >> chan->backend.buf_size_order; } /* buf_offset mask selects only the offset within the current buffer. */ static inline -unsigned long buf_offset(unsigned long offset, struct channel *chan) +unsigned long buf_offset(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return offset & (chan->backend.buf_size - 1); } /* subbuf_offset mask selects the offset within the current subbuffer. */ static inline -unsigned long subbuf_offset(unsigned long offset, struct channel *chan) +unsigned long subbuf_offset(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return offset & (chan->backend.subbuf_size - 1); } /* subbuf_trunc mask selects the subbuffer number. */ static inline -unsigned long subbuf_trunc(unsigned long offset, struct channel *chan) +unsigned long subbuf_trunc(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return offset & ~(chan->backend.subbuf_size - 1); } /* subbuf_align aligns the offset to the next subbuffer. */ static inline -unsigned long subbuf_align(unsigned long offset, struct channel *chan) +unsigned long subbuf_align(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return (offset + chan->backend.subbuf_size) & ~(chan->backend.subbuf_size - 1); @@ -89,7 +77,8 @@ unsigned long subbuf_align(unsigned long offset, struct channel *chan) /* subbuf_index returns the index of the current subbuffer within the buffer. */ static inline -unsigned long subbuf_index(unsigned long offset, struct channel *chan) +unsigned long subbuf_index(unsigned long offset, + struct lttng_ust_lib_ring_buffer_channel *chan) { return buf_offset(offset, chan) >> chan->backend.subbuf_size_order; } @@ -157,17 +146,21 @@ int last_tsc_overflow(const struct lttng_ust_lib_ring_buffer_config *config, } #endif +__attribute__((visibility("hidden"))) extern -int lib_ring_buffer_reserve_slow(struct lttng_ust_lib_ring_buffer_ctx *ctx); +int lib_ring_buffer_reserve_slow(struct lttng_ust_lib_ring_buffer_ctx *ctx, + void *client_ctx); +__attribute__((visibility("hidden"))) extern void lib_ring_buffer_switch_slow(struct lttng_ust_lib_ring_buffer *buf, enum switch_mode mode, struct lttng_ust_shm_handle *handle); +__attribute__((visibility("hidden"))) void lib_ring_buffer_check_deliver_slow(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan, + struct lttng_ust_lib_ring_buffer_channel *chan, unsigned long offset, unsigned long commit_count, unsigned long idx, @@ -178,7 +171,7 @@ void lib_ring_buffer_check_deliver_slow(const struct lttng_ust_lib_ring_buffer_c static inline void lib_ring_buffer_reserve_push_reader(struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan, + struct lttng_ust_lib_ring_buffer_channel *chan, unsigned long offset) { unsigned long consumed_old, consumed_new; @@ -204,10 +197,43 @@ void lib_ring_buffer_reserve_push_reader(struct lttng_ust_lib_ring_buffer *buf, consumed_new) != consumed_old)); } +/* + * Move consumed position to the beginning of subbuffer in which the + * write offset is. Should only be used on ring buffers that are not + * actively being written into, because clear_reader does not take into + * account the commit counters when moving the consumed position, which + * can make concurrent trace producers or consumers observe consumed + * position further than the write offset, which breaks ring buffer + * algorithm guarantees. + */ +static inline +void lib_ring_buffer_clear_reader(struct lttng_ust_lib_ring_buffer *buf, + struct lttng_ust_shm_handle *handle) +{ + struct lttng_ust_lib_ring_buffer_channel *chan; + const struct lttng_ust_lib_ring_buffer_config *config; + unsigned long offset, consumed_old, consumed_new; + + chan = shmp(handle, buf->backend.chan); + if (!chan) + return; + config = &chan->backend.config; + + do { + offset = v_read(config, &buf->offset); + consumed_old = uatomic_read(&buf->consumed); + CHAN_WARN_ON(chan, (long) (subbuf_trunc(offset, chan) + - subbuf_trunc(consumed_old, chan)) + < 0); + consumed_new = subbuf_trunc(offset, chan); + } while (caa_unlikely(uatomic_cmpxchg(&buf->consumed, consumed_old, + consumed_new) != consumed_old)); +} + static inline int lib_ring_buffer_pending_data(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan) + struct lttng_ust_lib_ring_buffer_channel *chan) { return !!subbuf_offset(v_read(config, &buf->offset), chan); } @@ -229,10 +255,11 @@ unsigned long lib_ring_buffer_get_data_size(const struct lttng_ust_lib_ring_buff static inline int lib_ring_buffer_reserve_committed(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan, + struct lttng_ust_lib_ring_buffer_channel *chan, struct lttng_ust_shm_handle *handle) { unsigned long offset, idx, commit_count; + struct commit_counters_hot *cc_hot; CHAN_WARN_ON(chan, config->alloc != RING_BUFFER_ALLOC_PER_CPU); CHAN_WARN_ON(chan, config->sync != RING_BUFFER_SYNC_PER_CPU); @@ -248,7 +275,10 @@ int lib_ring_buffer_reserve_committed(const struct lttng_ust_lib_ring_buffer_con do { offset = v_read(config, &buf->offset); idx = subbuf_index(offset, chan); - commit_count = v_read(config, &shmp_index(handle, buf->commit_hot, idx)->cc); + cc_hot = shmp_index(handle, buf->commit_hot, idx); + if (caa_unlikely(!cc_hot)) + return 0; + commit_count = v_read(config, &cc_hot->cc); } while (offset != v_read(config, &buf->offset)); return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) @@ -265,7 +295,7 @@ int lib_ring_buffer_reserve_committed(const struct lttng_ust_lib_ring_buffer_con static inline void lib_ring_buffer_check_deliver(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan, + struct lttng_ust_lib_ring_buffer_channel *chan, unsigned long offset, unsigned long commit_count, unsigned long idx, @@ -293,7 +323,7 @@ void lib_ring_buffer_check_deliver(const struct lttng_ust_lib_ring_buffer_config static inline void lib_ring_buffer_write_commit_counter(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer *buf, - struct channel *chan, + struct lttng_ust_lib_ring_buffer_channel *chan, unsigned long buf_offset, unsigned long commit_count, struct lttng_ust_shm_handle *handle, @@ -318,14 +348,18 @@ void lib_ring_buffer_write_commit_counter(const struct lttng_ust_lib_ring_buffer v_set(config, &cc_hot->seq, commit_count); } +__attribute__((visibility("hidden"))) extern int lib_ring_buffer_create(struct lttng_ust_lib_ring_buffer *buf, struct channel_backend *chanb, int cpu, struct lttng_ust_shm_handle *handle, struct shm_object *shmobj); + +__attribute__((visibility("hidden"))) extern void lib_ring_buffer_free(struct lttng_ust_lib_ring_buffer *buf, struct lttng_ust_shm_handle *handle); /* Keep track of trap nesting inside ring buffer code */ +__attribute__((visibility("hidden"))) extern DECLARE_URCU_TLS(unsigned int, lib_ring_buffer_nesting); #endif /* _LTTNG_RING_BUFFER_FRONTEND_INTERNAL_H */