[lttng-tools.git] / tests / regression / tools / notification / test_notification_multi_app

#!/bin/bash
#
# Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
#
# SPDX-License-Identifier: LGPL-2.1-only

TEST_DESC="Notification"

CURDIR=$(dirname $0)/
TESTDIR=$CURDIR/../../../

TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)

TESTAPP_PATH="$TESTDIR/utils/testapp"
TESTAPP_NAME="gen-ust-events"
TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
TESTAPP_STATE_FILE="$(mktemp -u)"

NR_ITER=1000
NR_USEC_WAIT=5

SESSION_NAME="my_session"
CHANNEL_NAME="my_channel"


DIR=$(readlink -f $TESTDIR)

PAGE_SIZE=$(getconf PAGE_SIZE)

NUM_TEST_UST=50
NUM_TEST_KERNEL=50
NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))

source $TESTDIR/utils/utils.sh

consumerd_pipe=()
file_sync_after_first_event=$(mktemp -u)

# MUST set TESTDIR before calling those functions
plan_tests $NUM_TESTS

print_test_banner "$TEST_DESC"

app_pids=()

function kernel_event_generator_toogle_state
{
	kernel_event_generator_suspended=$((kernel_event_generator_suspended==0))

}
function kernel_event_generator
{
	state_file=$1
	kernel_event_generator_suspended=0
	trap kernel_event_generator_toogle_state SIGUSR1

	while (true); do
		if [[ $kernel_event_generator_suspended -eq "1" ]]; then
			touch $state_file
			sleep 0.5
		else
			if [[ -f $state_file ]]; then
				rm $state_file 2> /dev/null
			fi
			echo -n "1000" > /proc/lttng-test-filter-event
		fi
	done
}

function ust_event_generator_toogle_state
{
	ust_event_generator_suspended=$((ust_event_generator_suspended==0))

}
function ust_event_generator
{
	state_file=$1
	ust_event_generator_suspended=0
	trap ust_event_generator_toogle_state SIGUSR1
	trap "exit" SIGTERM SIGINT
	while (true); do
		if [[ $ust_event_generator_suspended -eq "1" ]]; then
			touch $state_file
			sleep 0.5
		else
			if [[ -f $state_file ]]; then
				rm $state_file 2> /dev/null
			fi
			taskset  -c 0 $TESTAPP_BIN -i $NR_ITER -w $NR_USEC_WAIT > /dev/null 2>&1
		fi
	done
}

function start_client {
	local pid=-1
	local output_file=$1
	local session_name=$2
	local channel_name=$3
	local domain_type=$4
	local buffer_usage_type=$5
	local buffer_usage_threshold_type=$6
	local buffer_usage_threshold_value=$7
	local nr_expected_notification=$8

	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	pid=$!

	app_pids+=("$pid")
}

function wait_for_message ()
{
	local directory=$1
	local file_pattern=$2
	local message=$3

	for file in $directory/${file_pattern}*; do
		while(true); do
			# Check for "error" message
			grep -q "error:" ${file}
			app_error=$?
			if [ $app_error -eq "0" ] ; then
				# An error occurred
				fail "Waiting message: error logged see file content: ${message}, ${file}"
				return 1
			fi

			grep -q "${message}" ${file}
			if [[ "$?" -ne "0" ]]; then
				# Lookup failed restart loop
				diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
				sleep 0.25
				continue
			fi
			break
		done
	done
	pass "Message received: ${message}"
	return 0
}

function print_errors ()
{
	local directory=$1
	local file_pattern=$2

	for file in $directory/${file_pattern}*; do
		# Check for "error" message
		error_message=$(grep "error:" ${file})
		if [[ "${error_message}x" != "x" ]]; then
			diag "Errors for application ${file}:"
			diag "${error_message}"
		fi
	done
}

function comm_consumerd ()
{
	local message=$1
	local pipe=$2
	echo -ne "${message}" > "${pipe}"
}

function stop_consumerd ()
{
	local pipe=$1
	comm_consumerd "1" "$pipe"
}

function resume_consumerd ()
{
	local pipe=$1
	comm_consumerd "\0" "$pipe"
}

function test_multi_app ()
{
	local domain_type=$1
	local event_generator_pid=$2

	local app_pids=()
	local low_output_file_pattern="low_app_output_file_"
	local high_output_file_pattern="high_app_output_file_"
	local output_dir=$(mktemp -d)

	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")

	local nr_notification_expected=5
	local nr_client_app=50
	local domain_string=""
	local event_name=""

	case $domain_type in
		ust)
			domain_string=LTTNG_DOMAIN_UST
			event_name="tp:tptest"
			;;
		kernel)
			domain_string=LTTNG_DOMAIN_KERNEL
			event_name="lttng_test_filter_event"
			;;
		*)
			fail "Invalid domain type"
			exit 1
			;;
	esac

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	consumerd_pipe=()
	for f in "$testpoint_pipe_path"*; do
		consumerd_pipe+=("$f")
	done

	for (( i = 0; i < $nr_client_app; i++ )); do
		low_app_output_file=$output_dir/${low_output_file_pattern}${i}
		high_app_output_file=$output_dir/${high_output_file_pattern}${i}
		start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected
		start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected
	done

	wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
	wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"

	# Test notification reception
	for (( i = 0; i < $nr_notification_expected; i++ )); do

		# Stop consumerd consumption to force high notification
		start_lttng_tracing_ok $SESSION_NAME
		for pipe in "${consumerd_pipe[@]}"; do
			stop_consumerd "${pipe}"
		done

		wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"

		# Put application in suspend mode to prevent double low
		# notification and synchronize on state file.
		kill -s SIGUSR1 $event_generator_pid
		while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
			sleep 0.5
		done

		# Resume consumerd
		for pipe in "${consumerd_pipe[@]}"; do
			resume_consumerd "${pipe}"
		done
		# Stop tracing forcing full buffer consumption
		stop_lttng_tracing_ok $SESSION_NAME

		# Check for notifications reception
		wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
		ret=$?
		ok $ret "Notifications $i received"
		if [[ $ret -ne "0" ]]; then
			# Error occurred bail out
			break;
		fi

		# Put application in active mode and synchronize on state file.
		kill -s SIGUSR1 $event_generator_pid
		while [ -f "${TESTAPP_STATE_FILE}" ]; do
			sleep 0.5
		done
	done

	wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for low notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		print_errors $output_dir "${low_output_file_pattern}"
	fi

	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		print_errors $output_dir "${high_output_file_pattern}"
	fi

	rm -rf $output_dir

	destroy_lttng_session_ok $SESSION_NAME
	stop_lttng_sessiond

	for pipe in "${consumerd_pipe[@]}"; do
		rm -rf "${pipe}"
	done
}

function test_multi_app_ust ()
{
	diag "Multi client app UST notification"
	ust_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_multi_app ust $generator_pid

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
}

function test_multi_app_kernel ()
{
	diag "Multi client app kernel notification"
	modprobe lttng-test

	kernel_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_multi_app kernel $generator_pid


	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

	rmmod lttng-test
}

function test_on_register_evaluation_ust ()
{
	diag "On register notification UST"

	# Start app in infinite loop
	ust_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_on_register_evaluation ust $generator_pid

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

}

function test_on_register_evaluation_kernel()
{
	diag "On register notification kernel"

	modprobe lttng-test

	kernel_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_on_register_evaluation kernel $generator_pid


	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

	rmmod lttng-test
}

function test_on_register_evaluation ()
{
	local domain_type=$1
	local event_generator_pid=$2

	local app_pids=()
	local high_output_file_pattern="high_app_output_file_on_register_evaluation"

	local output_dir=$(mktemp -d)
	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	local domain_string=""
	local event_name=""

	case $domain_type in
		ust)
			domain_string=LTTNG_DOMAIN_UST
			event_name="tp:tptest"
			;;
		kernel)
			domain_string=LTTNG_DOMAIN_KERNEL
			event_name="lttng_test_filter_event"
			;;
		*)
			fail "Invalid domain type"
			exit 1
			;;
	esac

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	consumerd_pipe=()
	for f in "$testpoint_pipe_path"*; do
		consumerd_pipe+=("$f")
	done

	high_app_output_file=${high_output_file_pattern}.first_receiver
	high_app_output_path=$output_dir/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1

	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"

	# Stop consumerd consumption to force high notification
	start_lttng_tracing_ok $SESSION_NAME

	for pipe in "${consumerd_pipe[@]}"; do
		stop_consumerd "${pipe}"
	done

	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"

	# Start a second receiver, the receiver should receive a high
	# notification on subscription
	high_app_output_file=${high_output_file_pattern}.second_receiver
	high_app_output_path=$output_dir/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"

	# Resume consumerd
	for pipe in "${consumerd_pipe[@]}"; do
		resume_consumerd "${pipe}"
	done

	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		# Keep the file
		print_errors "${high_output_file_pattern}"
	fi

	rm -rf $output_dir

	destroy_lttng_session_ok $SESSION_NAME
	stop_lttng_sessiond

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null

	for pipe in "${consumerd_pipe[@]}"; do
		rm -rf "${pipe}"
	done

}


TESTS=(
	test_multi_app_ust
	test_on_register_evaluation_ust
)

if [ "$(id -u)" == "0" ]; then
	validate_lttng_modules_present
	TESTS+=(
	test_multi_app_kernel
	test_on_register_evaluation_kernel
)
else
        skip 0 "Root access is needed. Skipping all kernel multi-app notification tests." $NUM_TEST_KERNEL
fi


for fct_test in ${TESTS[@]};
do
	TRACE_PATH=$(mktemp -d)

	${fct_test}
	if [ $? -ne 0 ]; then
		break;
	fi

	# Only delete if successful
	rm -rf $TRACE_PATH
done
Commit	Line	Data
	1	#!/bin/bash
	2	#
	3	# Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
	4	#
	5	# SPDX-License-Identifier: LGPL-2.1-only
	6
	7	TEST_DESC="Notification"
	8
	9	CURDIR=$(dirname $0)/
	10	TESTDIR=$CURDIR/../../../
	11
	12	TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)
	13
	14	TESTAPP_PATH="$TESTDIR/utils/testapp"
	15	TESTAPP_NAME="gen-ust-events"
	16	TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
	17	TESTAPP_STATE_FILE="$(mktemp -u)"
	18
	19	NR_ITER=1000
	20	NR_USEC_WAIT=5
	21
	22	SESSION_NAME="my_session"
	23	CHANNEL_NAME="my_channel"
	24
	25
	26	DIR=$(readlink -f $TESTDIR)
	27
	28	PAGE_SIZE=$(getconf PAGE_SIZE)
	29
	30	NUM_TEST_UST=50
	31	NUM_TEST_KERNEL=50
	32	NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))
	33
	34	source $TESTDIR/utils/utils.sh
	35
	36	consumerd_pipe=()
	37	file_sync_after_first_event=$(mktemp -u)
	38
	39	# MUST set TESTDIR before calling those functions
	40	plan_tests $NUM_TESTS
	41
	42	print_test_banner "$TEST_DESC"
	43
	44	app_pids=()
	45
	46	function kernel_event_generator_toogle_state
	47	{
	48	kernel_event_generator_suspended=$((kernel_event_generator_suspended==0))
	49
	50	}
	51	function kernel_event_generator
	52	{
	53	state_file=$1
	54	kernel_event_generator_suspended=0
	55	trap kernel_event_generator_toogle_state SIGUSR1
	56
	57	while (true); do
	58	if [[ $kernel_event_generator_suspended -eq "1" ]]; then
	59	touch $state_file
	60	sleep 0.5
	61	else
	62	if [[ -f $state_file ]]; then
	63	rm $state_file 2> /dev/null
	64	fi
	65	echo -n "1000" > /proc/lttng-test-filter-event
	66	fi
	67	done
	68	}
	69
	70	function ust_event_generator_toogle_state
	71	{
	72	ust_event_generator_suspended=$((ust_event_generator_suspended==0))
	73
	74	}
	75	function ust_event_generator
	76	{
	77	state_file=$1
	78	ust_event_generator_suspended=0
	79	trap ust_event_generator_toogle_state SIGUSR1
	80	trap "exit" SIGTERM SIGINT
	81	while (true); do
	82	if [[ $ust_event_generator_suspended -eq "1" ]]; then
	83	touch $state_file
	84	sleep 0.5
	85	else
	86	if [[ -f $state_file ]]; then
	87	rm $state_file 2> /dev/null
	88	fi
	89	taskset -c 0 $TESTAPP_BIN -i $NR_ITER -w $NR_USEC_WAIT > /dev/null 2>&1
	90	fi
	91	done
	92	}
	93
	94	function start_client {
	95	local pid=-1
	96	local output_file=$1
	97	local session_name=$2
	98	local channel_name=$3
	99	local domain_type=$4
	100	local buffer_usage_type=$5
	101	local buffer_usage_threshold_type=$6
	102	local buffer_usage_threshold_value=$7
	103	local nr_expected_notification=$8
	104
	105	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	106	pid=$!
	107
	108	app_pids+=("$pid")
	109	}
	110
	111	function wait_for_message ()
	112	{
	113	local directory=$1
	114	local file_pattern=$2
	115	local message=$3
	116
	117	for file in $directory/${file_pattern}*; do
	118	while(true); do
	119	# Check for "error" message
	120	grep -q "error:" ${file}
	121	app_error=$?
	122	if [ $app_error -eq "0" ] ; then
	123	# An error occurred
	124	fail "Waiting message: error logged see file content: ${message}, ${file}"
	125	return 1
	126	fi
	127
	128	grep -q "${message}" ${file}
	129	if [[ "$?" -ne "0" ]]; then
	130	# Lookup failed restart loop
	131	diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
	132	sleep 0.25
	133	continue
	134	fi
	135	break
	136	done
	137	done
	138	pass "Message received: ${message}"
	139	return 0
	140	}
	141
	142	function print_errors ()
	143	{
	144	local directory=$1
	145	local file_pattern=$2
	146
	147	for file in $directory/${file_pattern}*; do
	148	# Check for "error" message
	149	error_message=$(grep "error:" ${file})
	150	if [[ "${error_message}x" != "x" ]]; then
	151	diag "Errors for application ${file}:"
	152	diag "${error_message}"
	153	fi
	154	done
	155	}
	156
	157	function comm_consumerd ()
	158	{
	159	local message=$1
	160	local pipe=$2
	161	echo -ne "${message}" > "${pipe}"
	162	}
	163
	164	function stop_consumerd ()
	165	{
	166	local pipe=$1
	167	comm_consumerd "1" "$pipe"
	168	}
	169
	170	function resume_consumerd ()
	171	{
	172	local pipe=$1
	173	comm_consumerd "\0" "$pipe"
	174	}
	175
	176	function test_multi_app ()
	177	{
	178	local domain_type=$1
	179	local event_generator_pid=$2
	180
	181	local app_pids=()
	182	local low_output_file_pattern="low_app_output_file_"
	183	local high_output_file_pattern="high_app_output_file_"
	184	local output_dir=$(mktemp -d)
	185
	186	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
	187	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	188
	189	local nr_notification_expected=5
	190	local nr_client_app=50
	191	local domain_string=""
	192	local event_name=""
	193
	194	case $domain_type in
	195	ust)
	196	domain_string=LTTNG_DOMAIN_UST
	197	event_name="tp:tptest"
	198	;;
	199	kernel)
	200	domain_string=LTTNG_DOMAIN_KERNEL
	201	event_name="lttng_test_filter_event"
	202	;;
	203	*)
	204	fail "Invalid domain type"
	205	exit 1
	206	;;
	207	esac
	208
	209	# Setup
	210	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	211	start_lttng_sessiond
	212
	213	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	214	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	215	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
	216
	217	# Fetch consumerd testpoint pipe information
	218	# This is needed since the testpoint create a pipe with the consumer type suffixed
	219	consumerd_pipe=()
	220	for f in "$testpoint_pipe_path"*; do
	221	consumerd_pipe+=("$f")
	222	done
	223
	224	for (( i = 0; i < $nr_client_app; i++ )); do
	225	low_app_output_file=$output_dir/${low_output_file_pattern}${i}
	226	high_app_output_file=$output_dir/${high_output_file_pattern}${i}
	227	start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected
	228	start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected
	229	done
	230
	231	wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
	232	wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"
	233
	234	# Test notification reception
	235	for (( i = 0; i < $nr_notification_expected; i++ )); do
	236
	237	# Stop consumerd consumption to force high notification
	238	start_lttng_tracing_ok $SESSION_NAME
	239	for pipe in "${consumerd_pipe[@]}"; do
	240	stop_consumerd "${pipe}"
	241	done
	242
	243	wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"
	244
	245	# Put application in suspend mode to prevent double low
	246	# notification and synchronize on state file.
	247	kill -s SIGUSR1 $event_generator_pid
	248	while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
	249	sleep 0.5
	250	done
	251
	252	# Resume consumerd
	253	for pipe in "${consumerd_pipe[@]}"; do
	254	resume_consumerd "${pipe}"
	255	done
	256	# Stop tracing forcing full buffer consumption
	257	stop_lttng_tracing_ok $SESSION_NAME
	258
	259	# Check for notifications reception
	260	wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
	261	ret=$?
	262	ok $ret "Notifications $i received"
	263	if [[ $ret -ne "0" ]]; then
	264	# Error occurred bail out
	265	break;
	266	fi
	267
	268	# Put application in active mode and synchronize on state file.
	269	kill -s SIGUSR1 $event_generator_pid
	270	while [ -f "${TESTAPP_STATE_FILE}" ]; do
	271	sleep 0.5
	272	done
	273	done
	274
	275	wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
	276	ret=$?
	277	ok $ret "Application for low notification terminated normally"
	278	if [[ $ret -ne "0" ]]; then
	279	print_errors $output_dir "${low_output_file_pattern}"
	280	fi
	281
	282	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	283	ret=$?
	284	ok $ret "Application for high notification terminated normally"
	285	if [[ $ret -ne "0" ]]; then
	286	print_errors $output_dir "${high_output_file_pattern}"
	287	fi
	288
	289	rm -rf $output_dir
	290
	291	destroy_lttng_session_ok $SESSION_NAME
	292	stop_lttng_sessiond
	293
	294	for pipe in "${consumerd_pipe[@]}"; do
	295	rm -rf "${pipe}"
	296	done
	297	}
	298
	299	function test_multi_app_ust ()
	300	{
	301	diag "Multi client app UST notification"
	302	ust_event_generator $TESTAPP_STATE_FILE &
	303	local generator_pid=$!
	304
	305	test_multi_app ust $generator_pid
	306
	307	kill -s SIGTERM $generator_pid 2> /dev/null
	308	wait $generator_pid 2> /dev/null
	309	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	310	}
	311
	312	function test_multi_app_kernel ()
	313	{
	314	diag "Multi client app kernel notification"
	315	modprobe lttng-test
	316
	317	kernel_event_generator $TESTAPP_STATE_FILE &
	318	local generator_pid=$!
	319
	320	test_multi_app kernel $generator_pid
	321
	322
	323	kill -s SIGTERM $generator_pid 2> /dev/null
	324	wait $generator_pid 2> /dev/null
	325	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	326
	327	rmmod lttng-test
	328	}
	329
	330	function test_on_register_evaluation_ust ()
	331	{
	332	diag "On register notification UST"
	333
	334	# Start app in infinite loop
	335	ust_event_generator $TESTAPP_STATE_FILE &
	336	local generator_pid=$!
	337
	338	test_on_register_evaluation ust $generator_pid
	339
	340	kill -s SIGTERM $generator_pid 2> /dev/null
	341	wait $generator_pid 2> /dev/null
	342	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	343
	344	}
	345
	346	function test_on_register_evaluation_kernel()
	347	{
	348	diag "On register notification kernel"
	349
	350	modprobe lttng-test
	351
	352	kernel_event_generator $TESTAPP_STATE_FILE &
	353	local generator_pid=$!
	354
	355	test_on_register_evaluation kernel $generator_pid
	356
	357
	358	kill -s SIGTERM $generator_pid 2> /dev/null
	359	wait $generator_pid 2> /dev/null
	360	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	361
	362	rmmod lttng-test
	363	}
	364
	365	function test_on_register_evaluation ()
	366	{
	367	local domain_type=$1
	368	local event_generator_pid=$2
	369
	370	local app_pids=()
	371	local high_output_file_pattern="high_app_output_file_on_register_evaluation"
	372
	373	local output_dir=$(mktemp -d)
	374	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
	375	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	376	local domain_string=""
	377	local event_name=""
	378
	379	case $domain_type in
	380	ust)
	381	domain_string=LTTNG_DOMAIN_UST
	382	event_name="tp:tptest"
	383	;;
	384	kernel)
	385	domain_string=LTTNG_DOMAIN_KERNEL
	386	event_name="lttng_test_filter_event"
	387	;;
	388	*)
	389	fail "Invalid domain type"
	390	exit 1
	391	;;
	392	esac
	393
	394	# Setup
	395	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	396	start_lttng_sessiond
	397
	398	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	399	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	400	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
	401
	402	# Fetch consumerd testpoint pipe information
	403	# This is needed since the testpoint create a pipe with the consumer type suffixed
	404	consumerd_pipe=()
	405	for f in "$testpoint_pipe_path"*; do
	406	consumerd_pipe+=("$f")
	407	done
	408
	409	high_app_output_file=${high_output_file_pattern}.first_receiver
	410	high_app_output_path=$output_dir/${high_app_output_file}
	411	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	412
	413	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	414
	415	# Stop consumerd consumption to force high notification
	416	start_lttng_tracing_ok $SESSION_NAME
	417
	418	for pipe in "${consumerd_pipe[@]}"; do
	419	stop_consumerd "${pipe}"
	420	done
	421
	422	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
	423
	424	# Start a second receiver, the receiver should receive a high
	425	# notification on subscription
	426	high_app_output_file=${high_output_file_pattern}.second_receiver
	427	high_app_output_path=$output_dir/${high_app_output_file}
	428	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	429	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	430	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
	431
	432	# Resume consumerd
	433	for pipe in "${consumerd_pipe[@]}"; do
	434	resume_consumerd "${pipe}"
	435	done
	436
	437	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	438	ret=$?
	439	ok $ret "Application for high notification terminated normally"
	440	if [[ $ret -ne "0" ]]; then
	441	# Keep the file
	442	print_errors "${high_output_file_pattern}"
	443	fi
	444
	445	rm -rf $output_dir
	446
	447	destroy_lttng_session_ok $SESSION_NAME
	448	stop_lttng_sessiond
	449
	450	kill -s SIGTERM $generator_pid 2> /dev/null
	451	wait $generator_pid 2> /dev/null
	452
	453	for pipe in "${consumerd_pipe[@]}"; do
	454	rm -rf "${pipe}"
	455	done
	456
	457	}
	458
	459
	460	TESTS=(
	461	test_multi_app_ust
	462	test_on_register_evaluation_ust
	463	)
	464
	465	if [ "$(id -u)" == "0" ]; then
	466	validate_lttng_modules_present
	467	TESTS+=(
	468	test_multi_app_kernel
	469	test_on_register_evaluation_kernel
	470	)
	471	else
	472	skip 0 "Root access is needed. Skipping all kernel multi-app notification tests." $NUM_TEST_KERNEL
	473	fi
	474
	475
	476	for fct_test in ${TESTS[@]};
	477	do
	478	TRACE_PATH=$(mktemp -d)
	479
	480	${fct_test}
	481	if [ $? -ne 0 ]; then
	482	break;
	483	fi
	484
	485	# Only delete if successful
	486	rm -rf $TRACE_PATH
	487	done