Observed issue
==============
lttng-ivc observed unexpected path generated for streamed snapshot:
joraj-alpa/
test-
20190319-120000-
20210113-110101/
snapshot-1-
20210113-110102-0/
* ust/
* ust/
pid/
app-ust-
2362198-
20210113-110101/
channel_0
channel_1
metadata
channel_2
channel_3
"ust" the domain subdir is present two-time instead off only one time.
The same problem is seen for kernel snapshots.
Cause
=====
Based on dissection the problem was introduced by commit
5da88b0f58d7f838068037ea449ddfb25d3e85ad [1]
For snapshots, the consumer output object of the snapshot output and
information is fetched from the *original* consumer output.
(code block around src/bin/lttng-sessiond/cmd.c:4748)
The snapshot consumer output does not contains the necessary information
to populate domain_subdir on copy (domain_subdir is '\0').
This would lead to a len evaluation of 1 for the consumer_path_offset in
setup_channel_trace_path:70. This would end up not "skipping" the "ust"
and "kernel" part of the path.
Solution
========
Part of the solution is to copy the domain_subdir from the original
session consumer output to the snapshot output.
Still, the problem was still present since that now that the
domain_subdir was not "\0", the value was suffixed to the passed
session_path. In the snapshot code path, "ust/" and "kernel/" were
already present in the session_path passed to setup_channel_trace_path.
A quick modification at the caller level in the snapshot code path fixes
the issues once and for all.
Tests
=======
The test suit is augmented for certain key tests to validate the
complete path of a trace.
Path validation is based on crude bash pattern matching.
Known drawbacks
=========
None
References
==========
[1] https://github.com/lttng/lttng-tools/commit/
5da88b0f58d7f838068037ea449ddfb25d3e85ad
Signed-off-by: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>
Signed-off-by: Jérémie Galarneau <jeremie.galarneau@efficios.com>
Change-Id: Ie7cd2d5471ee3a942fa511e2f4cab09e3aa499e4
consumer_copy_output(snapshot_output->consumer);
strcpy(snapshot_kernel_consumer_output->chunk_path,
snapshot_chunk_name);
+
+ /* Copy the original domain subdir. */
+ strcpy(snapshot_kernel_consumer_output->domain_subdir,
+ original_kernel_consumer_output->domain_subdir);
+
ret = consumer_copy_sockets(snapshot_kernel_consumer_output,
original_kernel_consumer_output);
if (ret < 0) {
consumer_copy_output(snapshot_output->consumer);
strcpy(snapshot_ust_consumer_output->chunk_path,
snapshot_chunk_name);
+
+ /* Copy the original domain subdir. */
+ strcpy(snapshot_ust_consumer_output->domain_subdir,
+ original_ust_consumer_output->domain_subdir);
+
ret = consumer_copy_sockets(snapshot_ust_consumer_output,
original_ust_consumer_output);
if (ret < 0) {
}
trace_path = setup_channel_trace_path(ksess->consumer,
- DEFAULT_KERNEL_TRACE_DIR, &consumer_path_offset);
+ "", &consumer_path_offset);
if (!trace_path) {
status = LTTNG_ERR_INVALID;
goto error;
memset(pathname, 0, sizeof(pathname));
ret = snprintf(pathname, sizeof(pathname),
- DEFAULT_UST_TRACE_DIR "/" DEFAULT_UST_TRACE_UID_PATH,
+ DEFAULT_UST_TRACE_UID_PATH,
reg->uid, reg->bits_per_long);
if (ret < 0) {
PERROR("snprintf snapshot path");
/* Add the UST default trace dir to path. */
memset(pathname, 0, sizeof(pathname));
- ret = snprintf(pathname, sizeof(pathname), DEFAULT_UST_TRACE_DIR "/%s",
+ ret = snprintf(pathname, sizeof(pathname), "%s",
ua_sess->path);
if (ret < 0) {
status = LTTNG_ERR_INVALID;
CURDIR=$(dirname $0)/
TESTDIR=$CURDIR/../..
-NUM_TESTS=20
+NUM_TESTS=21
TESTCMD="/bin/true"
eval ${TESTCMD}
stop_lttng_tracing_ok
+ validate_trace_path_kernel "$TRACE_PATH" ""
validate_trace "sched_switch" $TRACE_PATH
validate_trace "sched_process_exit" $TRACE_PATH
validate_trace "sched_process_fork" $TRACE_PATH
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=37
+NUM_TESTS=42
source "$TESTDIR/utils/utils.sh"
destroy_lttng_session_ok $session_name
# validate test
+ validate_trace_path_ust_uid_network "$TRACE_PATH" "" "$base_path"
if validate_trace $EVENT_NAME "$TRACE_PATH/$HOSTNAME/$base_path"; then
# only delete if successful
rm -rf "$TRACE_PATH"
destroy_lttng_session_ok $session_name
# validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "" "snapshot-1" 0 "$base_path"
if validate_trace $EVENT_NAME "$TRACE_PATH/$HOSTNAME/$base_path"; then
# only delete if successful
rm -rf "$TRACE_PATH"
destroy_lttng_session_ok $session_name
# validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "" "snapshot-0" 0 "$base_path"
if validate_trace $EVENT_NAME "$TRACE_PATH/$HOSTNAME/$base_path"; then
# only delete if successful
rm -rf "$TRACE_PATH"
destroy_lttng_session_ok $session_name
# validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "" "snapshot-1" 0 "$base_path"
if validate_trace $EVENT_NAME "$TRACE_PATH/$HOSTNAME/$base_path"; then
# only delete if successful
rm -rf "$TRACE_PATH"
destroy_lttng_session_ok $session_name
# validate test
+ validate_trace_path_ust_uid_network "$TRACE_PATH" "" "$base_path"
if validate_trace $EVENT_NAME "$TRACE_PATH/$HOSTNAME/$base_path"; then
# only delete if successful
rm -rf "$TRACE_PATH"
DIR=$(readlink -f $TESTDIR)
-NUM_TESTS=10
+NUM_TESTS=11
source $TESTDIR/utils/utils.sh
clean_live_tracing
+ validate_trace_path_kernel_network "$TRACE_PATH" "$SESSION_NAME"
validate_trace $EVENT_NAME $TRACE_PATH
rm -rf $TRACE_PATH
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=2060
+NUM_TESTS=2061
source $TESTDIR/utils/utils.sh
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_kernel_snapshot "$TRACE_PATH" "" "snapshot-1" 0 ""
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
rm -rf $TRACE_PATH
- else
- break
fi
}
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=61
+NUM_TESTS=66
source $TESTDIR/utils/utils.sh
start_lttng_tracing_ok $SESSION_NAME
lttng_snapshot_record $SESSION_NAME
stop_lttng_tracing_ok $SESSION_NAME
+ validate_trace_path_kernel_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-1" $(( i - 1 )) ""
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-1*
if [ $? -ne 0 ]; then
return 1
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=75
+NUM_TESTS=85
source $TESTDIR/utils/utils.sh
lttng_snapshot_record $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-1*
if [ $? -ne 0 ]; then
stop_test_apps
lttng_snapshot_record $SESSION_NAME
# Validate test with the next ID since a del output was done prior.
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-2" 1
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-2*
if [ $? -ne 0 ]; then
stop_test_apps
stop_lttng_tracing_ok $SESSION_NAME
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-1*
out=$?
stop_lttng_tracing_ok $SESSION_NAME
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-1*
out=$?
if ls $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/$name* &> /dev/null; then
ok 0 "Custom name snapshot exists"
# Validate test
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "$name" 0
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/$name-*
out=$?
else
create_lttng_session_no_output $SESSION_NAME
enable_lttng_mmap_overwrite_ust_channel $SESSION_NAME $CHANNEL_NAME
enable_ust_lttng_event_ok $SESSION_NAME $EVENT_NAME $CHANNEL_NAME
- snapshot_add_output $SESSION_NAME "net://localhost" $name
+ snapshot_add_output $SESSION_NAME "net://localhost"
start_test_app
- for i in {1..5};
+ for i in {0..4};
do
start_lttng_tracing_ok $SESSION_NAME
lttng_snapshot_record $SESSION_NAME
stop_lttng_tracing_ok $SESSION_NAME
+ validate_trace_path_ust_uid_snapshot_network "$TRACE_PATH" "$SESSION_NAME" "snapshot-1" $i
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*/snapshot-1*
if [ $? -ne 0 ]; then
return 1
TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
APPS_PID=
-NUM_TESTS=100
+NUM_TESTS=105
TRACE_PATH=$(mktemp -d)
fi
NR_SNAPSHOT=$1
-NUM_TESTS=$(($NUM_TESTS + ($NR_SNAPSHOT * 2)))
+NUM_TESTS=$(($NUM_TESTS + ($NR_SNAPSHOT * 3)))
function start_test_app()
{
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" 0
validate_trace $LM_EVENT $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
destroy_lttng_session_ok $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" 0
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
rm -rf $TRACE_PATH/snapshot/* 2>/dev/null
lttng_snapshot_record $SESSION_NAME
# Validate test
+ validate_trace_path_ust_uid_snapshot "$TRACE_PATH" "" "snapshot-1" $((i - 1))
validate_trace $EVENT_NAME $TRACE_PATH/
if [ $? -eq 0 ]; then
# Only delete if successful
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=10
+NUM_TESTS=11
source $TESTDIR/utils/utils.sh
${fct_test}
# Validate test
+ validate_trace_path_kernel_network "$TRACE_PATH" "$SESSION_NAME"
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*
if [ $? -eq 0 ]; then
# Only delete if successful
TRACE_PATH=$(mktemp -d)
-NUM_TESTS=16
+NUM_TESTS=18
source $TESTDIR/utils/utils.sh
${fct_test}
# Validate test
+ validate_trace_path_ust_uid_network "$TRACE_PATH" "$SESSION_NAME"
validate_trace $EVENT_NAME $TRACE_PATH/$HOSTNAME/$SESSION_NAME*
if [ $? -eq 0 ]; then
# Only delete if successful
TESTAPP_NAME="gen-ust-events"
TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
-NUM_TESTS=66
+NUM_TESTS=74
source $TESTDIR/utils/utils.sh
# Validate tracing data, we should at least have some events
+ validate_trace_path_ust_pid "$trace_path" "" "gen-ust-eve-ust"
validate_trace $event_name $trace_path
rm -rf $trace_path
# Validate tracing data, we should at least have some events
+ validate_trace_path_ust_pid "$trace_path" "" "gen-ust-eve-ust"
validate_trace $event_name $trace_path
rm -rf $trace_path
OUTPUT_DEST=/dev/null
ERROR_OUTPUT_DEST=/dev/null
+# To match 20201127-175802
+date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
+# The size of a long on this system
+system_long_bit_size=$(getconf LONG_BIT)
+
# Minimal kernel version supported for session daemon tests
KERNEL_MAJOR_VERSION=2
KERNEL_MINOR_VERSION=6
{
lttng_remove_trigger 0 "$@"
}
+
+function validate_path_pattern ()
+{
+ local message=$1
+ local pattern=$2
+ # Base path is only used in error case and is used to list the content
+ # of the base path.
+ local base_path=$3
+
+
+ [ -f $pattern ]
+ ret=$?
+ ok $ret "$message"
+
+ if [ "$ret" -ne "0" ]; then
+ diag "Path pattern expected: $pattern"
+ # List the tracepath for more info. We use find as a recursive
+ # directory lister.
+ diag "The base path content:"
+ find "$base_path" -print
+ fi
+}
+
+function validate_trace_path_ust_uid ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local uid=$UID
+ local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+ validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_network ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local base_path=$3
+ local uid=$UID
+ local hostname=$HOSTNAME
+ local pattern
+ local ret
+
+ # If the session was given a network base path (e.g
+ # 127.0.0.1/my/custom/path on creation, there is no session name
+ # component to the path on the relayd side. Caller can simply not pass a
+ # session name for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ if [ -n "$base_path" ]; then
+ fail "Session name and base path are mutually exclusive"
+ return
+ fi
+ fi
+
+ pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+ validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_snapshot_network ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local snapshot_name=$3
+ local snapshot_number=$4
+ local base_path=$5
+ local hostname=$HOSTNAME
+ local uid=$UID
+ local pattern
+ local ret
+
+ # If the session/output was given a network base path (e.g
+ # 127.0.0.1/my/custom/path on creation, there is no session name
+ # component to the path on the relayd side. Caller can simply not pass a
+ # session name for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ if [ -n "$base_path" ]; then
+ fail "Session name and base path are mutually exclusive"
+ return
+ fi
+ fi
+
+ pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+ validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_snapshot ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local snapshot_name=$3
+ local snapshot_number=$4
+ local base_path=$5
+ local uid=$UID
+ local pattern
+ local ret
+
+ # If the session/output was given a network base path (e.g
+ # 127.0.0.1/my/custom/path) on creation, there is no session name
+ # component to the path on the relayd side. Caller can simply not pass a
+ # session name for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ if [ -n "$base_path" ]; then
+ fail "Session name and base path are mutually exclusive"
+ return
+ fi
+ fi
+
+ pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+ validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_pid ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local app_string=$3
+ local pid=$4
+ local pattern
+ local ret
+
+ # If the session was given a trace path on creation, there is no session
+ # name component to the path. Caller can simply not pass a session name
+ # for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ fi
+
+ pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
+
+ validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local pattern
+
+ # If the session was given a trace path on creation, there is no session
+ # name component to the path. Caller can simply not pass a session name
+ # for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ fi
+
+ pattern="$trace_path/$session_name/kernel/metadata"
+
+ validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_network ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local hostname=$HOSTNAME
+ local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
+
+ validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_snapshot ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local snapshot_name=$3
+ local snapshot_number=$4
+ local base_path=$5
+ local pattern
+ local ret
+
+ # If the session/output was given a network base path (e.g
+ # 127.0.0.1/my/custom/path on creation, there is no session name
+ # component to the path on the relayd side. Caller can simply not pass a
+ # session name for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ if [ -n "$base_path" ]; then
+ fail "Session name and base path are mutually exclusive"
+ return
+ fi
+ fi
+
+ pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
+
+ validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_snapshot_network ()
+{
+ local trace_path=$1
+ local session_name=$2
+ local snapshot_name=$3
+ local snapshot_number=$4
+ local base_path=$5
+ local hostname=$HOSTNAME
+ local pattern
+ local ret
+
+ # If the session/output was given a network base path (e.g
+ # 127.0.0.1/my/custom/path on creation, there is no session name
+ # component to the path on the relayd side. Caller can simply not pass a
+ # session name for this scenario.
+ if [ -n "$session_name" ]; then
+ session_name="$session_name-$date_time_pattern"
+ if [ -n "$base_path" ]; then
+ fail "Session name and base path are mutually exclusive"
+ return
+ fi
+ fi
+
+ pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
+
+ validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
+}