jjb: lava: kprobe-fuzzing: Split testcase into multiple independent ones

With the current configuration, the fuzzing is done in a single Lava
testcase. If the kernel crashes during the testcase, the Lava slave will
be rebooted once the timeout expires(currently 2 hours). If the testcase
crashes at the beginning of the testcase, the job still has to wait for
the timeout to expire. That causes a lot of wasted time.

We overcome this by split the fuzzing into multiple smaller testcases
all with a smaller timeout and a specific range of instrumentation
points to test. This has the benefits of increasing our testing coverage
and resource efficiency.

Signed-off-by: Francis Deslauriers <francis.deslauriers@efficios.com>

diff --git a/lava/system-tests/kprobe-fuzzing-tests.yml b/lava/system-tests/kprobe-fuzzing-tests.yml
index 56812bac329b6896227e3905875f07bf3d420c10..a0bce4a41094cc77618922ff0cb7e0acf72b3b29 100644 (file)
--- a/lava/system-tests/kprobe-fuzzing-tests.yml
+++ b/lava/system-tests/kprobe-fuzzing-tests.yml
@@ -12,10 +12,12 @@ install:
                 - ulimit -c unlimited
                 - mkdir -p coredump
                 - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
+params:
+  ROUND_NB: 0
 run:
         steps:
                 - cd ci/
-                - lava-test-case run-fuzzing --shell "python3 ./scripts/system-tests/run-kprobe-fuzzing.py /root/instr_points.txt.gz"
+                - lava-test-case run-fuzzing --shell "python3 ./scripts/system-tests/run-kprobe-fuzzing.py /root/instr_points.txt.gz $ROUND_NB"
                 - cd ..
                 - tar czf coredump.tar.gz coredump
                 - lava-test-case-attach run-fuzzing coredump.tar.gz

diff --git a/scripts/system-tests/lava-submit.py b/scripts/system-tests/lava-submit.py
index 62df12b171bbd39d9b0fd89d3327a9e7573f4ffc..682e2249e11cc7fe5229baf081330a6e564059fe 100644 (file)
--- a/scripts/system-tests/lava-submit.py
+++ b/scripts/system-tests/lava-submit.py
@@ -284,7 +284,7 @@ def get_kprobes_generate_data_cmd():
         })
     return command
 
-def get_kprobes_test_cmd():
+def get_kprobes_test_cmd(round_nb):
     command = OrderedDict({
         'command': 'lava_test_shell',
         'parameters': {
@@ -292,10 +292,11 @@ def get_kprobes_test_cmd():
                 {
                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
                     'revision': 'master',
-                    'testdef': 'lava/system-tests/kprobe-fuzzing-tests.yml'
+                    'testdef': 'lava/system-tests/kprobe-fuzzing-tests.yml',
+                    'parameters': { 'ROUND_NB': str(round_nb) }
                 }
-                ],
-            'timeout': 7200
+            ],
+            'timeout': 1000
             }
         })
     return command
@@ -464,7 +465,8 @@ def main():
             return -1
         j['actions'].append(get_config_cmd('kvm'))
         j['actions'].append(get_kprobes_generate_data_cmd())
-        j['actions'].append(get_kprobes_test_cmd())
+        for i in range(10):
+            j['actions'].append(get_kprobes_test_cmd(round_nb=i))
         j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
     else:
         assert False, 'Unknown test type'

diff --git a/scripts/system-tests/run-kprobe-fuzzing.py b/scripts/system-tests/run-kprobe-fuzzing.py
index 9e4a2a017e0e880758cef4f082a4e3f08e7c8ec3..aed583e08a1a52bd7c2617c10d8f93ccd18e765c 100644 (file)
--- a/scripts/system-tests/run-kprobe-fuzzing.py
+++ b/scripts/system-tests/run-kprobe-fuzzing.py
@@ -21,6 +21,7 @@ import subprocess
 import sys
 
 NB_KPROBES_PER_ITER=500
+NB_KPROBES_PER_ROUND=20000
 
 def load_instr_points(instr_points_archive):
     print('Reading instrumentation points from \'{}\'.'.format(instr_points_archive), end='')
@@ -87,23 +88,31 @@ def print_dashed_line():
     print('-'*100)
 
 def main():
-    assert(len(sys.argv) == 2)
+    assert(len(sys.argv) == 3)
 
     instr_point_archive = sys.argv[1]
+    round_nb = int(sys.argv[2])
     # Load instrumentation points to disk and attach it to lava test run.
     instrumentation_points = load_instr_points(instr_point_archive)
 
+    # We are past the end of the instrumentation point list.
+    if len(instrumentation_points)/NB_KPROBES_PER_ROUND <= round_nb:
+        print('No instrumentation point for round {}.'.format(round_nb))
+        return
+
     mount_tracingfs()
 
     # Loop over the list by enabling ranges of NB_KPROBES_PER_ITER kprobes.
-    for i in range(int(len(instrumentation_points)/NB_KPROBES_PER_ITER)):
+    for i in range(int(NB_KPROBES_PER_ROUND/NB_KPROBES_PER_ITER)):
         print_dashed_line()
-        print('Time now: {}, {} to {}'.format(datetime.datetime.now(), i*NB_KPROBES_PER_ITER, (i+1)*NB_KPROBES_PER_ITER))
-        set_kprobe_tracing_state(0)
-        enable_kprobe_events(instrumentation_points[i*NB_KPROBES_PER_ITER:(i+1)*NB_KPROBES_PER_ITER])
+        lower_bound = (round_nb * NB_KPROBES_PER_ROUND) + (i * NB_KPROBES_PER_ITER)
+        upper_bound = lower_bound + NB_KPROBES_PER_ITER
+        print('Time now: {}, {} to {}'.format(datetime.datetime.now(), lower_bound , upper_bound))
+        enable_kprobe_events(instrumentation_points[lower_bound:upper_bound])
         set_kprobe_tracing_state(1)
         run_workload()
         print('\n')
+        set_kprobe_tracing_state(0)
 
 if __name__ == "__main__":
     main()