From: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>
Date: Thu, 1 Nov 2018 21:13:56 +0000 (-0400)
Subject: Use s3 (minio) for object storage of system_test.
X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=6252703ba33efacb583f10df01b2454c5fde1d74;p=lttng-ci.git

Use s3 (minio) for object storage of system_test.

Work toward the use of lava 2.

Keep old normal storage for now.

Signed-off-by: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>
---

diff --git a/jobs/system-tests.yaml b/jobs/system-tests.yaml
index 0887e5b..67e148c 100644
--- a/jobs/system-tests.yaml
+++ b/jobs/system-tests.yaml
@@ -302,6 +302,18 @@
       - string:
           name: 'BUILD_DEVICE'
           description: 'The target device. (kvm or baremetal)'
+      - string:
+          name: 'S3_STORAGE_KERNEL_FOLDER'
+          description: 'Path to store the Kernel image'
+      - string:
+          name: 'S3_STORAGE_KERNEL_IMAGE'
+          description: 'Path to store the Kernel IMAGE'
+      - string:
+          name: 'S3_STORAGE_LINUX_MODULES'
+          description: 'Path to store the Kernel Modules'
+      - string:
+          name: 'S3_STORAGE_LTTNG_MODULES'
+          description: 'Path to store the LTTng Modules'
 
     properties:
       - build-discarder:
diff --git a/scripts/system-tests/build-kernel.sh b/scripts/system-tests/build-kernel.sh
index 7f0cca7..d8def1c 100644
--- a/scripts/system-tests/build-kernel.sh
+++ b/scripts/system-tests/build-kernel.sh
@@ -29,3 +29,8 @@ $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$STORAGE_USER@$STORAGE_HOST
 $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_CONFIG"
 $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LINUX_MODULES"
 $SCP_COMMAND "$LINUX_PATH/Module.symvers" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS"
+
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$S3_STORAGE_KERNEL_IMAGE"
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$S3_STORAGE_KERNEL_CONFIG"
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$S3_STORAGE_LINUX_MODULES"
+$S3_COMMAND put "$LINUX_PATH/Module.symvers" "$S3_STORAGE_KERNEL_MODULE_SYMVERS"
diff --git a/scripts/system-tests/build-modules.sh b/scripts/system-tests/build-modules.sh
index 04a8c0f..8bbeccc 100644
--- a/scripts/system-tests/build-modules.sh
+++ b/scripts/system-tests/build-modules.sh
@@ -26,3 +26,4 @@ KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" module
 tar -czf "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./
 
 $SCP_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LTTNG_MODULES"
+$S3_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$S3_STORAGE_LTTNG_MODULES"
diff --git a/scripts/system-tests/check-build-needs.sh b/scripts/system-tests/check-build-needs.sh
index 162a7c4..21acde4 100644
--- a/scripts/system-tests/check-build-needs.sh
+++ b/scripts/system-tests/check-build-needs.sh
@@ -27,10 +27,23 @@ if [ $? -ne 0 ]; then
   NEED_MODULES_BUILD=1
 fi
 
+$S3_COMMAND info "$S3_STORAGE_KERNEL_IMAGE"
+if [ $? -ne 0 ]; then
+  NEED_KERNEL_BUILD=1
+  # We need to build the lttng modules if the kernel has changed.
+  NEED_MODULES_BUILD=1
+fi
+
 $SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_LTTNG_MODULES"
 if [ $? -ne 0 ]; then
   NEED_MODULES_BUILD=1
 fi
+
+$S3_COMMAND info "$S3_STORAGE_LTTNG_MODULES"
+if [ $? -ne 0 ]; then
+  NEED_MODULES_BUILD=1
+fi
+
 set -e
 
 # We need to fetch the kernel source and lttng-modules to build either the
diff --git a/scripts/system-tests/generate-properties-master.sh b/scripts/system-tests/generate-properties-master.sh
index bb2e25b..3665835 100644
--- a/scripts/system-tests/generate-properties-master.sh
+++ b/scripts/system-tests/generate-properties-master.sh
@@ -16,6 +16,19 @@
 
 touch properties.txt
 
+echo "# Setup endpoint
+host_base = obj.internal.efficios.com
+host_bucket = obj.internal.efficios.com
+bucket_location = us-east-1
+use_https = True
+
+# Setup access keys
+access_key = jenkins
+secret_key = echo123456
+
+# Enable S3 v4 signature APIs
+signature_v2 = False" > "$WORKSPACE/s3cfg"
+
 LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci"
 echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt
 KERNEL_COMMIT_ID=$KERNEL_TAG_ID
@@ -37,3 +50,11 @@ echo "STORAGE_KERNEL_FOLDER=$BASE_STORAGE_FOLDER/kernel" >> properties.txt
 echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt
 echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt
 echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt
+
+BASE_S3_STORAGE="s3://lava"
+
+echo "BASE_S3_STORAGE=$BASE_S3_STORAGE" >> properties.txt
+echo "S3_STORAGE_KERNEL_FOLDER=$BASE_S3_STORAGE/kernel/" >> properties.txt
+echo "S3_STORAGE_KERNEL_IMAGE=$BASE_S3_STORAGE/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt
+echo "S3_STORAGE_LINUX_MODULES=$BASE_S3_STORAGE/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt
+echo "S3_STORAGE_LTTNG_MODULES=$BASE_S3_STORAGE/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt
diff --git a/scripts/system-tests/generate-properties-slave.sh b/scripts/system-tests/generate-properties-slave.sh
index 426e3b5..182d350 100644
--- a/scripts/system-tests/generate-properties-slave.sh
+++ b/scripts/system-tests/generate-properties-slave.sh
@@ -16,6 +16,19 @@
 
 touch properties.txt
 
+echo "# Setup endpoint
+host_base = obj.internal.efficios.com
+host_bucket = obj.internal.efficios.com
+bucket_location = us-east-1
+use_https = True
+
+# Setup access keys
+access_key = jenkins
+secret_key = echo123456
+
+# Enable S3 v4 signature APIs
+signature_v2 = False" > "$WORKSPACE/s3cfg"
+
 # Use all CPU cores
 NPROC=$(nproc)
 echo "NPROC=$NPROC" >> properties.txt
@@ -42,9 +55,12 @@ echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt
 
 echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt
 echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt
+echo "S3_STORAGE_KERNEL_MODULE_SYMVERS=$S3_STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt
+echo "S3_STORAGE_KERNEL_CONFIG=$S3_STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt
 
 echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt
 echo "STORAGE_USER=jenkins-lava" >> properties.txt
 
 echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
 echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
+echo S3_COMMAND="s3cmd -c ${WORKSPACE}/s3cfg" >> properties.txt
diff --git a/scripts/system-tests/inject-ssh-commands.sh b/scripts/system-tests/inject-ssh-commands.sh
index 74e4dde..8f3598c 100644
--- a/scripts/system-tests/inject-ssh-commands.sh
+++ b/scripts/system-tests/inject-ssh-commands.sh
@@ -16,3 +16,4 @@
 
 echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
 echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
+echo S3_COMMAND="s3cmd -c ${WORKSPACE}/s3cfg" >> properties.txt