Commit | Line | Data |
---|---|---|
d329b32d KS |
1 | #!/usr/bin/bash -eux |
2 | ||
3 | CLEANUP=() | |
4 | ||
5 | function cleanup { | |
6 | set +e | |
7 | for (( index=${#CLEANUP[@]}-1 ; index >= 0 ; index-- )) ;do | |
8 | ${CLEANUP[$index]} | |
9 | done | |
10 | CLEANUP=() | |
11 | set -e | |
12 | } | |
13 | ||
14 | function fail { | |
15 | CODE="${1:-1}" | |
16 | REASON="${2:-Unknown reason}" | |
17 | cleanup | |
18 | echo "${REASON}" >&2 | |
19 | exit "${CODE}" | |
20 | } | |
21 | ||
22 | trap cleanup EXIT TERM INT | |
23 | ||
24 | env | |
25 | ||
26 | REQUIRED_VARIABLES=( | |
27 | OS # OS name | |
28 | RELEASE # OS release | |
29 | ARCH # The image architecture | |
30 | IMAGE_TYPE # The image type to create | |
31 | VARIANT # The variant of the base image to use | |
32 | PROFILE # The ansible group to apply to the new image | |
33 | GIT_BRANCH # The git branch of the automation repo to checkout | |
34 | GIT_URL # The git URL of the automation repo to checkout | |
35 | LXD_CLIENT_CERT # Path to LXD client certificate | |
36 | LXD_CLIENT_KEY # Path to LXD client certificate key | |
37 | SSH_PRIVATE_KEY # Path to SSH private key | |
38 | TEST # 'true' to test launching published image | |
39 | ) | |
40 | MISSING_VARS=0 | |
41 | for var in "${REQUIRED_VARIABLES[@]}" ; do | |
42 | if [ ! -v "$var" ] ; then | |
43 | MISSING_VARS=1 | |
44 | echo "Missing required variable: '${var}'" >&2 | |
45 | fi | |
46 | done | |
47 | if [[ ! "${MISSING_VARS}" == "0" ]] ; then | |
48 | fail 1 "Missing required variables" | |
49 | fi | |
50 | ||
51 | # Default optional variables | |
52 | INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-30}" | |
53 | NETWORK_SLEEP="${NETWORK_SLEEP:-15}" | |
54 | ||
55 | # Dependencies | |
56 | apt-get -y install lxd-client ansible jq | |
57 | ||
58 | # Configuration | |
59 | mkdir -p ~/.config/lxc | |
60 | cp "${LXD_CLIENT_CERT}" ~/.config/lxc/client.crt | |
61 | cp "${LXD_CLIENT_KEY}" ~/.config/lxc/client.key | |
62 | CLEANUP+=( | |
63 | "rm -f ${HOME}/.config/lxc/client.crt" | |
64 | "rm -f ${HOME}/.config/lxc/client.key" | |
65 | ) | |
66 | lxc remote add ci --accept-certificate --auth-type tls "${LXD_HOST}" | |
67 | lxc remote switch ci | |
68 | ||
69 | # Clone lttng-ci | |
70 | git clone -b "${GIT_BRANCH}" "${GIT_URL}" ci | |
71 | cd ci/automation/ansible || exit 1 | |
72 | ||
73 | SOURCE_IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}" | |
74 | # Include IMAGE_TYPE since an alias may only be defined once even if the | |
75 | # type of the image differs | |
76 | TARGET_IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}/${PROFILE}/${IMAGE_TYPE}" | |
77 | INSTANCE_NAME='' | |
78 | # Try from local cache | |
79 | VM_ARG=() | |
80 | if [ "${IMAGE_TYPE}" == "vm" ] ; then | |
81 | VM_ARG=("--vm") | |
82 | fi | |
83 | ||
84 | set +e | |
85 | # Test | |
86 | # It's possible that concurrent image creation when running parallel jobs causes | |
87 | # an error during the launch: | |
88 | # Error: Failed instance creation: UNIQUE constraint failed: images.project_id, images.fingerprint | |
89 | # C.f. https://github.com/canonical/lxd/issues/11636 | |
90 | # | |
91 | TRIES_MAX=3 | |
92 | TRIES=0 | |
93 | while [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; do | |
94 | if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" "${SOURCE_IMAGE_NAME}") ; then | |
95 | # Try from images | |
96 | if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" images:"${SOURCE_IMAGE_NAME}") ; then | |
97 | TRIES=$((TRIES + 1)) | |
98 | echo "Failed to deployed ephemereal instance attempt ${TRIES}/${TRIES_MAX}" | |
99 | if [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; then | |
100 | continue | |
101 | fi | |
102 | fail 1 "Failed to deploy ephemereal instance" | |
103 | else | |
104 | break | |
105 | fi | |
106 | else | |
107 | break | |
108 | fi | |
109 | done | |
110 | INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d ':' -f 2 | tr -d ' ')" | |
111 | set -e | |
112 | ||
113 | CLEANUP+=( | |
114 | "lxc stop ${INSTANCE_NAME}" | |
115 | ) | |
116 | ||
117 | # VMs may take more time to start, wait until instance is running | |
118 | TIME_REMAINING="${INSTANCE_START_TIMEOUT}" | |
119 | while true ; do | |
120 | set +e | |
121 | INSTANCE_STATUS=$(lxc exec "${INSTANCE_NAME}" hostname) | |
122 | set -e | |
123 | if [[ "${INSTANCE_STATUS}" == "${INSTANCE_NAME}" ]] ; then | |
124 | break | |
125 | fi | |
126 | sleep 1 | |
127 | TIME_REMAINING=$((TIME_REMAINING - 1)) | |
128 | if [ "${TIME_REMAINING}" -lt "0" ] ; then | |
129 | fail 1 "Timed out waiting for instance to become available via 'lxc exec'" | |
130 | fi | |
131 | done | |
132 | ||
133 | # Wait for cloud-init to finish | |
134 | if [[ "${VARIANT}" == "cloud" ]] ; then | |
135 | lxc exec "${INSTANCE_NAME}" -- cloud-init status -w | |
136 | fi | |
137 | ||
138 | # Wait for instance to have an ip address (@TODO: is there a better approach?) | |
139 | sleep "${NETWORK_SLEEP}" | |
140 | ||
141 | # @TODO: Handle case when iputils2 is not installed | |
142 | INSTANCE_IP='' | |
143 | POTENTIAL_INTERFACES=(eth0 enp5s0) | |
144 | lxc exec "${INSTANCE_NAME}" -- ip a | |
145 | set +e | |
146 | for interface in "${POTENTIAL_INTERFACES[@]}" ; do | |
147 | if ! DEV_INFO="$(lxc exec "${INSTANCE_NAME}" -- ip a show dev "${interface}")" ; then | |
148 | continue | |
149 | fi | |
150 | INSTANCE_IP="$(echo "${DEV_INFO}" | grep -Eo 'inet [^ ]* ' | cut -d' ' -f2 | cut -d'/' -f1)" | |
151 | if [[ "${INSTANCE_IP}" != "" ]] ; then | |
152 | break | |
153 | fi | |
154 | done | |
155 | set -e | |
156 | if [[ "${INSTANCE_IP}" == "" ]] ; then | |
157 | fail 1 "Failed to determine instance IP address" | |
158 | fi | |
159 | ||
160 | ssh-keyscan "${INSTANCE_IP}" >> ~/.ssh/known_hosts2 | |
161 | #lxc exec "${INSTANCE_NAME}" -- bash -c 'for i in /etc/ssh/ssh_host_*_key ; do ssh-keygen -l -f "$i" ; done' >> "${HOME}/.ssh/known_hosts" | |
162 | CLEANUP+=( | |
163 | "rm -f ${HOME}/.ssh/known_hosts2" | |
164 | ) | |
165 | cp "${SSH_PRIVATE_KEY}" ~/.ssh/id_rsa | |
166 | ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub | |
167 | CLEANUP+=( | |
168 | "rm -f ${HOME}/.ssh/id_rsa.pub" | |
169 | "rm -f ${HOME}/.ssh/id_rsa" | |
170 | ) | |
171 | lxc file push ~/.ssh/id_rsa.pub "ci:${INSTANCE_NAME}/root/.ssh/authorized_keys2" | |
172 | ||
173 | # Confirm working SSH connection | |
174 | if ! ssh "${INSTANCE_IP}" hostname ; then | |
175 | fail 1 "Unable to reach ephemereal instance over SSH" | |
176 | fi | |
177 | ||
178 | # Run playbook | |
179 | cat > fake-inventory <<EOF | |
180 | [${PROFILE/-/_}] | |
181 | ${INSTANCE_IP} | |
182 | EOF | |
183 | CLEANUP+=( | |
184 | "rm -f $(pwd)/fake-inventory" | |
185 | ) | |
186 | ||
187 | LANG=C ANSIBLE_STRATEGY=linear ansible-playbook site.yml \ | |
188 | -e '{"compilers_legacy_install": false, "jenkins_user": false, "lttng_modules_checkout_repo": false}' \ | |
189 | -l "${INSTANCE_IP}" -i fake-inventory | |
190 | ||
191 | # Cleanup instance side | |
192 | # @TODO: Distro switch for apt/dnf/yum/etc. | |
193 | lxc exec "${INSTANCE_NAME}" -- apt-get clean | |
194 | lxc exec "${INSTANCE_NAME}" -- rm -rf /root/.ssh/authorized_keys2 | |
195 | lxc exec "${INSTANCE_NAME}" -- cloud-init clean | |
196 | lxc exec "${INSTANCE_NAME}" -- bash -c 'history -cw' | |
197 | ||
198 | # Publish | |
199 | lxc publish "${INSTANCE_NAME}" --alias "${TARGET_IMAGE_NAME}" -f | |
200 | ||
201 | TRIES=0 | |
202 | ||
203 | if [[ "${TEST}" == "true" ]] ; then | |
204 | set +e | |
205 | while [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; do | |
206 | if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" "${TARGET_IMAGE_NAME}") ; then | |
207 | TRIES=$((TRIES + 1)) | |
208 | echo "Failed to launch instance try ${TRIES}/${TRIES_MAX}" | |
209 | if [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; then | |
210 | sleep $((1 + RANDOM % 10)) | |
211 | continue | |
212 | fi | |
213 | fail 1 "Failed to launch an instance using newly published image '${TARGET_IMAGE_NAME}'" | |
214 | else | |
215 | INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d':' -f2 | tr -d ' ')" | |
216 | CLEANUP+=( | |
217 | "lxc stop -f ${INSTANCE_NAME}" | |
218 | ) | |
219 | break | |
220 | fi | |
221 | done | |
222 | set -e | |
223 | fi |