From a2e5135c912e93466a64aa9bb0a91a29f83477be Mon Sep 17 00:00:00 2001 From: Kienan Stewart Date: Tue, 30 Jan 2024 15:53:51 -0500 Subject: [PATCH] jjb/ansible: Add support for building SLES images Change-Id: I89af9759bb7b3c8352ea19f1f6890842cb4efe9a Signed-off-by: Kienan Stewart --- .../playbooks/post-imagebuild-clean.yml | 23 ++ automation/images/sles-12.5.yml | 281 +++++++++++++++++ automation/images/sles-15.4.yml | 292 ++++++++++++++++++ jobs/images.yml | 139 ++++++--- pipelines/images/distrobuild.sh | 12 +- pipelines/images/imagebuild.sh | 10 +- 6 files changed, 704 insertions(+), 53 deletions(-) create mode 100644 automation/ansible/playbooks/post-imagebuild-clean.yml create mode 100644 automation/images/sles-12.5.yml create mode 100644 automation/images/sles-15.4.yml diff --git a/automation/ansible/playbooks/post-imagebuild-clean.yml b/automation/ansible/playbooks/post-imagebuild-clean.yml new file mode 100644 index 0000000..d4eb23d --- /dev/null +++ b/automation/ansible/playbooks/post-imagebuild-clean.yml @@ -0,0 +1,23 @@ +--- +- hosts: all + tasks: + - when: ansible_os_family == 'Debian' + ansible.builtin.command: + argv: ['apt-get', 'clean'] + - when: ansible_os_family == 'Suse' + ansible.builtin.command: + argv: ['zypper', 'clean'] + - when: ansible_distribution == 'SLES' + block: + - ansible.builtin.command: + argv: ['SUSEConnect', '-d'] + - ansible.builtin.command: + argv: ['SUSEConnect', '--cleanup'] + - ansible.builtin.command: + argv: ['cloud-init', 'clean'] + ignore_errors: true + - ansible.builtin.shell: + cmd: 'history -cw' + - ansible.builtin.file: + path: /root/.ssh/authorized_keys2 + state: absent diff --git a/automation/images/sles-12.5.yml b/automation/images/sles-12.5.yml new file mode 100644 index 0000000..89b2c0e --- /dev/null +++ b/automation/images/sles-12.5.yml @@ -0,0 +1,281 @@ +--- +# based on https://github.com/lxc/lxc-ci/blob/main/images/opensuse.yaml +image: + distribution: 'sles' + release: '12.5' + architecture: 'x86_64' + variant: cloud + +environment: + variables: + - key: SLES_REGISTRATION_CODE + value: XXSLES_REGISTRATION_CODE_amd64XX + +# This rootfs requires some preparation +# 1. Download the SLES qemu-kvm minimal image +# 2. Use qemu-nbd to map the image to a device +# `qemu-nbd -c /dev/nbd0 /path/to/SLES-qemu-kvm.img` +# 3. Mount the principal btrfs partition on /mnt +# * Note: the SLES images make extensive use of btrfs subvols +# for /home, /opt, /root/, /srv, various direcotries inside /var, +# /boot/grub2/*, /usr/local/, and more. The mappings can be found +# in /etc/fstab. +# * For lxd/incus, the rootfs will be ext4; however, to prepare the +# initial archive many of the subvols will need to mounted. +# 4. Bind mount /dev/ to /mnt/dev +# `for i in dev proc sys ; do mount -o bind /$i /mnt/$i ; done` +# 5. ***Using chroot in /mnt*** mount the subvols +# `chroot /mnt mount -a` +# +# @TODO: Register and install more or the packages (eg., kernel) to avoid +# redoing the work each time an image is built from the rootfs. +# +# 6. Outside the chroot, prepare the archive file +# `tar -czf /path/to/rootfs.tgz -C /mnt --exclude './.snapshots/*' \ +# --exclude './dev/*' --exclude './proc/*' --exclude './sys/*' ./` +# 7. Unmount bind mounts +# `for i in $(findmnt -R -l -k -n -o TARGET /mnt) ; do umount $i; done; umount /mnt` +# 8. Disconnect the nbd device +# `qemu-nbd -d /dev/nbd0` +# +source: + downloader: rootfs-http + url: https://obj.internal.efficios.com/jenkins/rootfs_amd64_sles12sp5.tar.gz + +targets: + lxc: + create_message: | + You just created an {{ image.description }} container. + config: + - type: all + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/opensuse.common.conf + + - type: user + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/opensuse.userns.conf + + - type: all + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/common.conf + + - type: user + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/userns.conf + + - type: all + content: |- + lxc.arch = {{ image.architecture_kernel }} + +files: + - name: hostname + path: /etc/hostname + generator: hostname + + - name: hosts + path: /etc/hosts + generator: hosts + + - path: /etc/machine-id + generator: dump + + - path: /var/lib/dbus/machine-id + generator: remove + + - name: ifcfg-eth0 + path: /etc/sysconfig/network/ifcfg-eth0 + generator: dump + content: |- + STARTMODE='auto' + BOOTPROTO='dhcp' + + - name: user-data + generator: cloud-init + variants: + - cloud + + - name: vendor-data + generator: cloud-init + variants: + - cloud + + - generator: fstab + types: + - vm + + - generator: incus-agent + types: + - vm + + - path: /etc/dracut.conf.d/incus.conf + generator: dump + content: |- + add_drivers+=" virtio_scsi virtio_pci sd_mod " + types: + - vm + + - path: /etc/fstab + generator: dump + content: "# empty fstab to silence cloud-init warnings" + types: + - container + variants: + - cloud + +packages: + manager: zypper + update: false + cleanup: false + sets: + - packages: + - jeos-firstboot + - jeos-licenses + action: remove + +actions: + - trigger: post-unpack + action: |- + #!/bin/sh + set -eux + + systemd-machine-id-setup + mount -t tmpfs tmpfs /sys/firmware + types: + - vm + + - trigger: post-unpack + action: |- + #!/bin/sh + set -eu + set +x + SUSEConnect -r $SLES_REGISTRATION_CODE + set -x + + - trigger: post-packages + action: |- + #!/bin/sh + # Fails during the packages step due to not supporting `--allow-downgrades` + zypper --non-interactive --gpg-auto-import-keys install elfutils \ + glib2-tools gzip iproute2 iputils openssh pigz rsync sudo which xz + + - trigger: post-packages + action: |- + #!/bin/sh + # Fails during the packages step due to not supporting `--allow-downgrades` + zypper --non-interactive --gpg-auto-import-keys install shim + types: + - vm + architectures: + - amd64 + + - trigger: post-packages + action: |- + #!/bin/sh + # Fails during the packages step due to not supporting `--allow-downgrades` + zypper --non-interactive --gpg-auto-import-keys install e2fsprogs + types: + - vm + + - trigger: post-packages + action: |- + #!/bin/sh + set -eux + + # Install cloud-init from a separate product + SUSEConnect --product sle-module-public-cloud/12/x86_64 + zypper --non-interactive --gpg-auto-import-keys install cloud-init-config-suse cloud-init + + # Enable the cloud-init systemd service + systemctl enable cloud-init.service cloud-config.service cloud-final.service + variants: + - cloud + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + for mod in virtio_scsi virtio_pci sd_mod ; do + echo "install $mod /bin/true" >> /etc/modprobe.d/99-local.conf + done + + # This gets around the kernel-default installation failing + dracut --regenerate-all --force + mount -t tmpfs tmpfs /sys/firmware + mkdir /sys/firmware/efi + grub2-mkconfig -o /boot/grub2/grub.cfg + + if which shim-install; then + shim-install --no-nvram --removable + shim-install --no-nvram + else + grub2-install --no-nvram --removable + grub2-install --no-nvram + fi + + grub2-mkconfig -o /boot/grub2/grub.cfg + sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub2/grub.cfg + zypper --non-interactive install kernel-default + + # If this isn't re-done, the VM won't boot + dracut --regenerate-all --force --add-drivers 'virtio_scsi virtio_pci sd_mod' + grub2-mkconfig -o /boot/grub2/grub.cfg + + if which shim-install; then + shim-install --no-nvram --removable + shim-install --no-nvram + else + grub2-install --no-nvram --removable + grub2-install --no-nvram + fi + + grub2-mkconfig -o /boot/grub2/grub.cfg + sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub2/grub.cfg + + umount /sys/firmware + types: + - vm + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + SUSEConnect -d + umount -l /etc/resolv.conf || true + rm /etc/resolv.conf + ln -sf /var/run/netconfig/resolv.conf /etc/resolv.conf + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + # Automatic disk resize + cat << EOF > /etc/systemd/system/incus-growpart.service + [Unit] + Description=Incus - grow root partition + + [Service] + Type=oneshot + ExecStartPre=-/usr/sbin/growpart /dev/sda 2 + ExecStart=/usr/sbin/resize2fs /dev/sda2 + + [Install] + WantedBy=default.target + EOF + systemctl enable incus-growpart + types: + - vm + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + # By default, sles systems don't check authorized_keys2, + # renable it. + sed -E -i 's/^AuthorizedKeysFile[\t ]+.ssh\/authorized_keys$/AuthorizedKeysFile .ssh\/authorized_keys .ssh\/authorized_keys2/g' /etc/ssh/sshd_config diff --git a/automation/images/sles-15.4.yml b/automation/images/sles-15.4.yml new file mode 100644 index 0000000..a378597 --- /dev/null +++ b/automation/images/sles-15.4.yml @@ -0,0 +1,292 @@ +--- +# based on https://github.com/lxc/lxc-ci/blob/main/images/opensuse.yaml +image: + distribution: 'sles' + release: '15.4' + architecture: 'amd64' + variant: cloud + +environment: + variables: + - key: SLES_REGISTRATION_CODE + value: XXSLES_REGISTRATION_CODE_amd64XX + +# This rootfs requires some preparation +# 1. Download the SLES qemu-kvm minimal image +# 2. Use qemu-nbd to map the image to a device +# `qemu-nbd -c /dev/nbd0 /path/to/SLES-qemu-kvm.img` +# 3. Mount the principal btrfs partition on /mnt +# * Note: the SLES images make extensive use of btrfs subvols +# for /home, /opt, /root/, /srv, various direcotries inside /var, +# /boot/grub2/*, /usr/local/, and more. The mappings can be found +# in /etc/fstab. +# * For lxd/incus, the rootfs will be ext4; however, to prepare the +# initial archive many of the subvols will need to mounted. +# 4. Bind mount /dev/ to /mnt/dev +# `for i in dev proc sys ; do mount -o bind /$i /mnt/$i ; done` +# 5. ***Using chroot in /mnt*** mount the subvols +# `chroot /mnt mount -a` +# +# @TODO: Register and install more or the packages (eg., kernel) to avoid +# redoing the work each time an image is built from the rootfs. +# +# 6. Outside the chroot, prepare the archive file +# `tar -czf /path/to/rootfs.tgz -C /mnt --exclude './.snapshots/*' \ +# --exclude './dev/*' --exclude './proc/*' --exclude './sys/*' ./` +# 7. Unmount bind mounts +# `for i in $(findmnt -R -l -k -n -o TARGET /mnt) ; do umount $i; done; umount /mnt` +# 8. Disconnect the nbd device +# `qemu-nbd -d /dev/nbd0` +# +source: + downloader: rootfs-http + url: https://obj.internal.efficios.com/jenkins/rootfs_amd64_sles15sp4.tar.gz + +targets: + lxc: + create_message: | + You just created an {{ image.description }} container. + config: + - type: all + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/opensuse.common.conf + + - type: user + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/opensuse.userns.conf + + - type: all + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/common.conf + + - type: user + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/userns.conf + + - type: all + content: |- + lxc.arch = {{ image.architecture_kernel }} + +files: + - name: hostname + path: /etc/hostname + generator: hostname + + - name: hosts + path: /etc/hosts + generator: hosts + + - path: /etc/machine-id + generator: dump + + - path: /var/lib/dbus/machine-id + generator: remove + + - name: ifcfg-eth0 + path: /etc/sysconfig/network/ifcfg-eth0 + generator: dump + content: |- + STARTMODE='auto' + BOOTPROTO='dhcp' + + - name: user-data + generator: cloud-init + variants: + - cloud + + - name: vendor-data + generator: cloud-init + variants: + - cloud + + - generator: fstab + types: + - vm + + - generator: incus-agent + types: + - vm + + - path: /etc/dracut.conf.d/incus.conf + generator: dump + content: |- + add_drivers+=" virtio_scsi virtio_pci sd_mod " + types: + - vm + + - path: /etc/fstab + generator: dump + content: "# empty fstab to silence cloud-init warnings" + types: + - container + variants: + - cloud + +packages: + manager: zypper + update: false + cleanup: true + sets: + - packages: + - jeos-firstboot + - jeos-licenses + action: remove + + - packages: + - elfutils + - file + - glib2-tools + - gzip + - hostname + - iproute2 + - iputils + - openssh-server + - pigz + - rsync + - sudo + - which + - xz + action: install + + - packages: + - shim + action: install + types: + - vm + architectures: + - amd64 + + - packages: + - e2fsprogs + action: install + types: + - vm + +actions: + - trigger: post-unpack + action: |- + #!/bin/sh + set -eux + + systemd-machine-id-setup + mount -t tmpfs tmpfs /sys/firmware + types: + - vm + + - trigger: post-unpack + action: |- + #!/bin/sh + set -eu + set +x + suseconnect -r $SLES_REGISTRATION_CODE + set -x + + - trigger: post-packages + action: |- + #!/bin/sh + set -eux + # These services don't run properly in containers + systemctl disable chronyd.service + systemctl disable auditd.service + systemctl disable klog.service + types: + - container + + - trigger: post-packages + action: |- + #!/bin/sh + set -eux + + # Install cloud-init from various RPMs + suseconnect --product sle-module-public-cloud/15.4/x86_64 + zypper --non-interactive --gpg-auto-import-keys install cloud-init-config-suse cloud-init + + # Enable the cloud-init systemd service + systemctl enable cloud-init.service cloud-config.service cloud-final.service + variants: + - cloud + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + # This gets around the kernel-default installation failing + dracut --regenerate-all --force + mount -t tmpfs tmpfs /sys/firmware + mkdir /sys/firmware/efi + grub2-mkconfig -o /boot/grub2/grub.cfg + + if which shim-install; then + shim-install --no-nvram --removable + shim-install --no-nvram + else + grub2-install --no-nvram --removable + grub2-install --no-nvram + fi + + grub2-mkconfig -o /boot/grub2/grub.cfg + sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub2/grub.cfg + zypper --non-interactive install kernel-default + + # If this isn't re-done, the VM won't boot + grub2-mkconfig -o /boot/grub2/grub.cfg + + if which shim-install; then + shim-install --no-nvram --removable + shim-install --no-nvram + else + grub2-install --no-nvram --removable + grub2-install --no-nvram + fi + + grub2-mkconfig -o /boot/grub2/grub.cfg + sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub2/grub.cfg + + umount /sys/firmware + types: + - vm + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + suseconnect -d + umount -l /etc/resolv.conf || true + rm /etc/resolv.conf + ln -sf /var/run/netconfig/resolv.conf /etc/resolv.conf + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + # Automatic disk resize + cat << EOF > /etc/systemd/system/incus-growpart.service + [Unit] + Description=Incus - grow root partition + + [Service] + Type=oneshot + ExecStartPre=-/usr/sbin/growpart /dev/sda 2 + ExecStart=/usr/sbin/resize2fs /dev/sda2 + + [Install] + WantedBy=default.target + EOF + systemctl enable incus-growpart + types: + - vm + + - trigger: post-files + action: |- + #!/bin/sh + set -eux + + # By default, sles systems don't check authorized_keys2, + # renable it. + sed -E -i 's/^AuthorizedKeysFile[\t ]+.ssh\/authorized_keys$/AuthorizedKeysFile .ssh\/authorized_keys .ssh\/authorized_keys2/g' /etc/ssh/sshd_config diff --git a/jobs/images.yml b/jobs/images.yml index 552cdb4..56fafb3 100644 --- a/jobs/images.yml +++ b/jobs/images.yml @@ -116,51 +116,6 @@ max-total: 4 matrix-builds: false -- _images_parameters_debian_defaults: &images_parameters_debian_defaults - name: 'image_parameters_debian_defaults' - parameters: - - bool: - name: 'SKIP_BASE_IMAGES' - default: false - - bool: - name: 'SKIP_PROFILE_IMAGES' - default: false - - choice: &images_parameters_arch_filter - name: 'ARCH_FILTER' - choices: - - all - - amd64 - - i386 - - arm64 - - armhf - - ppc64el - - riscv64 - - s390x - - choice: &images_parameters_image_type_filter - name: 'IMAGE_TYPE_FILTER' - choices: - - all - - lxd - - vm - - choice: &images_parameters_profile_filter - name: 'PROFILE_FILTER' - choices: - - all - - ci-node - - developer - - choice: - name: 'RELEASE_FILTER' - choices: - - all - - bullseye - - bookworm - - trixie - - sid - - string: - <<: *images_parameters_GIT_URL - - string: - <<: *images_parameters_GIT_BRANCH - ## Defaults - defaults: name: imagebuilder @@ -186,6 +141,13 @@ - file: credential-id: '0debf23b-191b-4cdf-8a25-04e9a7092a67' variable: LXD_CLIENT_KEY + - text: + credential-id: SLES_REGISTRATION_CODE_amd64 + variable: SLES_REGISTRATION_CODE_amd64 + # When it needs to match ansible_architecture + - text: + credential-id: SLES_REGISTRATION_CODE_amd64 + variable: SLES_REGISTRATION_CODE_x86_64 - inject: {} ## Templates @@ -197,7 +159,7 @@

Job is managed by Jenkins Job Builder

project-type: pipeline - <<: *images_parameters_debian_defaults + parameters: '{obj:parameters}' IMAGE_TYPES: - lxd - vm @@ -247,6 +209,91 @@ - sid jobs: - 'images_imagebuilder_{OS}' + parameters: + - bool: &images_parameters_SKIP_BASE_IMAGES + name: 'SKIP_BASE_IMAGES' + default: false + - bool: &images_parameters_SKIP_PROFILE_IMAGES + name: 'SKIP_PROFILE_IMAGES' + default: false + - choice: &images_parameters_ARCH_FILTER + name: 'ARCH_FILTER' + choices: + - all + - amd64 + - i386 + - arm64 + - armhf + - ppc64el + - riscv64 + - s390x + - choice: &images_parameters_IMAGE_TYPE_FILTER + name: 'IMAGE_TYPE_FILTER' + choices: + - all + - lxd + - vm + - choice: &images_parameters_PROFILE_FILTER + name: 'PROFILE_FILTER' + choices: + - all + - ci-node + - developer + - choice: + name: 'RELEASE_FILTER' + choices: + - all + - bullseye + - bookworm + - trixie + - sid + - string: + <<: *images_parameters_GIT_URL + - string: + <<: *images_parameters_GIT_BRANCH + +- project: + name: images_imagebuilder_SLES + OS: + - sles + ARCHES: + - amd64 + RELEASES: + # 12.5 Doesn't currently work well with LXD either as a container or a VM + # * 12.5 has systemd 228, which doesn't support cgroups v2. Hosts _could_ + # be configured to disable the unified cgroup hierarchy in order to have + # this version systemd work correctly. + # * Running as a QEMU VM, virtio_scsi isn't correctly able to bring the + # virtual disks online. The version of lxd currently deployed doesn't + # support the io.bus option to use virtio-blk instead. + # - '12.5' + - '15.4' + parameters: + - bool: + <<: *images_parameters_SKIP_BASE_IMAGES + default: true + - bool: + <<: *images_parameters_SKIP_PROFILE_IMAGES + - choice: + <<: *images_parameters_ARCH_FILTER + choices: + - all + - amd64 + - choice: + <<: *images_parameters_IMAGE_TYPE_FILTER + - choice: + <<: *images_parameters_PROFILE_FILTER + - choice: + name: 'RELEASE_FILTER' + choices: + - 'all' + - '15.4' + - string: + <<: *images_parameters_GIT_URL + - string: + <<: *images_parameters_GIT_BRANCH + jobs: + - 'images_imagebuilder_{OS}' - project: name: images_basejobs jobs: diff --git a/pipelines/images/distrobuild.sh b/pipelines/images/distrobuild.sh index a4e6a6c..1500182 100644 --- a/pipelines/images/distrobuild.sh +++ b/pipelines/images/distrobuild.sh @@ -52,7 +52,7 @@ if [[ ! "${MISSING_VARS}" == "0" ]] ; then fi # Optional variables -INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-30}" +INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-60}" VM_ARG=() # Install lxd-client @@ -129,6 +129,16 @@ if [[ "${IMAGE_FILE}" == "" ]] ; then fail 1 "Unable to find image file for '${OS}' in ${IMAGE_DIRS[@]}" fi +if grep -q -E 'XX[A-Za-z0-9_]+XX' "${IMAGE_FILE}" ; then + while read -r VAR ; do + echo "${VAR}" + SHELLVAR=$(echo "${VAR}" | sed 's/^XX//g' | sed 's/XX$//g') + set +x + sed -i "s/${VAR}/${!SHELLVAR:-VARIABLENOTFOUND}/g" "${IMAGE_FILE}" + set -x + done < <(grep -E -o 'XX[A-Za-z0-9_]+XX' "${IMAGE_FILE}") +fi + DISTROBUILDER_ARGS=( distrobuilder build-incus diff --git a/pipelines/images/imagebuild.sh b/pipelines/images/imagebuild.sh index ecc192d..8d164ae 100644 --- a/pipelines/images/imagebuild.sh +++ b/pipelines/images/imagebuild.sh @@ -49,7 +49,7 @@ if [[ ! "${MISSING_VARS}" == "0" ]] ; then fi # Default optional variables -INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-30}" +INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-60}" NETWORK_SLEEP="${NETWORK_SLEEP:-15}" # Dependencies @@ -189,11 +189,9 @@ LANG=C ANSIBLE_STRATEGY=linear ansible-playbook site.yml \ -l "${INSTANCE_IP}" -i fake-inventory # Cleanup instance side -# @TODO: Distro switch for apt/dnf/yum/etc. -lxc exec "${INSTANCE_NAME}" -- apt-get clean -lxc exec "${INSTANCE_NAME}" -- rm -rf /root/.ssh/authorized_keys2 -lxc exec "${INSTANCE_NAME}" -- cloud-init clean -lxc exec "${INSTANCE_NAME}" -- bash -c 'history -cw' +LANG=C ANSIBLE_STRATEGY=linear ansible-playbook \ + playbooks/post-imagebuild-clean.yml \ + -l "${INSTANCE_IP}" -i fake-inventory # Publish lxc publish "${INSTANCE_NAME}" --alias "${TARGET_IMAGE_NAME}" -f -- 2.34.1