From 455bd8ef6b90f5686eb69fcfcbd6860930a45b87 Mon Sep 17 00:00:00 2001 From: Kienan Stewart Date: Mon, 31 Jul 2023 17:21:49 -0400 Subject: [PATCH] ansible: Add rootnodes in new cluster Change-Id: I3fcb71c3d84c82b570a5411747effe8bc17d4736 --- automation/ansible/README.md | 2 +- automation/ansible/group_vars/hosts.yml | 3 + automation/ansible/hosts | 11 +- automation/ansible/playbooks/ci-instances.yml | 27 +- .../ansible/playbooks/vm_template.xml.j2 | 1 + .../ansible/roles/libvirt/tasks/main.yml | 10 +- automation/ansible/roles/libvirt/tasks/vm.yml | 19 ++ .../ansible/scripts/update_jenkins_node.py | 24 +- automation/ansible/vars/ci-instances.yml | 244 ++++++++++++++++++ 9 files changed, 321 insertions(+), 20 deletions(-) create mode 100644 automation/ansible/group_vars/hosts.yml create mode 120000 automation/ansible/playbooks/vm_template.xml.j2 create mode 100644 automation/ansible/roles/libvirt/tasks/vm.yml diff --git a/automation/ansible/README.md b/automation/ansible/README.md index e75e72b..fac4b7c 100644 --- a/automation/ansible/README.md +++ b/automation/ansible/README.md @@ -141,4 +141,4 @@ To automatically provision instances, perform certain operations, and update DNS 1. Update `vars/ci-instances.yml` 2. Open a kerberos ticket with `kinit` -3. Run the playbook, eg. `ansible-playbook -l ci-host-XX.internal.efficios.com playbooks/ci-instances.yml` +3. Run the playbook, eg. `ansible-playbook playbooks/ci-instances.yml` diff --git a/automation/ansible/group_vars/hosts.yml b/automation/ansible/group_vars/hosts.yml new file mode 100644 index 0000000..9b5fabc --- /dev/null +++ b/automation/ansible/group_vars/hosts.yml @@ -0,0 +1,3 @@ +--- +libvirt_extra_users: + - jenkins diff --git a/automation/ansible/hosts b/automation/ansible/hosts index fb74e55..22f95ab 100644 --- a/automation/ansible/hosts +++ b/automation/ansible/hosts @@ -141,8 +141,17 @@ ci-rootnode-bionic-amd64-01 ci-rootnode-bionic-amd64-02 ci-rootnode-bionic-amd64-03 ci-rootnode-bionic-amd64-06 -ci-rootnode-deb11-i386-01 ci-rootnode-jammy-amd64-07-01 +ci-rootnode-deb11-i386-01 +ci-rootnode-deb12-amd64-1a-01 +ci-rootnode-deb12-amd64-1b-01 +ci-rootnode-deb12-amd64-1c-01 +ci-rootnode-deb12-amd64-1d-01 +ci-rootnode-deb12-amd64-2a-01 +ci-rootnode-deb12-amd64-2b-01 +ci-rootnode-deb12-amd64-2c-01 +ci-rootnode-deb12-amd64-2d-01 +ci-rootnode-deb12-i386-1a-01 ci-rootnode-deb12-i386-01-01 [node_sles] diff --git a/automation/ansible/playbooks/ci-instances.yml b/automation/ansible/playbooks/ci-instances.yml index a4619d5..30d5d42 100644 --- a/automation/ansible/playbooks/ci-instances.yml +++ b/automation/ansible/playbooks/ci-instances.yml @@ -1,9 +1,11 @@ --- -- hosts: "{{lxd_host}}" +- hosts: lxd_cluster_ci:localhost vars_files: - ../vars/ci-instances.yml - ../roles/lxd/defaults/main.yml + - ../roles/libvirt/vars/main.yml vars: + skip_libvirt: false skip_lxd: false skip_dns: false skip_jenkins: false @@ -11,7 +13,7 @@ jenkins_default_credentials: "c3e4f9f2-3e89-474d-bc75-6251a13e1053" tasks: - name: Manage instances - when: not skip_lxd + when: item.lxd|default(false) and not skip_lxd and inventory_hostname == lxd_host include_tasks: file: '../roles/lxd/tasks/container.yml' vars: @@ -26,30 +28,36 @@ recursive=true )}} with_items: "{{containers}}" + - name: Manage Libvirt VMs + when: item.vm|default(false) and not skip_libvirt and inventory_hostname == item.vm.host|default(libvirt_host) + include_tasks: + file: '../roles/libvirt/tasks/vm.yml' + vars: + object: "{{item.vm}}" + with_items: "{{containers}}" - name: Update DNS entries - delegate_to: localhost - when: not skip_dns + when: not skip_dns and inventory_hostname == 'localhost' with_items: "{{containers}}" ansible.builtin.command: argv: [ - '../scripts/update_dns_entry.py', '-n', "{{item.lxd.name}}", + '../scripts/update_dns_entry.py', '-n', "{{item.lxd.name|default(item.vm.name)}}", '-z', "{{search_domain}}", '-v', "{{item.meta.address}}", '-s', "{{name_server}}", ] - name: Update Jenkins nodes - delegate_to: localhost - when: not skip_jenkins + when: not skip_jenkins and inventory_hostname == 'localhost' with_items: "{{containers}}" vars: - node_name: "{{item.jenkins.node_name|default(item.lxd.name)}}" + node_name: "{{item.jenkins.node_name|default(item.lxd.name|default(item.vm.name))}}" node_ip: "{{item.meta.address|default(None)}}" - node_host: "{{item.meta.jenkins.node_host|default(item.lxd.name + '.' + search_domain)}}" + node_host: "{{item.meta.jenkins.node_host|default(item.lxd.name|default(item.vm.name) + '.' + search_domain)}}" node_label: "{{item.jenkins.label|default('')}}" node_state: "{{item.jenkins.state|default('online')}}" node_credentials: "{{item.jenkins.credentials|default(jenkins_default_credentials)}}" node_message: "{{item.jenkins.message|default('Set offline by ansible')}}" node_mode: "{{item.jenkins.mode|default('NORMAL')}}" + node_json: "{{item.jenkins.config|default({})|to_json}}" ansible.builtin.command: argv: [ '../scripts/update_jenkins_node.py', '-n', "{{node_name}}", @@ -60,4 +68,5 @@ '-c', "mode={{node_mode}}", '-s', "{{node_state}}", '-m', "{{node_message}}", '-f', "{{jenkins_config|expanduser}}", + '-j', "{{node_json}}", ] diff --git a/automation/ansible/playbooks/vm_template.xml.j2 b/automation/ansible/playbooks/vm_template.xml.j2 new file mode 120000 index 0000000..29b0542 --- /dev/null +++ b/automation/ansible/playbooks/vm_template.xml.j2 @@ -0,0 +1 @@ +../roles/libvirt/templates/vm_template.xml.j2 \ No newline at end of file diff --git a/automation/ansible/roles/libvirt/tasks/main.yml b/automation/ansible/roles/libvirt/tasks/main.yml index a23ce3b..d773102 100644 --- a/automation/ansible/roles/libvirt/tasks/main.yml +++ b/automation/ansible/roles/libvirt/tasks/main.yml @@ -1,12 +1,20 @@ --- - name: Install virtualization packages apt: - name: ['qemu-kvm', 'libvirt-daemon-system'] + name: ['qemu-kvm', 'libvirt-daemon-system', 'python3-libvirt'] - name: Run libvirtd systemd: name: libvirtd enabled: true state: started +- name: Configure extra libvirt user groups + ansible.builtin.user: + name: "{{item}}" + groups: + - libvirt + - libvirt-qemu + append: true + with_items: "{{libvirt_extra_users|default([])}}" - name: Download ISOs loop: "{{ isos }}" get_url: diff --git a/automation/ansible/roles/libvirt/tasks/vm.yml b/automation/ansible/roles/libvirt/tasks/vm.yml new file mode 100644 index 0000000..bcf0407 --- /dev/null +++ b/automation/ansible/roles/libvirt/tasks/vm.yml @@ -0,0 +1,19 @@ +--- +- name: Merge defaults + set_fact: + vm: "{{vm_defaults|combine(object)}}" +- name: Create VM disk + when: vm.disk + ansible.builtin.command: + argv: [ + 'qemu-img', 'create', '-f', 'qcow2', + "{{vm.disk}}", "{{vm.disk_capacity}}", + ] + creates: "{{vm.disk}}" +- name: Define VM + # Note: is vm.uuid is not set and the template is changed, those changes will not be applied + # Note: many changes will require the VM to be destroyed then started again + community.libvirt.virt: + command: define + xml: "{{lookup('template', vm.template|default('vm_template.xml.j2'))}}" + autostart: true diff --git a/automation/ansible/scripts/update_jenkins_node.py b/automation/ansible/scripts/update_jenkins_node.py index 9c4b734..02df70e 100755 --- a/automation/ansible/scripts/update_jenkins_node.py +++ b/automation/ansible/scripts/update_jenkins_node.py @@ -2,6 +2,7 @@ import argparse import configparser +import json import sys import xml.etree.ElementTree @@ -45,6 +46,10 @@ def get_argument_parser(): '-m', '--message', default='', help='A message to set for the offline reason of a node' ) + parser.add_argument( + '-j', '--json', default='', + help='Additional config in a json dictionary which will be appended after -c items', + ) return parser @@ -101,14 +106,12 @@ def manage_node(url, user, password, node, state, offline_message='', config={}) element.attrib[value['attrib']] = value['value'] updated = True if updated: - server.reconfig_node( - node, - xml.etree.ElementTree.tostring( - node_config, - xml_declaration=True, - encoding='unicode' - ) + xml_string = xml.etree.ElementTree.tostring( + node_config, + xml_declaration=True, + encoding='unicode' ) + server.reconfig_node(node, xml_string) changed = True # Online/offline node_info = server.get_node_info(node) @@ -146,7 +149,12 @@ if __name__ == '__main__': 'attrib': value.split('=', 1)[1] if '=' in value else None, 'value': value.split('=', 1)[0] if '=' in value else value, } - print(node_config) + if args.json: + for key, value in json.loads(args.json).items(): + node_config[key] = { + 'attrib': value.split('=', 1)[1] if '=' in value else None, + 'value': value.split('=', 1)[0] if '=' in value else value, + } manage_node( args.url, args.user, args.password, args.node, args.state, args.message, node_config diff --git a/automation/ansible/vars/ci-instances.yml b/automation/ansible/vars/ci-instances.yml index 56170b6..fba5728 100644 --- a/automation/ansible/vars/ci-instances.yml +++ b/automation/ansible/vars/ci-instances.yml @@ -5,6 +5,8 @@ search_domain: internal.efficios.com name_server: smb-adc02.internal.efficios.com # The host to use for delegating lxd commands lxd_host: ci-host-amd64-1a.internal.efficios.com +# Default host to create libvirt VMs on if not specified +libvirt_host: ci-host-amd64-1b.internal.efficios.com # @see https://docs.ansible.com/ansible/latest/collections/community/general/lxd_container_module.html#ansible-collections-community-general-lxd-container-module # @example a container instance with the default image (deb12 amd64) @@ -67,6 +69,18 @@ lxd_host: ci-host-amd64-1a.internal.efficios.com # Text console (may not have output depending on boot settings) # lxc console ci:ci-rootnode-example # +# @example Define a VM using libvirt +# - vm: +# # host: 'ci-host-example' +# name: 'ci-rootnode-example' +# disk: '/path/to/disk' +# +# @note libvirt VMs are created using the task roles/libvirt/vm.yml. +# If `vm.host` not specified, then the host will default to the one defined +# in `libvirt_host`. +# +# @note: `meta.address` doesn't set any additional information for libvirt VMs. +# containers: - meta: address: 172.18.16.1 @@ -124,3 +138,233 @@ containers: jenkins: label: 'deb12-amd64 deb12' mode: EXCLUSIVE + # "Root" nodes in libvirt, since Jenkins can use the "libvirt agents" plugin + # to revert to a specific snapshot before running a job + # + # @NOTE: For Jenkins, the node configuration was initially created by cloning + # or creating the nodes as they use a type of node which isn't 'simple' to + # create through the API. The entire XML configuration would have to be written + # from scratch. In effect, templates may be a better long term solution than + # the current structure. + # + - jenkins: + name: 'ci-rootnode-deb12-amd64-1a-01' + label: 'deb12 deb12-amd64-rootnode' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1a.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-1a-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1a.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-1a-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-1a-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-1a-01.qcow' + host: 'ci-host-amd64-1a.internal.efficios.com' + meta: + address: 172.18.17.1 + - jenkins: + name: 'ci-rootnode-deb12-i386-1a-01' + label: 'deb12 deb12-i386-rootnode' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1a.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-i386-1a-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1a.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-i386-1a-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-i386-1a-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-i386-1a-01.qcow' + host: 'ci-host-amd64-1a.internal.efficios.com' + meta: + address: 172.18.17.2 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-1b-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1b.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-1b-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1b.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-1b-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-1b-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-1b-01.qcow' + host: 'ci-host-amd64-1b.internal.efficios.com' + meta: + address: 172.18.17.3 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-1c-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1c.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-1c-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-1c-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-1c-01.qcow' + host: 'ci-host-amd64-1c.internal.efficios.com' + meta: + address: 172.18.17.4 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-1d-01' + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1d.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-1d-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-1d.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-1d-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-1d-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-1d-01.qcow' + host: 'ci-host-amd64-1d.internal.efficios.com' + meta: + address: 172.18.17.5 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-2a-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2a.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-2a-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2a.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-2a-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-2a-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-2a-01.qcow' + host: 'ci-host-amd64-2a.internal.efficios.com' + meta: + address: 172.18.17.6 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-2b-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2b.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-2b-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2b.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-2b-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-2b-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-2b-01.qcow' + host: 'ci-host-amd64-2b.internal.efficios.com' + meta: + address: 172.18.17.7 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-2c-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2c.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-2c-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2c.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-2c-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-2c-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-2c-01.qcow' + host: 'ci-host-amd64-2c.internal.efficios.com' + meta: + address: 172.18.17.8 + - jenkins: + label: 'deb12 deb12-amd64-rootnode' + name: 'ci-rootnode-deb12-amd64-2d-01' + mode: EXCLUSIVE + config: + remoteFS: '/root' + launcher/hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2d.internal.efficios.com' + launcher/virtualMachineName: 'ci-rootnode-deb12-amd64-2d-01' + launcher/delegate: 'hudson.plugins.sshslaves.SSHLauncher=class' + launcher/delegate/port: '22' + launcher/delegate/host: 'ci-rootnode-deb12-amd64-1a-01.internal.efficios.com' + launcher/delegate/credentialsId: 'bb5a81cf-346b-43fc-8586-3dc5e43801be' + ./hypervisorDescription: 'QEMU+ssh - ci-host-amd64-2d.internal.efficios.com' + ./virtualMachineName: 'ci-rootnode-deb12-amd64-2d-01' + snapshotName: '' + beforeJobSnapshotName: 'base-configuration' + startupWaitingPeriodSeconds: '20' + launcher/waitTimeMs: '20000' + shutdownMethod: 'destroy' + vm: + name: 'ci-rootnode-deb12-amd64-2d-01' + disk: '/var/lib/libvirt/images/ci-rootnode-deb12-amd64-2d-01.qcow' + host: 'ci-host-amd64-2d.internal.efficios.com' + meta: + address: 172.18.17.9 -- 2.34.1