From: Kienan Stewart Date: Wed, 9 Aug 2023 15:52:54 +0000 (-0400) Subject: ansible: Add and manage replacement git-mirror instance X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=09bc9215c416b482a0ef2f0d701e1d25a314c34b;p=lttng-ci.git ansible: Add and manage replacement git-mirror instance Signed-off-by: Kienan Stewart Change-Id: Ib38ceeca6f83b05e975ad7013fcd6d2d52e793f3 --- diff --git a/automation/ansible/host_vars/git-mirror02.yml b/automation/ansible/host_vars/git-mirror02.yml new file mode 100644 index 0000000..f5961e8 --- /dev/null +++ b/automation/ansible/host_vars/git-mirror02.yml @@ -0,0 +1,61 @@ +--- +cgit_sections: + - section: git + clone-url: git://git-mirror.internal.efficios.com/git/$CGIT_REPO_URL + scan-path: /storage/git-mirror/gitdaemon/export/git + - section: kernel + clone-url: git://git-mirror.internal.efficios.com/kernel/$CGIT_REPO_URL + project-list: /storage/git-mirror/gitdaemon/kernel/projects.list + scan-path: /storage/git-mirror/gitdaemon/export/kernel + - section: lttng + clone-url: git://git-mirror.internal.efficios.com/lttng/$CGIT_REPO_URL + project-list: /storage/git-mirror/gitdaemon/lttng/projects.list + scan-path: /storage/git-mirror/gitdaemon/export/lttng + - section: efficios + clone-url: git://git-mirror.internal.efficios.com/efficios/$CGIT_REPO_URL + project-list: /storage/git-mirror/gitdaemon/efficios/projects.list + scan-path: /storage/git-mirror/gitdaemon/export/efficios +cgit_export_links: + efficios: '../efficios' + lttng: '../lttng' + git: '../git' + kernel: '../kernel/pub/scm/linux/kernel/git' + +git_daemon_conf: + GIT_DAEMON_ENABLE: 'true' + GIT_DAEMON_USER: 'gitdaemon' + GIT_DAEMON_OPTIONS: '"--export-all --max-connections=128"' + GIT_DAEMON_BASE_PATH: '/storage/git-mirror/gitdaemon/export' + GIT_DAEMON_DIRECTORY: '/storage/git-mirror/gitdaemon/export' + +grokmirror_instances: + efficios.com: + core: + toplevel: '/storage/git-mirror/gitdaemon/efficios' + log: '/var/log/grokmirror/efficioscom.log' + remote: + site: 'https://git.efficios.com' + pull: + exclude: '*/deliverable/*' + lttng.org: + core: + toplevel: '/storage/git-mirror/gitdaemon/lttng' + log: '/var/log/grokmirror/lttngorg.log' + remote: + site: 'https://git.lttng.org' + kernel.org: + core: + toplevel: '/storage/git-mirror/gitdaemon/kernel' + log: '/var/log/grokmirror/kernelorg.log' + remote: + site: 'https://git.kernel.org' + pull: + projectslist_trimtop: '/pub/scm/linux/kernel/git' + include: + - '/pub/scm/linux/kernel/git/torvalds/linux.git' + - '/pub/scm/linux/kernel/git/stable/linux.git' + - '/pub/scm/linux/kernel/git/next/linux-next.git' + - '/pub/scm/linux/kernel/git/rt/linux-rt-devel.git' + - '/pub/scm/linux/kernel/git/rt/linux-stable-rt.git' + - '/pub/scm/linux/kernel/git/rseq/linux-rseq.git' + refresh: '3600' diff --git a/automation/ansible/hosts b/automation/ansible/hosts index 22f95ab..1b18de0 100644 --- a/automation/ansible/hosts +++ b/automation/ansible/hosts @@ -182,3 +182,6 @@ node [py3_hosts:vars] ansible_python_interpreter=python3 + +[gitmirror] +git-mirror02 \ No newline at end of file diff --git a/automation/ansible/roles/gitmirror/defaults/main.yml b/automation/ansible/roles/gitmirror/defaults/main.yml new file mode 100644 index 0000000..6d7bdaf --- /dev/null +++ b/automation/ansible/roles/gitmirror/defaults/main.yml @@ -0,0 +1,73 @@ +--- +cgit_configuration: + - ['root-title', 'EfficiOS local git mirror'] + - ['root-desc', 'mirror of common repositories used by the CI'] + - ['enable-git-config', '1'] + - ['enable-index-links', '1'] + - ['enable-log-filecount', '1'] + - ['enable-log-linecount', '1'] + - ['css', '/cgit-css/cgit.css'] + - ['logo', '/cgit-css/cgit.png'] + - ['favicon', '/cgit-css/favicon.ico'] +cgit_export_links: {} +cgit_sections: {} +cgit_tls_cert: '/etc/ssl/certs/internal.efficios.com.pem' +cgit_tls_key: '/etc/ssl/private/internal.efficios.com.key' + +git_daemon_conf: + GIT_DAEMON_ENABLE: 'false' + GIT_DAEMON_USER: 'gitdaemon' + GIT_DAEMON_BASE_PATH: '/var/lib' + GIT_DAEMON_DIRECTORY: '/var/lib/git' + +grokmirror_instances: {} +grokmirror_instance_defaults: + core: + manifest: '${toplevel}/manifest.js.gz' + loglevel: 'info' + objstore: '${toplevel}/objstore' + manifest: + pretty: 'no' + fetch_objstore: 'no' + ignore: + - /testing/* + - /private/* + check_export_ok: 'no' + remote: + manifest: '${site}/manifest.js.gz' + pull: + projectslist: '${core:toplevel}/projects.list' + projectslist_trimtop: '' + post_update_hook: '' + purge: 'yes' + purgeprotect: '5' + default_owner: 'Grokmirror User' + remotename: '_grokmirror' + pull:threads: '2' + retries: '3' + include: '*' + refresh: '600' + fsck: + frequency: '30' + statusfile: '${core:toplevel}/fsck.status.js' + ignore_errors: + - 'notice:' + - 'warning: disabling bitmap writing' + - 'ignoring extra bitmap file' + - 'missingTaggerEntry' + - 'missingSpaceBeforeDate' + reclone_on_errors: + - 'fatal: bad tree object' + - 'fatal: Failed to traverse parents' + - 'missing commit' + - 'missing blob' + - 'missing tree' + - 'broken link' + repack: 'yes' + extra_repack_flags: '' + extra_repack_flags_full: '--window=250 --depth=50' + commitgraph: 'yes' + prune: 'yes' + precious: 'yes' + baselines: '' + islandcores: '' diff --git a/automation/ansible/roles/gitmirror/files/grokmirror_2.0.11-1_all.deb b/automation/ansible/roles/gitmirror/files/grokmirror_2.0.11-1_all.deb new file mode 100644 index 0000000..e12f34f Binary files /dev/null and b/automation/ansible/roles/gitmirror/files/grokmirror_2.0.11-1_all.deb differ diff --git a/automation/ansible/roles/gitmirror/files/update.sh b/automation/ansible/roles/gitmirror/files/update.sh new file mode 100644 index 0000000..d4379db --- /dev/null +++ b/automation/ansible/roles/gitmirror/files/update.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -eu + +# sudo -u gitdaemon git clone --mirror git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git + +update_git() { + local repodir="$1" + local origin="$2" + + if [ ! -d "${repodir}" ] ; then + git clone --bare "${origin}" "${repodir}" + fi + + pushd "$repodir" + + git remote update + #git gc + mkdir -p info/web + git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' --count=1 >info/web/last-modified + + popd +} + +## +# Vanilla composite repo +## + +update_git linux-all.git/ https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + +pushd linux-all.git/ +## Add stable if needed +if ! git remote | grep -q stable ; then + git remote add stable https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git + git fetch stable +fi +## Delete broken tree tags +git tag -d v2.6.11 || true +git tag -d v2.6.11-tree || true +popd + +## +# Ubuntu kernels +## + +update_git ubuntu-xenial.git/ https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/xenial + +update_git ubuntu-bionic.git/ git://kernel.ubuntu.com/ubuntu/ubuntu-bionic.git + +update_git ubuntu-focal.git/ git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/focal + +update_git ubuntu-jammy.git/ git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/jammy + +## +# GDB repos +## + +update_git binutils-gdb.git/ git://sourceware.org/git/binutils-gdb.git diff --git a/automation/ansible/roles/gitmirror/handlers/main.yml b/automation/ansible/roles/gitmirror/handlers/main.yml new file mode 100644 index 0000000..ca512a8 --- /dev/null +++ b/automation/ansible/roles/gitmirror/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Restart apache + ansible.builtin.systemd: + name: apache2 + state: restarted +- name: Restart git-daemon + ansible.builtin.systemd: + name: git-daemon + state: restarted +- name: Restart grok-pull + ansible.builtin.systemd: + name: "grok-pull@{{name}}" + state: restarted diff --git a/automation/ansible/roles/gitmirror/tasks/main.yml b/automation/ansible/roles/gitmirror/tasks/main.yml new file mode 100644 index 0000000..e726602 --- /dev/null +++ b/automation/ansible/roles/gitmirror/tasks/main.yml @@ -0,0 +1,137 @@ +--- +- name: Install cgit + ansible.builtin.apt: + name: ['cgit', 'apache2'] +- include_role: + name: common + tasks_from: certs +- name: Configure cgit + ansible.builtin.template: + src: cgitrc.j2 + dest: /etc/cgitrc +- name: Configure apache2 + block: + - name: Enable cgit configuration + ansible.builtin.command: + argv: ['a2enconf', 'cgit.conf'] + creates: /etc/apache2/conf-enabled/cgit.conf + notify: + - Restart apache + - name: Disable default site + ansible.builtin.command: + argv: ['a2dissite', '000-default.conf'] + removes: /etc/apache2/sites-enabled/000-default.conf + notify: + - Restart apache + - name: Enable https + ansible.builtin.command: + argv: ['a2enmod', 'ssl'] + creates: '/etc/apache2/mods-enabled/ssl.conf' + notify: + - Restart apache + - name: Enable cgid + ansible.builtin.command: + argv: ['a2enmod', 'cgid'] + creates: '/etc/apache2/mods-enabled/cgid.conf' + notify: + - Restart apache + - name: Manage apache2 cgit site configuration + ansible.builtin.template: + src: apache2.conf.j2 + dest: /etc/apache2/sites-available/cgit.conf + notify: + - Restart apache + - name: Enable cgit site configuration + ansible.builtin.command: + argv: ['a2ensite', 'cgit.conf'] + creates: /etc/apache2/sites-enabled/cgit.conf + notify: + - Restart apache +- name: Install git-daemon + ansible.builtin.apt: + name: ['git-daemon-sysvinit'] +- name: Create git-daemon base path + when: git_daemon_conf.GIT_DAEMON_BASE_PATH + ansible.builtin.file: + path: "{{git_daemon_conf.GIT_DAEMON_BASE_PATH}}" + state: directory + owner: gitdaemon + group: gitdaemon +- name: Configure git-daemon + with_items: "{{git_daemon_conf|dict2items}}" + community.general.ini_file: + path: /etc/default/git-daemon + option: "{{item.key}}" + value: "{{item.value}}" + no_extra_spaces: true + notify: + - Restart git-daemon +- name: Create git-daemon group + ansible.builtin.group: + system: true + name: gitdaemon +- name: Add www-data to the git-daemon group + ansible.builtin.user: + name: www-data + append: true + groups: ['gitdaemon'] +- name: Install grokmirror + block: + - ansible.builtin.copy: + src: files/grokmirror_2.0.11-1_all.deb + dest: /root/grokmirror.deb + register: grokmirror_deb + - when: grokmirror_deb.changed + ansible.builtin.command: + argv: ['apt', 'install', '-y', '/root/grokmirror.deb'] + - ansible.builtin.file: + state: directory + dest: /etc/grokmirror +- name: Create grokmirror log directory + ansible.builtin.file: + state: directory + dest: /var/log/grokmirror + owner: gitdaemon + group: root + mode: '0755' +- ansible.builtin.file: + state: directory + dest: /storage/git-mirror/gitdaemon + owner: gitdaemon + group: gitdaemon +- name: Create grokmirror configs + include_tasks: + file: mirror_instance.yml + with_items: "{{grokmirror_instances|dict2items}}" + vars: + name: "{{item.key}}" + mirror: "{{item.value}}" +- name: Create legacy pull directory + ansible.builtin.file: + state: directory + dest: /storage/git-mirror/gitdaemon/git + owner: gitdaemon + group: gitdaemon +- name: Create legacy pull script + ansible.builtin.copy: + src: update.sh + dest: /storage/git-mirror/gitdaemon/git/update.sh + mode: '0750' + owner: gitdaemon + group: gitdaemon +- name: Install cron + ansible.builtin.apt: + name: cron +- name: Manage legacy pull cron job + ansible.builtin.cron: + name: gitmirror + cron_file: gitmirror + minute: '*/30' + job: 'cd /storage/git-mirror/gitdaemon/git && ./update.sh >/dev/null 2>&1' + user: gitdaemon +- name: Link cgit export directories + with_items: "{{cgit_export_links|dict2items}}" + ansible.builtin.file: + state: link + src: "{{item.value}}" + path: "/storage/git-mirror/gitdaemon/export/{{item.key}}" diff --git a/automation/ansible/roles/gitmirror/tasks/mirror_instance.yml b/automation/ansible/roles/gitmirror/tasks/mirror_instance.yml new file mode 100644 index 0000000..22c7703 --- /dev/null +++ b/automation/ansible/roles/gitmirror/tasks/mirror_instance.yml @@ -0,0 +1,22 @@ +--- +- name: Create storage directory + ansible.builtin.file: + path: "{{mirror.core.toplevel}}" + state: directory + owner: gitdaemon + group: gitdaemon + +- name: Deploy config file + ansible.builtin.template: + dest: "/etc/grokmirror/{{name}}.conf" + src: grokmirror.conf.j2 + vars: + name: "{{name}}" + data: "{{grokmirror_instance_defaults|combine(mirror, recursive=true)}}" + notify: + - Restart grok-pull + +- name: Systemd service + ansible.builtin.systemd: + name: "grok-pull@{{name}}" + state: started diff --git a/automation/ansible/roles/gitmirror/templates/apache2.conf.j2 b/automation/ansible/roles/gitmirror/templates/apache2.conf.j2 new file mode 100644 index 0000000..d302abf --- /dev/null +++ b/automation/ansible/roles/gitmirror/templates/apache2.conf.j2 @@ -0,0 +1,25 @@ + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + RedirectMatch ^/$ /cgit/ + + +{% if cgit_tls_key and cgit_tls_cert %} + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + SSLEngine on + SSLCertificateFile {{cgit_tls_cert}} + SSLCertificateKeyFile {{cgit_tls_key}} + + RedirectMatch ^/$ /cgit/ + + + +{% endif %} \ No newline at end of file diff --git a/automation/ansible/roles/gitmirror/templates/cgitrc.j2 b/automation/ansible/roles/gitmirror/templates/cgitrc.j2 new file mode 100644 index 0000000..e584e5e --- /dev/null +++ b/automation/ansible/roles/gitmirror/templates/cgitrc.j2 @@ -0,0 +1,10 @@ +{% for entry in cgit_configuration %} +{{entry[0]}}={{entry[1]}} +{% endfor %} + +{% for section in cgit_sections %} +{% for key, value in section.items() %} +{{key}}={{value}} +{% endfor %} + +{% endfor %} \ No newline at end of file diff --git a/automation/ansible/roles/gitmirror/templates/grokmirror.conf.j2 b/automation/ansible/roles/gitmirror/templates/grokmirror.conf.j2 new file mode 100644 index 0000000..cd8ac11 --- /dev/null +++ b/automation/ansible/roles/gitmirror/templates/grokmirror.conf.j2 @@ -0,0 +1,16 @@ +# {{name}} +# +{% for section, conf in data.items() %} +[{{section}}] +{% for key, value in conf.items() %} +{% if value is not string and value is iterable %} +{{key}} = {{value[0]}} +{% for i in range(1, value|length) %} + {{value[i]}} +{% endfor %} +{% else %} +{{key}} = {{value}} +{% endif %} +{% endfor %} + +{% endfor %} \ No newline at end of file diff --git a/automation/ansible/site.yml b/automation/ansible/site.yml index b946476..af3d745 100644 --- a/automation/ansible/site.yml +++ b/automation/ansible/site.yml @@ -10,3 +10,7 @@ - import_playbook: node_standalone.yml - import_playbook: node_sles.yml - import_playbook: node_el.yml +- hosts: gitmirror + roles: + - common + - gitmirror diff --git a/automation/ansible/vars/ci-instances.yml b/automation/ansible/vars/ci-instances.yml index fba5728..d69f3c0 100644 --- a/automation/ansible/vars/ci-instances.yml +++ b/automation/ansible/vars/ci-instances.yml @@ -82,6 +82,22 @@ libvirt_host: ci-host-amd64-1b.internal.efficios.com # @note: `meta.address` doesn't set any additional information for libvirt VMs. # containers: + - meta: + address: 172.18.18.1 + lxd: + name: git-mirror02 + target: ci-host-amd64-1c + config: + # Using a privileged container here means we can avoid doing + # uid/gid mapping and reduce the work done during file IO + security.privileged: 'true' + devices: + git-mirror: + path: /storage/git-mirror + # This was created manually: + # zfs create -o mountpoint=/storage/git-mirror tank/lxd/custom/git-mirror + source: /storage/git-mirror + type: disk - meta: address: 172.18.16.1 lxd: