ansible/playbooks/deploy_kube.yml
2026-10-11 22:29:14 +02:00

231 lines
7.5 KiB
YAML

---
# Build, upgrade or reconfigure the k3s cluster (k3s-design.md): 3 masters with
# embedded etcd, 3 workers carrying Longhorn. Safe to rerun: on the running
# cluster it changes nothing unless the inventory says otherwise.
#
# ansible-playbook playbooks/deploy_kube.yml -CD # dry run first
# ansible-playbook playbooks/deploy_kube.yml
# ansible-playbook playbooks/deploy_kube.yml --tags kubeconfig # only refresh kube/config
#
# Upgrade: bump k3s_version in group_vars/k3s.yml and rerun. The cluster-init
# master goes first, then the other masters, then the workers, one node at a
# time, each waiting for Ready. Restarting k3s leaves the pods running.
#
# Creating the VMs themselves is not done here (k3s-build-vms.md).
- name: Check the k3s inventory
hosts: k3s
gather_facts: true
tags: [always]
tasks:
- name: Only VMs can be k3s nodes
ansible.builtin.assert:
that: host_type == 'vm'
fail_msg: "{{ inventory_hostname }} is a {{ host_type }}. k3s needs a VM (own kernel, cgroups, iSCSI for Longhorn)."
quiet: true
- name: Exactly one master bootstraps the cluster
ansible.builtin.assert:
that: groups['k3s_masters'] | map('extract', hostvars) | selectattr('k3s_cluster_init', 'defined') | selectattr('k3s_cluster_init') | list | length == 1
fail_msg: "Set k3s_cluster_init: true on exactly one host of k3s_masters."
quiet: true
run_once: true
- name: Group the cluster-init master
ansible.builtin.group_by:
key: k3s_init
when: k3s_cluster_init | default(false)
changed_when: false
- name: Longhorn prerequisites
hosts: k3s_workers
gather_facts: false
tags: [longhorn]
tasks:
- name: Install iSCSI and NFS clients
ansible.builtin.apt:
name: [open-iscsi, nfs-common]
state: present
update_cache: true
cache_valid_time: 3600
- name: Start iscsid
ansible.builtin.systemd_service:
name: iscsid
enabled: true
state: started
- name: Load iscsi_tcp at boot
ansible.builtin.copy:
dest: /etc/modules-load.d/longhorn.conf
content: "iscsi_tcp\n"
owner: root
group: root
mode: "0644"
- name: Load iscsi_tcp now
community.general.modprobe:
name: iscsi_tcp
state: present
- name: List services
ansible.builtin.service_facts:
# multipathd grabs Longhorn's block devices.
- name: Disable multipathd
ansible.builtin.systemd_service:
name: multipathd.service
enabled: false
state: stopped
when: "'multipathd.service' in ansible_facts.services"
- name: Stop its socket (static unit, can't be disabled)
ansible.builtin.systemd_service:
name: multipathd.socket
state: stopped
when: "'multipathd.service' in ansible_facts.services"
- name: Check whether the Longhorn disk is already mounted
ansible.builtin.command: findmnt -no SOURCE {{ k3s_longhorn_path }}
register: k3s_longhorn_mount
changed_when: false
failed_when: false
check_mode: false
# Formatting only happens on an empty disk of the expected size, like
# argocd/scripts/prep-worker.sh. Anything else stops the play.
- name: Format the empty Longhorn disk
when: k3s_longhorn_mount.stdout != k3s_longhorn_disk
block:
- name: Inspect {{ k3s_longhorn_disk }}
ansible.builtin.shell: |
set -o pipefail
echo "size=$(lsblk -bdno SIZE {{ k3s_longhorn_disk }})"
echo "parts=$(lsblk -no NAME {{ k3s_longhorn_disk }} | tail -n +2 | wc -l)"
echo "fs=$(blkid -o value -s TYPE {{ k3s_longhorn_disk }} || true)"
args:
executable: /bin/bash
register: k3s_disk_info
changed_when: false
check_mode: false
- name: Refuse anything but an empty disk of the expected size
ansible.builtin.assert:
that:
- "'size=' ~ k3s_longhorn_disk_size in k3s_disk_info.stdout_lines"
- "'parts=0' in k3s_disk_info.stdout_lines"
- "'fs=' in k3s_disk_info.stdout_lines"
fail_msg: >-
{{ k3s_longhorn_disk }} is not an empty {{ k3s_longhorn_disk_size }}-byte disk
({{ k3s_disk_info.stdout_lines | join(', ') }}). Not formatting it.
- name: Create the ext4 filesystem
community.general.filesystem:
dev: "{{ k3s_longhorn_disk }}"
fstype: ext4
opts: -L longhorn
- name: Mount the Longhorn disk
ansible.posix.mount:
src: LABEL=longhorn
path: "{{ k3s_longhorn_path }}"
fstype: ext4
opts: defaults,noatime
dump: "0"
passno: "2"
state: mounted
- name: First master
hosts: k3s_init
gather_facts: false
tags: [k3s]
vars:
k3s_role: server
k3s_service: k3s
k3s_init_host: "{{ inventory_hostname }}"
tasks:
# Without a vaulted k3s_token, a new cluster generates its own.
- name: Use the vaulted token
ansible.builtin.set_fact:
k3s_join_token: "{{ k3s_token }}"
no_log: true
when: k3s_token is defined
- name: Install or update k3s
ansible.builtin.include_tasks: tasks/k3s-node.yml
- name: Join token
hosts: k3s
gather_facts: false
tags: [k3s]
vars:
k3s_init_host: "{{ groups['k3s_init'] | first }}"
tasks:
- name: Read the token from the first master
ansible.builtin.slurp:
src: /var/lib/rancher/k3s/server/token
delegate_to: "{{ k3s_init_host }}"
run_once: true
register: k3s_token_file
no_log: true
when: k3s_token is not defined
# A dry run of a brand-new cluster has no token yet.
ignore_errors: "{{ ansible_check_mode }}"
- name: Set the join token
ansible.builtin.set_fact:
k3s_join_token: "{{ k3s_token | default(k3s_token_file.content | b64decode | trim) }}"
no_log: true
when: k3s_token is defined or k3s_token_file.content is defined
- name: Other masters
hosts: k3s_masters:!k3s_init
gather_facts: false
serial: 1
tags: [k3s]
vars:
k3s_role: server
k3s_service: k3s
k3s_init_host: "{{ groups['k3s_init'] | first }}"
k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443
tasks:
- name: Install or update k3s
ansible.builtin.include_tasks: tasks/k3s-node.yml
- name: Workers
hosts: k3s_workers
gather_facts: false
serial: 1
tags: [k3s]
vars:
k3s_role: agent
k3s_service: k3s-agent
k3s_init_host: "{{ groups['k3s_init'] | first }}"
k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443
tasks:
- name: Install or update k3s
ansible.builtin.include_tasks: tasks/k3s-node.yml
- name: Admin kubeconfig
hosts: k3s_init
gather_facts: false
tags: [k3s, kubeconfig]
tasks:
- name: Read the admin kubeconfig
ansible.builtin.slurp:
src: /etc/rancher/k3s/k3s.yaml
register: k3s_kubeconfig
no_log: true
# Same server and names as ~/.kube/k3s-homelab.yaml.
- name: Write kube/config
ansible.builtin.copy:
dest: "{{ playbook_dir }}/../kube/config"
content: >-
{{ k3s_kubeconfig.content | b64decode
| replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443')
| regex_replace('(?m)^(\s*(?:- )?(?:name|cluster|user|current-context): )default$', '\1k3s-homelab') }}
mode: "0600"
delegate_to: localhost
become: false
diff: false # client certificate and key