2026-10-11 22:29:14 +02:00
|
|
|
---
|
|
|
|
|
# Build, upgrade or reconfigure the k3s cluster (k3s-design.md): 3 masters with
|
|
|
|
|
# embedded etcd, 3 workers carrying Longhorn. Safe to rerun: on the running
|
|
|
|
|
# cluster it changes nothing unless the inventory says otherwise.
|
|
|
|
|
#
|
|
|
|
|
# ansible-playbook playbooks/deploy_kube.yml -CD # dry run first
|
|
|
|
|
# ansible-playbook playbooks/deploy_kube.yml
|
|
|
|
|
# ansible-playbook playbooks/deploy_kube.yml --tags kubeconfig # only refresh kube/config
|
|
|
|
|
#
|
|
|
|
|
# Upgrade: bump k3s_version in group_vars/k3s.yml and rerun. The cluster-init
|
|
|
|
|
# master goes first, then the other masters, then the workers, one node at a
|
|
|
|
|
# time, each waiting for Ready. Restarting k3s leaves the pods running.
|
|
|
|
|
#
|
|
|
|
|
# Creating the VMs themselves is not done here (k3s-build-vms.md).
|
|
|
|
|
|
|
|
|
|
- name: Check the k3s inventory
|
|
|
|
|
hosts: k3s
|
|
|
|
|
gather_facts: true
|
|
|
|
|
tags: [always]
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Only VMs can be k3s nodes
|
|
|
|
|
ansible.builtin.assert:
|
|
|
|
|
that: host_type == 'vm'
|
|
|
|
|
fail_msg: "{{ inventory_hostname }} is a {{ host_type }}. k3s needs a VM (own kernel, cgroups, iSCSI for Longhorn)."
|
|
|
|
|
quiet: true
|
|
|
|
|
|
|
|
|
|
- name: Exactly one master bootstraps the cluster
|
|
|
|
|
ansible.builtin.assert:
|
|
|
|
|
that: groups['k3s_masters'] | map('extract', hostvars) | selectattr('k3s_cluster_init', 'defined') | selectattr('k3s_cluster_init') | list | length == 1
|
|
|
|
|
fail_msg: "Set k3s_cluster_init: true on exactly one host of k3s_masters."
|
|
|
|
|
quiet: true
|
|
|
|
|
run_once: true
|
|
|
|
|
|
|
|
|
|
- name: Group the cluster-init master
|
|
|
|
|
ansible.builtin.group_by:
|
|
|
|
|
key: k3s_init
|
|
|
|
|
when: k3s_cluster_init | default(false)
|
|
|
|
|
changed_when: false
|
|
|
|
|
|
|
|
|
|
- name: Longhorn prerequisites
|
|
|
|
|
hosts: k3s_workers
|
|
|
|
|
gather_facts: false
|
|
|
|
|
tags: [longhorn]
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Install iSCSI and NFS clients
|
|
|
|
|
ansible.builtin.apt:
|
|
|
|
|
name: [open-iscsi, nfs-common]
|
|
|
|
|
state: present
|
|
|
|
|
update_cache: true
|
|
|
|
|
cache_valid_time: 3600
|
|
|
|
|
|
|
|
|
|
- name: Start iscsid
|
|
|
|
|
ansible.builtin.systemd_service:
|
|
|
|
|
name: iscsid
|
|
|
|
|
enabled: true
|
|
|
|
|
state: started
|
|
|
|
|
|
|
|
|
|
- name: Load iscsi_tcp at boot
|
|
|
|
|
ansible.builtin.copy:
|
|
|
|
|
dest: /etc/modules-load.d/longhorn.conf
|
|
|
|
|
content: "iscsi_tcp\n"
|
|
|
|
|
owner: root
|
|
|
|
|
group: root
|
|
|
|
|
mode: "0644"
|
|
|
|
|
|
|
|
|
|
- name: Load iscsi_tcp now
|
|
|
|
|
community.general.modprobe:
|
|
|
|
|
name: iscsi_tcp
|
|
|
|
|
state: present
|
|
|
|
|
|
|
|
|
|
- name: List services
|
|
|
|
|
ansible.builtin.service_facts:
|
|
|
|
|
|
|
|
|
|
# multipathd grabs Longhorn's block devices.
|
|
|
|
|
- name: Disable multipathd
|
|
|
|
|
ansible.builtin.systemd_service:
|
|
|
|
|
name: multipathd.service
|
|
|
|
|
enabled: false
|
|
|
|
|
state: stopped
|
|
|
|
|
when: "'multipathd.service' in ansible_facts.services"
|
|
|
|
|
|
|
|
|
|
- name: Stop its socket (static unit, can't be disabled)
|
|
|
|
|
ansible.builtin.systemd_service:
|
|
|
|
|
name: multipathd.socket
|
|
|
|
|
state: stopped
|
|
|
|
|
when: "'multipathd.service' in ansible_facts.services"
|
|
|
|
|
|
|
|
|
|
- name: Check whether the Longhorn disk is already mounted
|
|
|
|
|
ansible.builtin.command: findmnt -no SOURCE {{ k3s_longhorn_path }}
|
|
|
|
|
register: k3s_longhorn_mount
|
|
|
|
|
changed_when: false
|
|
|
|
|
failed_when: false
|
|
|
|
|
check_mode: false
|
|
|
|
|
|
|
|
|
|
# Formatting only happens on an empty disk of the expected size, like
|
|
|
|
|
# argocd/scripts/prep-worker.sh. Anything else stops the play.
|
|
|
|
|
- name: Format the empty Longhorn disk
|
|
|
|
|
when: k3s_longhorn_mount.stdout != k3s_longhorn_disk
|
|
|
|
|
block:
|
|
|
|
|
- name: Inspect {{ k3s_longhorn_disk }}
|
|
|
|
|
ansible.builtin.shell: |
|
|
|
|
|
set -o pipefail
|
|
|
|
|
echo "size=$(lsblk -bdno SIZE {{ k3s_longhorn_disk }})"
|
|
|
|
|
echo "parts=$(lsblk -no NAME {{ k3s_longhorn_disk }} | tail -n +2 | wc -l)"
|
|
|
|
|
echo "fs=$(blkid -o value -s TYPE {{ k3s_longhorn_disk }} || true)"
|
|
|
|
|
args:
|
|
|
|
|
executable: /bin/bash
|
|
|
|
|
register: k3s_disk_info
|
|
|
|
|
changed_when: false
|
|
|
|
|
check_mode: false
|
|
|
|
|
|
|
|
|
|
- name: Refuse anything but an empty disk of the expected size
|
|
|
|
|
ansible.builtin.assert:
|
|
|
|
|
that:
|
|
|
|
|
- "'size=' ~ k3s_longhorn_disk_size in k3s_disk_info.stdout_lines"
|
|
|
|
|
- "'parts=0' in k3s_disk_info.stdout_lines"
|
|
|
|
|
- "'fs=' in k3s_disk_info.stdout_lines"
|
|
|
|
|
fail_msg: >-
|
|
|
|
|
{{ k3s_longhorn_disk }} is not an empty {{ k3s_longhorn_disk_size }}-byte disk
|
|
|
|
|
({{ k3s_disk_info.stdout_lines | join(', ') }}). Not formatting it.
|
|
|
|
|
|
|
|
|
|
- name: Create the ext4 filesystem
|
|
|
|
|
community.general.filesystem:
|
|
|
|
|
dev: "{{ k3s_longhorn_disk }}"
|
|
|
|
|
fstype: ext4
|
|
|
|
|
opts: -L longhorn
|
|
|
|
|
|
|
|
|
|
- name: Mount the Longhorn disk
|
|
|
|
|
ansible.posix.mount:
|
|
|
|
|
src: LABEL=longhorn
|
|
|
|
|
path: "{{ k3s_longhorn_path }}"
|
|
|
|
|
fstype: ext4
|
|
|
|
|
opts: defaults,noatime
|
|
|
|
|
dump: "0"
|
|
|
|
|
passno: "2"
|
|
|
|
|
state: mounted
|
|
|
|
|
|
|
|
|
|
- name: First master
|
|
|
|
|
hosts: k3s_init
|
|
|
|
|
gather_facts: false
|
|
|
|
|
tags: [k3s]
|
|
|
|
|
vars:
|
|
|
|
|
k3s_role: server
|
|
|
|
|
k3s_service: k3s
|
|
|
|
|
k3s_init_host: "{{ inventory_hostname }}"
|
|
|
|
|
tasks:
|
|
|
|
|
# Without a vaulted k3s_token, a new cluster generates its own.
|
|
|
|
|
- name: Use the vaulted token
|
|
|
|
|
ansible.builtin.set_fact:
|
|
|
|
|
k3s_join_token: "{{ k3s_token }}"
|
|
|
|
|
no_log: true
|
|
|
|
|
when: k3s_token is defined
|
|
|
|
|
|
|
|
|
|
- name: Install or update k3s
|
|
|
|
|
ansible.builtin.include_tasks: tasks/k3s-node.yml
|
|
|
|
|
|
|
|
|
|
- name: Join token
|
|
|
|
|
hosts: k3s
|
|
|
|
|
gather_facts: false
|
|
|
|
|
tags: [k3s]
|
|
|
|
|
vars:
|
|
|
|
|
k3s_init_host: "{{ groups['k3s_init'] | first }}"
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Read the token from the first master
|
|
|
|
|
ansible.builtin.slurp:
|
|
|
|
|
src: /var/lib/rancher/k3s/server/token
|
|
|
|
|
delegate_to: "{{ k3s_init_host }}"
|
|
|
|
|
run_once: true
|
|
|
|
|
register: k3s_token_file
|
|
|
|
|
no_log: true
|
|
|
|
|
when: k3s_token is not defined
|
|
|
|
|
# A dry run of a brand-new cluster has no token yet.
|
|
|
|
|
ignore_errors: "{{ ansible_check_mode }}"
|
|
|
|
|
|
|
|
|
|
- name: Set the join token
|
|
|
|
|
ansible.builtin.set_fact:
|
|
|
|
|
k3s_join_token: "{{ k3s_token | default(k3s_token_file.content | b64decode | trim) }}"
|
|
|
|
|
no_log: true
|
|
|
|
|
when: k3s_token is defined or k3s_token_file.content is defined
|
|
|
|
|
|
|
|
|
|
- name: Other masters
|
|
|
|
|
hosts: k3s_masters:!k3s_init
|
|
|
|
|
gather_facts: false
|
|
|
|
|
serial: 1
|
|
|
|
|
tags: [k3s]
|
|
|
|
|
vars:
|
|
|
|
|
k3s_role: server
|
|
|
|
|
k3s_service: k3s
|
|
|
|
|
k3s_init_host: "{{ groups['k3s_init'] | first }}"
|
|
|
|
|
k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Install or update k3s
|
|
|
|
|
ansible.builtin.include_tasks: tasks/k3s-node.yml
|
|
|
|
|
|
|
|
|
|
- name: Workers
|
|
|
|
|
hosts: k3s_workers
|
|
|
|
|
gather_facts: false
|
|
|
|
|
serial: 1
|
|
|
|
|
tags: [k3s]
|
|
|
|
|
vars:
|
|
|
|
|
k3s_role: agent
|
|
|
|
|
k3s_service: k3s-agent
|
|
|
|
|
k3s_init_host: "{{ groups['k3s_init'] | first }}"
|
|
|
|
|
k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Install or update k3s
|
|
|
|
|
ansible.builtin.include_tasks: tasks/k3s-node.yml
|
|
|
|
|
|
|
|
|
|
- name: Admin kubeconfig
|
|
|
|
|
hosts: k3s_init
|
|
|
|
|
gather_facts: false
|
|
|
|
|
tags: [k3s, kubeconfig]
|
|
|
|
|
tasks:
|
|
|
|
|
- name: Read the admin kubeconfig
|
|
|
|
|
ansible.builtin.slurp:
|
|
|
|
|
src: /etc/rancher/k3s/k3s.yaml
|
|
|
|
|
register: k3s_kubeconfig
|
|
|
|
|
no_log: true
|
|
|
|
|
|
|
|
|
|
# Same server and names as ~/.kube/k3s-homelab.yaml.
|
|
|
|
|
- name: Write kube/config
|
|
|
|
|
ansible.builtin.copy:
|
|
|
|
|
dest: "{{ playbook_dir }}/../kube/config"
|
|
|
|
|
content: >-
|
|
|
|
|
{{ k3s_kubeconfig.content | b64decode
|
|
|
|
|
| replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443')
|
|
|
|
|
| regex_replace('(?m)^(\s*(?:- )?(?:name|cluster|user|current-context): )default$', '\1k3s-homelab') }}
|
|
|
|
|
mode: "0600"
|
|
|
|
|
delegate_to: localhost
|
|
|
|
|
become: false
|
|
|
|
|
diff: false # client certificate and key
|