--- # Build, upgrade or reconfigure the k3s cluster (k3s-design.md): 3 masters with # embedded etcd, 3 workers carrying Longhorn. Safe to rerun: on the running # cluster it changes nothing unless the inventory says otherwise. # # ansible-playbook playbooks/deploy_kube.yml -CD # dry run first # ansible-playbook playbooks/deploy_kube.yml # ansible-playbook playbooks/deploy_kube.yml --tags kubeconfig # only refresh kube/config # # Upgrade: bump k3s_version in group_vars/k3s.yml and rerun. The cluster-init # master goes first, then the other masters, then the workers, one node at a # time, each waiting for Ready. Restarting k3s leaves the pods running. # # Creating the VMs themselves is not done here (k3s-build-vms.md). - name: Check the k3s inventory hosts: k3s gather_facts: true tags: [always] tasks: - name: Only VMs can be k3s nodes ansible.builtin.assert: that: host_type == 'vm' fail_msg: "{{ inventory_hostname }} is a {{ host_type }}. k3s needs a VM (own kernel, cgroups, iSCSI for Longhorn)." quiet: true - name: Exactly one master bootstraps the cluster ansible.builtin.assert: that: groups['k3s_masters'] | map('extract', hostvars) | selectattr('k3s_cluster_init', 'defined') | selectattr('k3s_cluster_init') | list | length == 1 fail_msg: "Set k3s_cluster_init: true on exactly one host of k3s_masters." quiet: true run_once: true - name: Group the cluster-init master ansible.builtin.group_by: key: k3s_init when: k3s_cluster_init | default(false) changed_when: false - name: Longhorn prerequisites hosts: k3s_workers gather_facts: false tags: [longhorn] tasks: - name: Install iSCSI and NFS clients ansible.builtin.apt: name: [open-iscsi, nfs-common] state: present update_cache: true cache_valid_time: 3600 - name: Start iscsid ansible.builtin.systemd_service: name: iscsid enabled: true state: started - name: Load iscsi_tcp at boot ansible.builtin.copy: dest: /etc/modules-load.d/longhorn.conf content: "iscsi_tcp\n" owner: root group: root mode: "0644" - name: Load iscsi_tcp now community.general.modprobe: name: iscsi_tcp state: present - name: List services ansible.builtin.service_facts: # multipathd grabs Longhorn's block devices. - name: Disable multipathd ansible.builtin.systemd_service: name: multipathd.service enabled: false state: stopped when: "'multipathd.service' in ansible_facts.services" - name: Stop its socket (static unit, can't be disabled) ansible.builtin.systemd_service: name: multipathd.socket state: stopped when: "'multipathd.service' in ansible_facts.services" - name: Check whether the Longhorn disk is already mounted ansible.builtin.command: findmnt -no SOURCE {{ k3s_longhorn_path }} register: k3s_longhorn_mount changed_when: false failed_when: false check_mode: false # Formatting only happens on an empty disk of the expected size, like # argocd/scripts/prep-worker.sh. Anything else stops the play. - name: Format the empty Longhorn disk when: k3s_longhorn_mount.stdout != k3s_longhorn_disk block: - name: Inspect {{ k3s_longhorn_disk }} ansible.builtin.shell: | set -o pipefail echo "size=$(lsblk -bdno SIZE {{ k3s_longhorn_disk }})" echo "parts=$(lsblk -no NAME {{ k3s_longhorn_disk }} | tail -n +2 | wc -l)" echo "fs=$(blkid -o value -s TYPE {{ k3s_longhorn_disk }} || true)" args: executable: /bin/bash register: k3s_disk_info changed_when: false check_mode: false - name: Refuse anything but an empty disk of the expected size ansible.builtin.assert: that: - "'size=' ~ k3s_longhorn_disk_size in k3s_disk_info.stdout_lines" - "'parts=0' in k3s_disk_info.stdout_lines" - "'fs=' in k3s_disk_info.stdout_lines" fail_msg: >- {{ k3s_longhorn_disk }} is not an empty {{ k3s_longhorn_disk_size }}-byte disk ({{ k3s_disk_info.stdout_lines | join(', ') }}). Not formatting it. - name: Create the ext4 filesystem community.general.filesystem: dev: "{{ k3s_longhorn_disk }}" fstype: ext4 opts: -L longhorn - name: Mount the Longhorn disk ansible.posix.mount: src: LABEL=longhorn path: "{{ k3s_longhorn_path }}" fstype: ext4 opts: defaults,noatime dump: "0" passno: "2" state: mounted - name: First master hosts: k3s_init gather_facts: false tags: [k3s] vars: k3s_role: server k3s_service: k3s k3s_init_host: "{{ inventory_hostname }}" tasks: # Without a vaulted k3s_token, a new cluster generates its own. - name: Use the vaulted token ansible.builtin.set_fact: k3s_join_token: "{{ k3s_token }}" no_log: true when: k3s_token is defined - name: Install or update k3s ansible.builtin.include_tasks: tasks/k3s-node.yml - name: Join token hosts: k3s gather_facts: false tags: [k3s] vars: k3s_init_host: "{{ groups['k3s_init'] | first }}" tasks: - name: Read the token from the first master ansible.builtin.slurp: src: /var/lib/rancher/k3s/server/token delegate_to: "{{ k3s_init_host }}" run_once: true register: k3s_token_file no_log: true when: k3s_token is not defined # A dry run of a brand-new cluster has no token yet. ignore_errors: "{{ ansible_check_mode }}" - name: Set the join token ansible.builtin.set_fact: k3s_join_token: "{{ k3s_token | default(k3s_token_file.content | b64decode | trim) }}" no_log: true when: k3s_token is defined or k3s_token_file.content is defined - name: Other masters hosts: k3s_masters:!k3s_init gather_facts: false serial: 1 tags: [k3s] vars: k3s_role: server k3s_service: k3s k3s_init_host: "{{ groups['k3s_init'] | first }}" k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443 tasks: - name: Install or update k3s ansible.builtin.include_tasks: tasks/k3s-node.yml - name: Workers hosts: k3s_workers gather_facts: false serial: 1 tags: [k3s] vars: k3s_role: agent k3s_service: k3s-agent k3s_init_host: "{{ groups['k3s_init'] | first }}" k3s_server_url: https://{{ hostvars[k3s_init_host].ansible_host }}:6443 tasks: - name: Install or update k3s ansible.builtin.include_tasks: tasks/k3s-node.yml - name: Admin kubeconfig hosts: k3s_init gather_facts: false tags: [k3s, kubeconfig] tasks: - name: Read the admin kubeconfig ansible.builtin.slurp: src: /etc/rancher/k3s/k3s.yaml register: k3s_kubeconfig no_log: true # Same server and names as ~/.kube/k3s-homelab.yaml. - name: Write kube/config ansible.builtin.copy: dest: "{{ playbook_dir }}/../kube/config" content: >- {{ k3s_kubeconfig.content | b64decode | replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443') | regex_replace('(?m)^(\s*(?:- )?(?:name|cluster|user|current-context): )default$', '\1k3s-homelab') }} mode: "0600" delegate_to: localhost become: false diff: false # client certificate and key