Test playbook using molecule (techno-tim#67)

* Test cluster using molecule

* Fix detection of first control node

* Include --flannel-iface and --node-ip as k3s arguments

* Store logs of k3s-init.service as GitHub job artifacts

.ansible-lint

@@ -1,3 +1,16 @@
 ---
+exclude_paths:
+ # default paths
+ - '.cache/'
+ - '.github/'
+ - 'test/fixtures/formatting-before/'
+ - 'test/fixtures/formatting-prettier/'
+
+ # The "converge" and "reset" playbooks use import_playbook in
+ # conjunction with the "env" lookup plugin, which lets the
+ # syntax check of ansible-lint fail.
+ - '**/converge.yml'
+ - '**/reset.yml'
+
 skip_list:
  - 'fqcn-builtins'

.github/workflows/test.yml

@@ -7,24 +7,21 @@ on:
  - master
 
 jobs:
- vagrant:
- name: Vagrant
+ molecule:
+ name: Molecule
  runs-on: macos-12
 
+ strategy:
+ matrix:
+ scenario:
+ - default
  env:
- HOMEBREW_NO_INSTALL_CLEANUP: 1
- VAGRANT_CWD: ${{ github.workspace }}/vagrant
+ PYTHON_VERSION: "3.10"
 
  steps:
  - name: Check out the codebase
  uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
 
- - name: Install Ansible
- run: brew install ansible
-
- - name: Install role dependencies
- run: ansible-galaxy install -r collections/requirements.yml
-
  - name: Configure VirtualBox
  run: >-
  sudo mkdir -p /etc/vbox &&
@@ -39,31 +36,29 @@ jobs:
  restore-keys: |
  vagrant-boxes
 
- - name: Create virtual machines
- run: vagrant up
- timeout-minutes: 10
-
- - name: Provision cluster using Ansible
- # Since Ansible sets up _all_ machines, it is sufficient to run it only
- # once (i.e, for a single node - we are choosing control1 here)
- run: vagrant provision control1 --provision-with ansible
- timeout-minutes: 25
-
- - name: Set up kubectl on the host
- run: brew install kubectl &&
- mkdir -p ~/.kube &&
- vagrant ssh control1 --command "cat ~/.kube/config" > ~/.kube/config
+ - name: Set up Python ${{ env.PYTHON_VERSION }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
 
- - name: Show cluster nodes
- run: kubectl describe -A nodes
+ - name: Install dependencies
+ run: >-
+ python3 -m pip install --upgrade pip &&
+ python3 -m pip install -r requirements.txt
 
- - name: Show cluster pods
- run: kubectl describe -A pods
+ - name: Test with molecule
+ run: molecule test --scenario-name ${{ matrix.scenario }}
+ env:
+ ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
 
- - name: Test cluster
- run: $VAGRANT_CWD/test_cluster.py --verbose --locals
- timeout-minutes: 5
+ - name: Upload log files
+ if: always() # do this even if a step before has failed
+ uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # 3.1.0
+ with:
+ name: logs
+ path: |
+ ${{ runner.temp }}/logs
 
- - name: Destroy virtual machines
+ - name: Delete old box versions
  if: always() # do this even if a step before has failed
- run: vagrant destroy --force
+ run: vagrant box prune --force

.gitignore

@@ -1 +1 @@
-.vagrant
+.env/

README.md

@@ -100,12 +100,12 @@ See the commands [here](https://docs.technotim.live/posts/k3s-etcd-ansible/#test
 
 Be sure to see [this post](https://github.com/techno-tim/k3s-ansible/discussions/20) on how to troubleshoot common problems
 
-### 🔷 Vagrant
+### Testing the playbook using molecule
 
-You may want to kickstart your k3s cluster by using Vagrant to quickly build you all needed VMs with one command.
-Head to the `vagrant` subfolder and type `vagrant up` to get your environment setup.
-After the VMs have got build, deploy k3s using the Ansible playbook `site.yml` by the
-`vagrant provision --provision-with ansible` command.
+This playbook includes a [molecule](https://molecule.rtfd.io/)-based test setup.
+It is run automatically in CI, but you can also run the tests locally.
+This might be helpful for quick feedback in a few cases.
+You can find more information about it [here](molecule/README.md).
 
 ## Thanks 🤝
 

collections/requirements.yml

@@ -2,3 +2,4 @@
 collections:
  - name: community.general
  - name: ansible.posix
+ - name: kubernetes.core

inventory/sample/group_vars/all.yml

@@ -17,9 +17,23 @@ apiserver_endpoint: "192.168.30.222"
 # this token should be alpha numeric only
 k3s_token: "some-SUPER-DEDEUPER-secret-password"
 
-# change these to your liking, the only required one is--disable servicelb
-extra_server_args: "--disable servicelb --disable traefik"
-extra_agent_args: ""
+# The IP on which the node is reachable in the cluster.
+# Here, a sensible default is provided, you can still override
+# it for each of your hosts, though.
+k3s_node_ip: '{{ ansible_facts[flannel_iface]["ipv4"]["address"] }}'
+
+# these arguments are recommended for servers as well as agents:
+extra_args: >-
+ --flannel-iface={{ flannel_iface }}
+ --node-ip={{ k3s_node_ip }}
+
+# change these to your liking, the only required one is --disable servicelb
+extra_server_args: >-
+ {{ extra_args }}
+ --disable servicelb
+ --disable traefik
+extra_agent_args: >-
+ {{ extra_args }}
 
 # image tag for kube-vip
 kube_vip_tag_version: "v0.5.0"

molecule/README.md

@@ -0,0 +1,67 @@
+# Test suites for `k3s-ansible`
+
+This folder contains the [molecule](https://molecule.rtfd.io/)-based test setup for this playbook.
+
+## Scenarios
+
+We have these scenarios:
+
+- **default**:
+ A 3 control + 2 worker node cluster based very closely on the [sample inventory](../inventory/sample/).
+
+## How to execute
+
+To test on your local machine, follow these steps:
+
+### System requirements
+
+Make sure that the following software packages are available on your system:
+
+- [Python 3](https://www.python.org/downloads)
+- [Vagrant](https://www.vagrantup.com/downloads)
+- [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+
+### Set up VirtualBox networking on Linux and macOS
+
+_You can safely skip this if you are working on Windows._
+
+Furthermore, the test cluster uses the `192.168.30.0/24` subnet which is [not set up by VirtualBox automatically](https://www.virtualbox.org/manual/ch06.html#network_hostonly).
+To set the subnet up for use with VirtualBox, please make sure that `/etc/vbox/networks.conf` exists and that it contains this line:
+
+```
+* 192.168.30.0/24`
+```
+
+### Install Python dependencies
+
+You will get [Molecule, Ansible and a few extra dependencies](../requirements.txt) via [pip](https://pip.pypa.io/).
+Usually, it is advisable to work in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for this:
+
+```bash
+cd /path/to/k3s-ansible
+
+# Create a virtualenv at ".env". You only need to do this once.
+python3 -m venv .env
+
+# Activate the virtualenv for your current shell session.
+# If you start a new session, you will have to repeat this.
+source .env/bin/activate
+
+# Install the required packages into the virtualenv.
+# These remain installed across shell sessions.
+python3 -m pip install -r requirements.txt
+```
+
+### Run molecule
+
+With the virtual environment from the previous step active in your shell session, you can now use molecule to test the playbook.
+Interesting commands are:
+
+- `molecule create`: Create virtual machines for the test cluster nodes.
+- `molecule destroy`: Delete the virtual machines for the test cluster nodes.
+- `molecule converge`: Run the `site` playbook on the nodes of the test cluster.
+- `molecule side_effect`: Run the `reset` playbook on the nodes of the test cluster.
+- `molecule verify`: Verify that the cluster works correctly.
+- `molecule test`: The "all-in-one" sequence of steps that is executed in CI.
+ This includes the `create`, `converge`, `verify`, `side_effect` and `destroy` steps.
+ See [`molecule.yml`](default/molecule.yml) for more details.

molecule/default/molecule.yml

@@ -0,0 +1,75 @@
+---
+dependency:
+ name: galaxy
+driver:
+ name: vagrant
+platforms:
+ - &control
+ name: control1
+ box: generic/ubuntu2204
+ memory: 2048
+ cpus: 2
+ config_options:
+ # We currently can not use public-key based authentication on Ubuntu 22.04,
+ # see: https://github.com/chef/bento/issues/1405
+ ssh.username: "vagrant"
+ ssh.password: "vagrant"
+ groups:
+ - k3s_cluster
+ - master
+ interfaces:
+ - network_name: private_network
+ ip: 192.168.30.38
+ - <<: *control
+ name: control2
+ interfaces:
+ - network_name: private_network
+ ip: 192.168.30.39
+ - <<: *control
+ name: control3
+ interfaces:
+ - network_name: private_network
+ ip: 192.168.30.40
+ - &node
+ <<: *control
+ name: node1
+ groups:
+ - k3s_cluster
+ - node
+ interfaces:
+ - network_name: private_network
+ ip: 192.168.30.41
+ - <<: *node
+ name: node2
+ interfaces:
+ - network_name: private_network
+ ip: 192.168.30.42
+provisioner:
+ name: ansible
+ playbooks:
+ converge: ../resources/converge.yml
+ side_effect: ../resources/reset.yml
+ verify: ../resources/verify.yml
+ inventory:
+ links:
+ group_vars: ../../inventory/sample/group_vars
+ env:
+ OVERRIDES_FILE: ../default/overrides.yml
+scenario:
+ test_sequence:
+ - dependency
+ - lint
+ - cleanup
+ - destroy
+ - syntax
+ - create
+ - prepare
+ - converge
+ # idempotence is not possible with the playbook in its current form.
+ - verify
+ # We are repurposing side_effect here to test the reset playbook.
+ # This is why we do not run it before verify (which tests the cluster),
+ # but after the verify step.
+ - side_effect
+ - cleanup
+ - destroy

molecule/default/overrides.yml

@@ -0,0 +1,11 @@
+---
+- name: Apply overrides
+ hosts: all
+ tasks:
+ - name: Override host variables
+ ansible.builtin.set_fact:
+ # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # noqa yaml[line-length]
+ flannel_iface: eth1
+
+ # The test VMs might be a bit slow, so we give them more time to join the cluster:
+ retry_count: 45

molecule/resources/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+ ansible.builtin.import_playbook: >-
+ {{ lookup("ansible.builtin.env", "OVERRIDES_FILE") }}
+
+- name: Converge
+ ansible.builtin.import_playbook: ../../site.yml

molecule/resources/reset.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+ ansible.builtin.import_playbook: >-
+ {{ lookup("ansible.builtin.env", "OVERRIDES_FILE") }}
+
+- name: Reset
+ ansible.builtin.import_playbook: ../../reset.yml

molecule/resources/verify.yml

@@ -0,0 +1,5 @@
+---
+- name: Verify
+ hosts: all
+ roles:
+ - verify/from_outside

molecule/resources/verify/from_outside/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# A host outside of the cluster from which the checks shall be performed
+outside_host: localhost
+
+# This kubernetes namespace will be used for testing
+testing_namespace: molecule-verify-from-outside
+
+# The directory in which the example manifests reside
+example_manifests_path: ../../../../example

molecule/resources/verify/from_outside/tasks/kubecfg-cleanup.yml

@@ -0,0 +1,5 @@
+---
+- name: Clean up kubecfg
+ ansible.builtin.file:
+ path: "{{ kubecfg.path }}"
+ state: absent

molecule/resources/verify/from_outside/tasks/kubecfg-fetch.yml

@@ -0,0 +1,19 @@
+---
+- name: Create temporary directory for kubecfg
+ ansible.builtin.tempfile:
+ state: directory
+ suffix: kubecfg
+ register: kubecfg
+- name: Gathering facts
+ delegate_to: "{{ groups['master'][0] }}"
+ ansible.builtin.gather_facts:
+- name: Download kubecfg
+ ansible.builtin.fetch:
+ src: "{{ ansible_env.HOME }}/.kube/config"
+ dest: "{{ kubecfg.path }}/"
+ flat: true
+ delegate_to: "{{ groups['master'][0] }}"
+ delegate_facts: true
+- name: Store path to kubecfg
+ ansible.builtin.set_fact:
+ kubecfg_path: "{{ kubecfg.path }}/config"

molecule/resources/verify/from_outside/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Verify
+ run_once: true
+ delegate_to: "{{ outside_host }}"
+ block:
+ - ansible.builtin.import_tasks: kubecfg-fetch.yml
+ - name: "TEST CASE: Get nodes"
+ ansible.builtin.include_tasks: test/get-nodes.yml
+ - name: "TEST CASE: Deploy example"
+ ansible.builtin.include_tasks: test/deploy-example.yml
+ always:
+ - ansible.builtin.import_tasks: kubecfg-cleanup.yml