Merge pull request #60 from shakenfist/feature-more-reliable-old-ci

More reliable old CI

README.md

@@ -89,6 +89,7 @@ with real users.
 
 | Option | Terraform definition | Description |
 |--------|----------------------|-------------|
+| MODE | All | Options are "deploy" (the default) or "hotfix". Deploy performs a full install, whereas hotfix skips steps to try and push only changes to Shaken Fist code as quickly as possible. |
 | CLOUD | All | The terraform definition to use |
 | ADMIN_PASSWORD | All | The admin password for the cloud once installed |
 | FLOATING_IP_BLOCK | All | The IP range to use for the floating network |

ansible-ci/tests/_setup.yml

@@ -5,11 +5,35 @@
  sf-client --simple instance delete $inst
  done
 
+ attempts=0
+ while [ `sf-client --simple instance list | grep -v uuid | wc -l` -gt 0 ]
+ do
+ attempts=$(( $attempts + 1))
+ sleep 10
+ if [ $attempts -gt 10 ]
+ then
+ echo "Failing after instance deletes did not complete"
+ exit 1
+ fi
+ done
+
  for net in `sf-client --simple network list | grep -v uuid | cut -f 1 -d ","`
  do
  sf-client --simple network delete $net
  done
 
+ attempts=0
+ while [ `sf-client --simple network list | grep -v uuid | wc -l` -gt 0 ]
+ do
+ attempts=$(( $attempts + 1))
+ sleep 10
+ if [ $attempts -gt 10 ]
+ then
+ echo "Failing after network deletes did not complete"
+ exit 1
+ fi
+ done
+
  for namespace in `sf-client --simple namespace list | grep -v uuid | grep -v system`
  do
  sf-client --simple namespace delete $namespace

ansible-ci/tests/_util_instance_start.yml

@@ -23,8 +23,9 @@
  debug:
  msg: "{{instance_create_out}}"
 
-- name: Pause very briefly
- shell: sleep 10
+- name: Wait for instance to boot
+ pause:
+ minutes: 2
 
 - name: Count the number of VMs after
  virt:

ansible-ci/tests/_util_network_cirros_validate.yml

@@ -7,6 +7,12 @@
  content: |
  #!/bin/bash -e
 
+ while [ `sf-client instance events {{uuid}} | grep -c "login prompt"` -lt 1 ]
+ do
+ echo "Waiting for login prompt..."
+ sleep 10
+ done
+
  [ `/opt/telnet_client.py {{console_port}} ifconfig eth0 | grep -c "{{netblock_octets}}"` -eq 1 ]
  [ `/opt/telnet_client.py {{console_port}} cat /etc/resolv.conf | grep -c "8.8.8.8"` -eq 1 ]
  [ `/opt/telnet_client.py {{console_port}} netstat -rn | grep -c "{{netblock_octets}}.1"` -eq 1 ]

ansible-ci/tests/test_cloudinit.yml

@@ -36,6 +36,12 @@
  content: |
  #!/bin/bash -e
 
+ while [ `sf-client instance events {{cirros_with_extras_uuid}} | grep -c "login prompt"` -lt 1 ]
+ do
+ echo "Waiting for login prompt..."
+ sleep 10
+ done
+
  [ `/opt/telnet_client.py {{cirros_with_extras_console_port}} exists /home/cirros/output.txt | grep -c "File exists"` -eq 2 ]
  [ `/opt/telnet_client.py {{cirros_with_extras_console_port}} exists /home/cirros/.ssh/authorized_keys | grep -c "File exists"` -eq 2 ]
  dest: /tmp/shell_script

ansible-ci/tests/test_instances_start_with_dhcp_correct.yml

@@ -108,6 +108,7 @@
  tasks:
  - include: _util_network_cirros_validate.yml
  console_port="{{cirros_console_port}}"
+ uuid="{{cirros_uuid}}"
  netblock_octets="192.168.242"
 
 - hosts: sf-1

ansible-ci/tests/test_networking.yml

@@ -138,6 +138,7 @@
 
  - include: _util_network_cirros_validate.yml
  console_port="{{cirros_net_two_console_port}}"
+ uuid="{{cirros_net_two_uuid}}"
  netblock_octets="192.168.240"
 
  - name: Check we can't talk to the other virtual network
@@ -179,6 +180,7 @@
 
  - include: _util_network_cirros_validate.yml
  console_port="{{cirros_net_three_console_port}}"
+ uuid="{{cirros_net_three_uuid}}"
  netblock_octets="192.168.242"
 
  - name: Check we can't talk to the other virtual network

ansible-ci/tests/test_state_changes.yml

@@ -10,37 +10,41 @@
  connection: ssh
 
  tasks:
- - include: _util_instance_reboot.yml
- uuid={{cirros_uuid}}
- type=soft
-
- - include: _util_network_cirros_validate.yml
- console_port="{{cirros_console_port}}"
- netblock_octets="192.168.242"
-
- - include: _util_instance_reboot.yml
- uuid={{cirros_uuid}}
- type=hard
-
- - include: _util_network_cirros_validate.yml
- console_port="{{cirros_console_port}}"
- netblock_octets="192.168.242"
-
- - include: _util_instance_halt.yml
- uuid={{cirros_uuid}}
- action_one=poweroff
- action_two=poweron
-
- - include: _util_network_cirros_validate.yml
- console_port="{{cirros_console_port}}"
- netblock_octets="192.168.242"
-
- - include: _util_instance_halt.yml
- uuid={{cirros_uuid}}
- ip={{cirros_uuid}}
- action_one=pause
- action_two=unpause
-
- - include: _util_network_cirros_validate.yml
- console_port="{{cirros_console_port}}"
- netblock_octets="192.168.242"
+ - include: _util_instance_reboot.yml
+ uuid={{cirros_uuid}}
+ type=soft
+
+ - include: _util_network_cirros_validate.yml
+ console_port="{{cirros_console_port}}"
+ uuid="{{cirros_uuid}}"
+ netblock_octets="192.168.242"
+
+ - include: _util_instance_reboot.yml
+ uuid={{cirros_uuid}}
+ type=hard
+
+ - include: _util_network_cirros_validate.yml
+ console_port="{{cirros_console_port}}"
+ uuid="{{cirros_uuid}}"
+ netblock_octets="192.168.242"
+
+ - include: _util_instance_halt.yml
+ uuid={{cirros_uuid}}
+ action_one=poweroff
+ action_two=poweron
+
+ - include: _util_network_cirros_validate.yml
+ console_port="{{cirros_console_port}}"
+ uuid="{{cirros_uuid}}"
+ netblock_octets="192.168.242"
+
+ - include: _util_instance_halt.yml
+ uuid={{cirros_uuid}}
+ ip={{cirros_uuid}}
+ action_one=pause
+ action_two=unpause
+
+ - include: _util_network_cirros_validate.yml
+ console_port="{{cirros_console_port}}"
+ uuid="{{cirros_uuid}}"
+ netblock_octets="192.168.242"