rewrite start/stop scripts

README.md

@@ -51,12 +51,10 @@ cp local.env.sample local.env
 cp install-config.yaml.sample install-config.yaml
 ```
 
-Edit **provider.tf** and change the connection string.
+Install the required Ansible collections.
 
-```
+```sh
-provider "libvirt" {
+ansible-galaxy collection install -r ansible/requirements.yaml
-  uri = "qemu+ssh://user@libvirt.server/system"
-}
 ```
 
 ### On the server

ansible/requirements.yaml

@@ -0,0 +1,2 @@
+collections:
+- name: community.libvirt

ansible/start.yaml

@@ -0,0 +1,55 @@
+- name: Add the libvirt server to the inventory
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - add_host:
+      hostname: '{{ lookup("env", "LIBVIRT_SERVER") }}'
+      ansible_host: '{{ lookup("env", "LIBVIRT_SERVER") }}'
+      ansible_user: '{{ lookup("env", "LIBVIRT_USER") }}'
+      groups: hypervisor
+
+- name: Start the OpenShift cluster
+  hosts: hypervisor
+  gather_facts: no
+  become: yes
+  vars:
+    lb: '{{ nodes | selectattr("role", "eq", "lb") | first }}'
+    storage: '{{ nodes | selectattr("role", "eq", "storage") | first }}'
+    workers: '{{ nodes | selectattr("role", "eq", "worker") | list }}'
+    masters: '{{ nodes | selectattr("role", "eq", "master") | list }}'
+  tasks:
+  - name: Configure name resolution for the cluster
+    template:
+      src: dnsmasq.conf.j2
+      dest: /etc/NetworkManager/dnsmasq.d/zone-{{ network_domain }}.conf
+  
+  - name: Restart dnsmasq
+    command: pkill -f [d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq
+  
+  - name: Start the Load Balancer and the Storage
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      state: running
+    loop:
+    - '{{ lb }}'
+    - '{{ storage }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Wait for the Load Balancer to appear
+    wait_for:
+      port: 443
+      host: '{{ lb.ip[1] }}'
+
+  - name: Wait for the NFS Server to appear
+    wait_for:
+      port: 2049
+      host: '{{ storage.ip }}'
+
+  - name: Start the Workers and the Masters
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      state: running
+    loop: '{{ masters + workers }}'
+    loop_control:
+      label: "{{ item.name }}"

ansible/stop.yaml

@@ -0,0 +1,77 @@
+- name: Add the libvirt server to the inventory
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - add_host:
+      hostname: '{{ lookup("env", "LIBVIRT_SERVER") }}'
+      ansible_host: '{{ lookup("env", "LIBVIRT_SERVER") }}'
+      ansible_user: '{{ lookup("env", "LIBVIRT_USER") }}'
+      groups: hypervisor
+
+- name: Stop the OpenShift cluster
+  hosts: hypervisor
+  gather_facts: no
+  become: yes
+  vars:
+    lb: '{{ nodes | selectattr("role", "eq", "lb") | first }}'
+    storage: '{{ nodes | selectattr("role", "eq", "storage") | first }}'
+    workers: '{{ nodes | selectattr("role", "eq", "worker") | list }}'
+    masters: '{{ nodes | selectattr("role", "eq", "master") | list }}'
+  tasks:
+  - name: Stop the workers
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      state: shutdown
+    loop: '{{ workers }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Wait for the workers to shutdown
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      command: info
+    register: vm
+    until: "vm[item.name].state == 'shutdown'"
+    retries: 24
+    delay: 5
+    loop: '{{ workers }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Stop the masters
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      state: shutdown
+    loop: '{{ masters }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Wait for the masters to shutdown
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      command: info
+    register: vm
+    until: "vm[item.name].state == 'shutdown'"
+    retries: 24
+    delay: 5
+    loop: '{{ masters }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Stop the remaining nodes
+    community.libvirt.virt:
+      name: '{{ item.name }}'
+      state: shutdown
+    loop:
+    - '{{ lb }}'
+    - '{{ storage }}'
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Unconfigure name resolution for the cluster
+    file:
+      path: /etc/NetworkManager/dnsmasq.d/zone-{{ network_domain }}.conf
+      state: absent
+  
+  - name: Restart dnsmasq
+    command: pkill -f [d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq

ansible/templates/dnsmasq.conf.j2

@@ -0,0 +1 @@
+server=/{{ network_domain }}/{{ dns_server }}

cluster

@@ -100,8 +100,7 @@ function start () {
     exit 1
   fi
 
-  scp "$cluster_name/start.sh" "$LIBVIRT_USER@$LIBVIRT_SERVER:/tmp/start.sh"
+  ansible-playbook -i "$cluster_name/inventory" ansible/start.yaml
-  ssh "$LIBVIRT_USER@$LIBVIRT_SERVER" /tmp/start.sh
 }
 
 function stop () {
@@ -112,8 +111,7 @@ function stop () {
     exit 1
   fi
 
-  scp "$cluster_name/stop.sh" "$LIBVIRT_USER@$LIBVIRT_SERVER:/tmp/stop.sh"
+  ansible-playbook -i "$cluster_name/inventory" ansible/stop.yaml
-  ssh "$LIBVIRT_USER@$LIBVIRT_SERVER" /tmp/stop.sh
 }
 
 function post_install_nfs () {
@@ -195,6 +193,32 @@ function post_install () {
   done
 }
 
+function shell () {
+  local cluster_name="${1:-}"
+
+  if [ ! -d "$cluster_name" ]; then
+    echo "Cluster '$cluster_name' does not exist!"
+    exit 1
+  fi
+
+  # Ansible
+  export DEFAULT_HOST_LIST="$PWD/$cluster_name"
+
+  # Terraform
+  export TF_CLI_ARGS="-var-file=$cluster_name/terraform.tfvars -state=$cluster_name/terraform.tfstate"
+
+  # OpenShift
+  export KUBECONFIG="$PWD/$cluster_name/auth/kubeconfig"
+  export OC_BINARY="$(which oc)"
+  export CLUSTER_NAME="$cluster_name"
+  export PS1="[$CLUSTER_NAME:\w] "
+  function oc () {
+    "$OC_BINARY" --insecure-skip-tls-verify "$@"
+  }
+  export -f oc
+  exec /bin/bash
+}
+
 if [ ! -e "local.env" ]; then
   echo "Please create local.env first!"
   exit 1
@@ -203,6 +227,7 @@ fi
 source local.env
 export LC_ALL=C
 export LANG=C
+export LIBVIRT_DEFAULT_URI="qemu+ssh://$LIBVIRT_USER@$LIBVIRT_SERVER/system"
 
 case "${1:-}" in
 init)
@@ -261,6 +286,14 @@ destroy)
   shift
   destroy "$@"
 ;;
+shell)
+  if [ -z "${2:-}" ]; then
+    echo "Usage: $0 shell cluster-name"
+    exit 1
+  fi
+  shift
+  shell "$@"
+;;
 post-install)
   if [ -z "${2:-}" ]; then
     echo "Usage: $0 post-install cluster-name"
@@ -270,7 +303,7 @@ post-install)
   post_install "$@"
 ;;
 *)
-  echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|ping|start|stop} cluster-name"
+  echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|shell|ping|start|stop} cluster-name"
   exit 1
 ;;
 esac

main.tf

@@ -25,11 +25,11 @@ terraform {
 }
 
 locals {
-  master_nodes     = { for i in libvirt_domain.master : i.name => i.network_interface.0.addresses[0] }
+  master_nodes     = [ for i in libvirt_domain.master : { name = i.name, ip = i.network_interface.0.addresses[0], role = "master" } ]
-  worker_nodes     = { for i in libvirt_domain.worker : i.name => i.network_interface.0.addresses[0] }
+  worker_nodes     = [ for i in libvirt_domain.worker : { name = i.name, ip = i.network_interface.0.addresses[0], role = "worker" } ]
-  bootstrap_nodes  = { for i in libvirt_domain.bootstrap : i.name => i.network_interface.0.addresses[0] }
+  bootstrap_nodes  = [ for i in libvirt_domain.bootstrap : { name = i.name, ip = i.network_interface.0.addresses[0], role = "bootstrap" } ]
-  additional_nodes = { (libvirt_domain.lb.name) = [ libvirt_domain.storage.network_interface.0.addresses[0], libvirt_domain.storage.network_interface.1.addresses[0] ], (libvirt_domain.storage.name) = libvirt_domain.storage.network_interface.0.addresses[0] }
+  additional_nodes = [ { name = (libvirt_domain.lb.name), ip = [ libvirt_domain.lb.network_interface.0.addresses[0], libvirt_domain.lb.network_interface.1.addresses[0] ], role = "lb" }, { name = (libvirt_domain.storage.name), ip = libvirt_domain.storage.network_interface.0.addresses[0], role = "storage" } ]
-  all_nodes        = merge(local.additional_nodes, local.master_nodes, local.worker_nodes, local.bootstrap_nodes)
+  all_nodes        = concat(local.additional_nodes, local.master_nodes, local.worker_nodes, local.bootstrap_nodes)
 }
 
 output "machines" {

post-install.tf

@@ -16,45 +16,8 @@ resource "local_file" "dns_config" {
   file_permission = "0644"
 }
 
-resource "local_file" "stop_sh" {
+resource "local_file" "ansible_inventory" {
-  content         = templatefile("${path.module}/templates/stop.sh", { masters = libvirt_domain.master.*.name, workers = libvirt_domain.worker.*.name, lb = libvirt_domain.lb.name, storage = libvirt_domain.storage.name })
+  content         = templatefile("${path.module}/templates/inventory", { nodes = local.all_nodes, network_domain = local.network_domain, dns_server = cidrhost(var.network_ip_range, 1) })
-  filename        = "${var.cluster_name}/stop.sh"
+  filename        = "${var.cluster_name}/inventory"
-  file_permission = "0755"
+  file_permission = "0644"
-}
-
-resource "local_file" "start_sh" {
-  content         = templatefile("${path.module}/templates/start.sh", { masters = local.master_nodes, workers = local.worker_nodes, others = local.additional_nodes })
-  filename        = "${var.cluster_name}/start.sh"
-  file_permission = "0755"
-}
-
-resource "null_resource" "dnsmasq_config" {
-  triggers = {
-    network_id = libvirt_network.ocp_net.id
-    libvirt_server = local.libvirt_server
-    libvirt_username = local.libvirt_username
-    network_domain = local.network_domain
-  }
-
-  connection {
-    type = "ssh"
-    host = self.triggers.libvirt_server
-    user = self.triggers.libvirt_username
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "echo 'server=/${local.network_domain}/${cidrhost(var.network_ip_range, 1)}' | sudo tee /etc/NetworkManager/dnsmasq.d/zone-${local.network_domain}.conf",
-      "sudo pkill -f '[d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq'"
-    ]
-  }
-
-  provisioner "remote-exec" {
-    when = destroy
-    on_failure = continue
-    inline = [
-      "sudo rm -f /etc/NetworkManager/dnsmasq.d/zone-${self.triggers.network_domain}.conf",
-      "sudo pkill -f '[d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq'"
-    ]
-  }
 }

provider.tf

@@ -1,14 +1,7 @@
 provider "libvirt" {
-  uri = "qemu:///system"
 }
 
 provider "gandi" {
   # key = "<livedns apikey>"
   # sharing_id = "<sharing id>"
 }
-
-locals {
-  # See post-install.tf
-  libvirt_server   = "hp-ml350.itix.fr"
-  libvirt_username = "nicolas"
-}

templates/inventory

@@ -0,0 +1,6 @@
+[hypervisor]
+
+[hypervisor:vars]
+network_domain=${network_domain}
+dns_server=${dns_server}
+nodes=${jsonencode(nodes)}

templates/start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-set -Eeuo pipefail
-trap "exit" INT
-
-function start () {
-  for i; do
-    sudo virsh start "$i" || true
-  done
-}
-
-function wait_for_ip () {
-  echo "Waiting for $1 to come online..."
-  while ! ping -n -c4 -i.2 $2 -q &>/dev/null; do
-    sleep 1
-  done
-}
-
-%{for host, ip in others~}
-start "${host}"
-wait_for_ip "${host}" "${ip}"
-%{endfor~}
-
-%{for host, ip in masters~}
-start "${host}"
-%{endfor~}
-
-%{for host, ip in workers~}
-start "${host}"
-%{endfor~}

templates/stop.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-
-set -Eeuo pipefail
-trap "exit" INT
-
-function stop_group () {
-  for i; do
-    sudo virsh shutdown "$i" --mode=agent || true
-  done
-
-  for i; do
-    echo "Waiting for $i to shutdown..."
-    while sudo virsh list --name | egrep -q "^$i\$"; do
-      sleep 1
-      continue
-    done
-  done
-}
-
-stop_group %{for host in workers}"${host}" %{endfor}
-stop_group %{for host in masters}"${host}" %{endfor}
-stop_group "${lb}" "${storage}"