From 99be7df1f6c4a49078e03c5e0b8d9f581ce43cdb Mon Sep 17 00:00:00 2001 From: Nicolas MASSE Date: Wed, 24 Mar 2021 13:32:34 +0100 Subject: [PATCH] rewrite start/stop scripts --- README.md | 8 ++-- ansible/requirements.yaml | 2 + ansible/start.yaml | 55 ++++++++++++++++++++++ ansible/stop.yaml | 77 +++++++++++++++++++++++++++++++ ansible/templates/dnsmasq.conf.j2 | 1 + cluster | 43 +++++++++++++++-- main.tf | 10 ++-- post-install.tf | 45 ++---------------- provider.tf | 7 --- templates/inventory | 6 +++ templates/start.sh | 30 ------------ templates/stop.sh | 22 --------- 12 files changed, 191 insertions(+), 115 deletions(-) create mode 100644 ansible/requirements.yaml create mode 100644 ansible/start.yaml create mode 100644 ansible/stop.yaml create mode 100644 ansible/templates/dnsmasq.conf.j2 create mode 100644 templates/inventory delete mode 100644 templates/start.sh delete mode 100644 templates/stop.sh diff --git a/README.md b/README.md index a7ed00b..3055aa0 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,10 @@ cp local.env.sample local.env cp install-config.yaml.sample install-config.yaml ``` -Edit **provider.tf** and change the connection string. +Install the required Ansible collections. -``` -provider "libvirt" { - uri = "qemu+ssh://user@libvirt.server/system" -} +```sh +ansible-galaxy collection install -r ansible/requirements.yaml ``` ### On the server diff --git a/ansible/requirements.yaml b/ansible/requirements.yaml new file mode 100644 index 0000000..206914b --- /dev/null +++ b/ansible/requirements.yaml @@ -0,0 +1,2 @@ +collections: +- name: community.libvirt diff --git a/ansible/start.yaml b/ansible/start.yaml new file mode 100644 index 0000000..f06a751 --- /dev/null +++ b/ansible/start.yaml @@ -0,0 +1,55 @@ +- name: Add the libvirt server to the inventory + hosts: localhost + gather_facts: no + tasks: + - add_host: + hostname: '{{ lookup("env", "LIBVIRT_SERVER") }}' + ansible_host: '{{ lookup("env", "LIBVIRT_SERVER") }}' + ansible_user: '{{ lookup("env", "LIBVIRT_USER") }}' + groups: hypervisor + +- name: Start the OpenShift cluster + hosts: hypervisor + gather_facts: no + become: yes + vars: + lb: '{{ nodes | selectattr("role", "eq", "lb") | first }}' + storage: '{{ nodes | selectattr("role", "eq", "storage") | first }}' + workers: '{{ nodes | selectattr("role", "eq", "worker") | list }}' + masters: '{{ nodes | selectattr("role", "eq", "master") | list }}' + tasks: + - name: Configure name resolution for the cluster + template: + src: dnsmasq.conf.j2 + dest: /etc/NetworkManager/dnsmasq.d/zone-{{ network_domain }}.conf + + - name: Restart dnsmasq + command: pkill -f [d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq + + - name: Start the Load Balancer and the Storage + community.libvirt.virt: + name: '{{ item.name }}' + state: running + loop: + - '{{ lb }}' + - '{{ storage }}' + loop_control: + label: "{{ item.name }}" + + - name: Wait for the Load Balancer to appear + wait_for: + port: 443 + host: '{{ lb.ip[1] }}' + + - name: Wait for the NFS Server to appear + wait_for: + port: 2049 + host: '{{ storage.ip }}' + + - name: Start the Workers and the Masters + community.libvirt.virt: + name: '{{ item.name }}' + state: running + loop: '{{ masters + workers }}' + loop_control: + label: "{{ item.name }}" diff --git a/ansible/stop.yaml b/ansible/stop.yaml new file mode 100644 index 0000000..761920c --- /dev/null +++ b/ansible/stop.yaml @@ -0,0 +1,77 @@ +- name: Add the libvirt server to the inventory + hosts: localhost + gather_facts: no + tasks: + - add_host: + hostname: '{{ lookup("env", "LIBVIRT_SERVER") }}' + ansible_host: '{{ lookup("env", "LIBVIRT_SERVER") }}' + ansible_user: '{{ lookup("env", "LIBVIRT_USER") }}' + groups: hypervisor + +- name: Stop the OpenShift cluster + hosts: hypervisor + gather_facts: no + become: yes + vars: + lb: '{{ nodes | selectattr("role", "eq", "lb") | first }}' + storage: '{{ nodes | selectattr("role", "eq", "storage") | first }}' + workers: '{{ nodes | selectattr("role", "eq", "worker") | list }}' + masters: '{{ nodes | selectattr("role", "eq", "master") | list }}' + tasks: + - name: Stop the workers + community.libvirt.virt: + name: '{{ item.name }}' + state: shutdown + loop: '{{ workers }}' + loop_control: + label: "{{ item.name }}" + + - name: Wait for the workers to shutdown + community.libvirt.virt: + name: '{{ item.name }}' + command: info + register: vm + until: "vm[item.name].state == 'shutdown'" + retries: 24 + delay: 5 + loop: '{{ workers }}' + loop_control: + label: "{{ item.name }}" + + - name: Stop the masters + community.libvirt.virt: + name: '{{ item.name }}' + state: shutdown + loop: '{{ masters }}' + loop_control: + label: "{{ item.name }}" + + - name: Wait for the masters to shutdown + community.libvirt.virt: + name: '{{ item.name }}' + command: info + register: vm + until: "vm[item.name].state == 'shutdown'" + retries: 24 + delay: 5 + loop: '{{ masters }}' + loop_control: + label: "{{ item.name }}" + + - name: Stop the remaining nodes + community.libvirt.virt: + name: '{{ item.name }}' + state: shutdown + loop: + - '{{ lb }}' + - '{{ storage }}' + loop_control: + label: "{{ item.name }}" + + - name: Unconfigure name resolution for the cluster + file: + path: /etc/NetworkManager/dnsmasq.d/zone-{{ network_domain }}.conf + state: absent + + - name: Restart dnsmasq + command: pkill -f [d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq diff --git a/ansible/templates/dnsmasq.conf.j2 b/ansible/templates/dnsmasq.conf.j2 new file mode 100644 index 0000000..8ded78a --- /dev/null +++ b/ansible/templates/dnsmasq.conf.j2 @@ -0,0 +1 @@ +server=/{{ network_domain }}/{{ dns_server }} diff --git a/cluster b/cluster index 8e73e4b..1d78f1e 100755 --- a/cluster +++ b/cluster @@ -100,8 +100,7 @@ function start () { exit 1 fi - scp "$cluster_name/start.sh" "$LIBVIRT_USER@$LIBVIRT_SERVER:/tmp/start.sh" - ssh "$LIBVIRT_USER@$LIBVIRT_SERVER" /tmp/start.sh + ansible-playbook -i "$cluster_name/inventory" ansible/start.yaml } function stop () { @@ -112,8 +111,7 @@ function stop () { exit 1 fi - scp "$cluster_name/stop.sh" "$LIBVIRT_USER@$LIBVIRT_SERVER:/tmp/stop.sh" - ssh "$LIBVIRT_USER@$LIBVIRT_SERVER" /tmp/stop.sh + ansible-playbook -i "$cluster_name/inventory" ansible/stop.yaml } function post_install_nfs () { @@ -195,6 +193,32 @@ function post_install () { done } +function shell () { + local cluster_name="${1:-}" + + if [ ! -d "$cluster_name" ]; then + echo "Cluster '$cluster_name' does not exist!" + exit 1 + fi + + # Ansible + export DEFAULT_HOST_LIST="$PWD/$cluster_name" + + # Terraform + export TF_CLI_ARGS="-var-file=$cluster_name/terraform.tfvars -state=$cluster_name/terraform.tfstate" + + # OpenShift + export KUBECONFIG="$PWD/$cluster_name/auth/kubeconfig" + export OC_BINARY="$(which oc)" + export CLUSTER_NAME="$cluster_name" + export PS1="[$CLUSTER_NAME:\w] " + function oc () { + "$OC_BINARY" --insecure-skip-tls-verify "$@" + } + export -f oc + exec /bin/bash +} + if [ ! -e "local.env" ]; then echo "Please create local.env first!" exit 1 @@ -203,6 +227,7 @@ fi source local.env export LC_ALL=C export LANG=C +export LIBVIRT_DEFAULT_URI="qemu+ssh://$LIBVIRT_USER@$LIBVIRT_SERVER/system" case "${1:-}" in init) @@ -261,6 +286,14 @@ destroy) shift destroy "$@" ;; +shell) + if [ -z "${2:-}" ]; then + echo "Usage: $0 shell cluster-name" + exit 1 + fi + shift + shell "$@" +;; post-install) if [ -z "${2:-}" ]; then echo "Usage: $0 post-install cluster-name" @@ -270,7 +303,7 @@ post-install) post_install "$@" ;; *) - echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|ping|start|stop} cluster-name" + echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|shell|ping|start|stop} cluster-name" exit 1 ;; esac diff --git a/main.tf b/main.tf index d5dd241..d2f5958 100644 --- a/main.tf +++ b/main.tf @@ -25,11 +25,11 @@ terraform { } locals { - master_nodes = { for i in libvirt_domain.master : i.name => i.network_interface.0.addresses[0] } - worker_nodes = { for i in libvirt_domain.worker : i.name => i.network_interface.0.addresses[0] } - bootstrap_nodes = { for i in libvirt_domain.bootstrap : i.name => i.network_interface.0.addresses[0] } - additional_nodes = { (libvirt_domain.lb.name) = [ libvirt_domain.storage.network_interface.0.addresses[0], libvirt_domain.storage.network_interface.1.addresses[0] ], (libvirt_domain.storage.name) = libvirt_domain.storage.network_interface.0.addresses[0] } - all_nodes = merge(local.additional_nodes, local.master_nodes, local.worker_nodes, local.bootstrap_nodes) + master_nodes = [ for i in libvirt_domain.master : { name = i.name, ip = i.network_interface.0.addresses[0], role = "master" } ] + worker_nodes = [ for i in libvirt_domain.worker : { name = i.name, ip = i.network_interface.0.addresses[0], role = "worker" } ] + bootstrap_nodes = [ for i in libvirt_domain.bootstrap : { name = i.name, ip = i.network_interface.0.addresses[0], role = "bootstrap" } ] + additional_nodes = [ { name = (libvirt_domain.lb.name), ip = [ libvirt_domain.lb.network_interface.0.addresses[0], libvirt_domain.lb.network_interface.1.addresses[0] ], role = "lb" }, { name = (libvirt_domain.storage.name), ip = libvirt_domain.storage.network_interface.0.addresses[0], role = "storage" } ] + all_nodes = concat(local.additional_nodes, local.master_nodes, local.worker_nodes, local.bootstrap_nodes) } output "machines" { diff --git a/post-install.tf b/post-install.tf index f5c3313..f630677 100644 --- a/post-install.tf +++ b/post-install.tf @@ -16,45 +16,8 @@ resource "local_file" "dns_config" { file_permission = "0644" } -resource "local_file" "stop_sh" { - content = templatefile("${path.module}/templates/stop.sh", { masters = libvirt_domain.master.*.name, workers = libvirt_domain.worker.*.name, lb = libvirt_domain.lb.name, storage = libvirt_domain.storage.name }) - filename = "${var.cluster_name}/stop.sh" - file_permission = "0755" -} - -resource "local_file" "start_sh" { - content = templatefile("${path.module}/templates/start.sh", { masters = local.master_nodes, workers = local.worker_nodes, others = local.additional_nodes }) - filename = "${var.cluster_name}/start.sh" - file_permission = "0755" -} - -resource "null_resource" "dnsmasq_config" { - triggers = { - network_id = libvirt_network.ocp_net.id - libvirt_server = local.libvirt_server - libvirt_username = local.libvirt_username - network_domain = local.network_domain - } - - connection { - type = "ssh" - host = self.triggers.libvirt_server - user = self.triggers.libvirt_username - } - - provisioner "remote-exec" { - inline = [ - "echo 'server=/${local.network_domain}/${cidrhost(var.network_ip_range, 1)}' | sudo tee /etc/NetworkManager/dnsmasq.d/zone-${local.network_domain}.conf", - "sudo pkill -f '[d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq'" - ] - } - - provisioner "remote-exec" { - when = destroy - on_failure = continue - inline = [ - "sudo rm -f /etc/NetworkManager/dnsmasq.d/zone-${self.triggers.network_domain}.conf", - "sudo pkill -f '[d]nsmasq.*--enable-dbus=org.freedesktop.NetworkManager.dnsmasq'" - ] - } +resource "local_file" "ansible_inventory" { + content = templatefile("${path.module}/templates/inventory", { nodes = local.all_nodes, network_domain = local.network_domain, dns_server = cidrhost(var.network_ip_range, 1) }) + filename = "${var.cluster_name}/inventory" + file_permission = "0644" } diff --git a/provider.tf b/provider.tf index cf18d56..f24a453 100644 --- a/provider.tf +++ b/provider.tf @@ -1,14 +1,7 @@ provider "libvirt" { - uri = "qemu:///system" } provider "gandi" { # key = "" # sharing_id = "" } - -locals { - # See post-install.tf - libvirt_server = "hp-ml350.itix.fr" - libvirt_username = "nicolas" -} diff --git a/templates/inventory b/templates/inventory new file mode 100644 index 0000000..16a7ed7 --- /dev/null +++ b/templates/inventory @@ -0,0 +1,6 @@ +[hypervisor] + +[hypervisor:vars] +network_domain=${network_domain} +dns_server=${dns_server} +nodes=${jsonencode(nodes)} \ No newline at end of file diff --git a/templates/start.sh b/templates/start.sh deleted file mode 100644 index df477be..0000000 --- a/templates/start.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -set -Eeuo pipefail -trap "exit" INT - -function start () { - for i; do - sudo virsh start "$i" || true - done -} - -function wait_for_ip () { - echo "Waiting for $1 to come online..." - while ! ping -n -c4 -i.2 $2 -q &>/dev/null; do - sleep 1 - done -} - -%{for host, ip in others~} -start "${host}" -wait_for_ip "${host}" "${ip}" -%{endfor~} - -%{for host, ip in masters~} -start "${host}" -%{endfor~} - -%{for host, ip in workers~} -start "${host}" -%{endfor~} diff --git a/templates/stop.sh b/templates/stop.sh deleted file mode 100644 index 346e9aa..0000000 --- a/templates/stop.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -set -Eeuo pipefail -trap "exit" INT - -function stop_group () { - for i; do - sudo virsh shutdown "$i" --mode=agent || true - done - - for i; do - echo "Waiting for $i to shutdown..." - while sudo virsh list --name | egrep -q "^$i\$"; do - sleep 1 - continue - done - done -} - -stop_group %{for host in workers}"${host}" %{endfor} -stop_group %{for host in masters}"${host}" %{endfor} -stop_group "${lb}" "${storage}"