openshift-4-installation/cluster

#!/bin/bash

set -Eeuo pipefail
trap "exit" INT

function init () {
  local cluster_name="${1:-}"

  if [ -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' already initialized !"
    exit 1
  fi

  cluster_name="$1"

  mkdir -p "$cluster_name"
  sed "s/__CLUSTER_NAME__/$cluster_name/" install-config.yaml > "$cluster_name/install-config.yaml"
  sed "s/__CLUSTER_NAME__/$cluster_name/" terraform.tfvars > "$cluster_name/terraform.tfvars"

  echo "Cluster $cluster_name initialized successfully!"
  echo
  echo "Review and adjust the following files to your needs:"
  echo "- $cluster_name/install-config.yaml"
  echo "- $cluster_name/terraform.tfvars"
  echo
  exit 0
}

function destroy () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  terraform destroy -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate"
  sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 1/' "$cluster_name/terraform.tfvars"
}

function apply () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  # Create installation files
  openshift-install create manifests --dir="$cluster_name"
  openshift-install create ignition-configs --dir="$cluster_name"

  # Provision the infrastructure and wait for bootstrap to complete
  terraform apply -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate" -auto-approve
  openshift-install --dir="$cluster_name" wait-for bootstrap-complete --log-level=info

  # Destroy the bootstrap node
  sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 0/' "$cluster_name/terraform.tfvars"
  terraform apply -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate" -auto-approve

  # Auto-approve all pending CSRs
  for i in {0..240}; do
    approve_csr "$cluster_name"
    sleep 15
  done &

  # Wait for the installation to complete
  openshift-install --dir="$cluster_name" wait-for install-complete
}

function ping () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" whoami
}

function approve_csr () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" get csr --no-headers \
     | awk '/Pending/ {print $1}' \
     | xargs --no-run-if-empty oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" adm certificate approve
}

function start () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  ansible-playbook -i "$cluster_name/inventory" ansible/start.yaml
}

function stop () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  ansible-playbook -i "$cluster_name/inventory" ansible/stop.yaml
}

function post_install_nfs () {
  local cluster_name="${1:-}"

  oc apply --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" -f "$cluster_name/registry-pv.yaml"
  oc patch --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type=json --patch-file=/dev/fd/0 <<EOF
[{"op": "remove", "path": "/spec/storage" },{"op": "add", "path": "/spec/storage", "value": {"pvc":{"claim": "registry-storage"}}}]
EOF
  oc apply --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" -f "$cluster_name/nfs-provisioner.yaml"
  oc patch --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"managementState": "Managed"}}
EOF
}

function post_install_le () {
  local cluster_name="${1:-}"

  cert_dn="$(openssl x509 -noout -subject -in "$cluster_name/cluster.crt")"
  cert_cn="${cert_dn#subject=CN = }"

  # Deploy certificate to ingress
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret tls router-certs-$(date "+%Y-%m-%d") --cert="$cluster_name/cluster.crt" --key="$cluster_name/cluster.key" -n openshift-ingress --dry-run -o yaml > "$cluster_name/router-certs.yaml"
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/router-certs.yaml" -n openshift-ingress
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" patch ingresscontroller default -n openshift-ingress-operator --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec": { "defaultCertificate": { "name": "router-certs-$(date "+%Y-%m-%d")" }}}
EOF

  # Deploy certificate to api
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret tls api-certs-$(date "+%Y-%m-%d") --cert="$cluster_name/cluster.crt" --key="$cluster_name/cluster.key" -n openshift-config --dry-run -o yaml > "$cluster_name/api-certs.yaml"
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/api-certs.yaml" -n openshift-config
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" patch apiserver cluster --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"servingCerts":{"namedCertificates":[{"names":["$cert_cn"],"servingCertificate":{"name": "api-certs-$(date "+%Y-%m-%d")"}}]}}}
EOF
}

function post_install_sso () {
  local cluster_name="${1:-}"

  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret generic redhat-sso-client-secret -n openshift-config --from-literal="clientSecret=$GOOGLE_CLIENT_SECRET" --dry-run -o yaml > "$cluster_name/sso-secret.yaml"
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/sso-secret.yaml"
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: config.openshift.io/v1
kind: OAuth
metadata:
  name: cluster
spec:
  identityProviders:
  - google:
      clientID: "$GOOGLE_CLIENT_ID"
      clientSecret:
        name: redhat-sso-client-secret
      hostedDomain: redhat.com
    mappingMethod: claim
    name: RedHatSSO
    type: Google
EOF
  oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" adm policy add-cluster-role-to-user cluster-admin "$OCP_ADMIN"
}

function post_install () {
  local cluster_name="${1:-}"
  shift

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  if [ $# -eq 0 ]; then
    set nfs sso le
  fi
  
  for i; do
    post_install_$i "$cluster_name"
  done
}

function shell () {
  local cluster_name="${1:-}"

  if [ ! -d "$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  # Ansible
  export DEFAULT_HOST_LIST="$PWD/$cluster_name"

  # Terraform
  export TF_CLI_ARGS_plan="-var-file=$cluster_name/terraform.tfvars -state=$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_apply="-var-file=$cluster_name/terraform.tfvars -state=$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_destroy="-var-file=$cluster_name/terraform.tfvars -state=$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_state_list="-state=$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_state_rm="-state=$cluster_name/terraform.tfstate"

  # OpenShift
  export KUBECONFIG="$PWD/$cluster_name/auth/kubeconfig"
  export OC_BINARY="$(which oc)"
  export CLUSTER_NAME="$cluster_name"
  export PS1="[$CLUSTER_NAME:\w] "
  function oc () {
    "$OC_BINARY" --insecure-skip-tls-verify "$@"
  }
  export -f oc
  exec /bin/bash
}

if [ ! -e "local.env" ]; then
  echo "Please create local.env first!"
  exit 1
fi

source local.env
export LC_ALL=C
export LANG=C
export LIBVIRT_DEFAULT_URI="qemu+ssh://$LIBVIRT_USER@$LIBVIRT_SERVER/system"

case "${1:-}" in
init)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 init cluster-name"
    exit 1
  fi
  shift
  bootstrap "$@"
;;
start)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 start cluster-name"
    exit 1
  fi
  shift
  start "$@"
;;
stop)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 stop cluster-name"
    exit 1
  fi
  shift
  stop "$@"
;;
apply)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 apply cluster-name"
    exit 1
  fi
  shift
  apply "$@"
;;
approve-csr)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 approve-csr cluster-name"
    exit 1
  fi
  shift
  approve_csr "$@"
;;
ping)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 ping cluster-name"
    exit 1
  fi
  shift
  ping "$@"
;;
destroy)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 destroy cluster-name"
    exit 1
  fi
  shift
  destroy "$@"
;;
shell)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 shell cluster-name"
    exit 1
  fi
  shift
  shell "$@"
;;
post-install)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 post-install cluster-name"
    exit 1
  fi
  shift
  post_install "$@"
;;
*)
  echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|shell|ping|start|stop} cluster-name"
  exit 1
;;
esac