openshift-4-installation/clusterctl

#!/bin/bash

set -Eeuo pipefail
trap "exit" INT

function assert_cluster_name () {
  local cluster_name="${1:-}"

  if [ ! -d ".clusters/$cluster_name" ]; then
    echo "Cluster '$cluster_name' does not exist!"
    exit 1
  fi

  if [ -f ".clusters/$cluster_name/local.env" ]; then
    source ".clusters/$cluster_name/local.env"
  fi
}

function init () {
  local cluster_name="${1:-}"

  if [ -d ".clusters/$cluster_name" ]; then
    echo "Cluster '$cluster_name' already initialized !"
    exit 1
  fi

  mkdir -p ".clusters/$cluster_name"
  sed "s/__CLUSTER_NAME__/$cluster_name/" install-config.yaml > ".clusters/$cluster_name/install-config.yaml"
  sed "s/__CLUSTER_NAME__/$cluster_name/" terraform.tfvars > ".clusters/$cluster_name/terraform.tfvars"
  grep LIBVIRT_DEFAULT_URI local.env > ".clusters/$cluster_name/local.env"

  echo "Cluster $cluster_name initialized successfully!"
  echo
  echo "Review and adjust the following files to your needs:"
  echo "- .clusters/$cluster_name/install-config.yaml"
  echo "- .clusters/$cluster_name/terraform.tfvars"
  echo "- .clusters/$cluster_name/local.env"
  echo
  exit 0
}

function destroy () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  terraform destroy -var-file=".clusters/$cluster_name/terraform.tfvars" -state=".clusters/$cluster_name/terraform.tfstate"
  sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 1/' ".clusters/$cluster_name/terraform.tfvars"
  if [ -f ".clusters/$cluster_name/install-config.yaml.bak" ]; then
    cp ".clusters/$cluster_name/install-config.yaml.bak" ".clusters/$cluster_name/install-config.yaml"
  fi
  rm -rf .clusters/$cluster_name/{*.ign,metadata.json,auth,.openshift*,manifests}
}

function prepare () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  # Make a backup since the openshift-install command will consume it
  if [ -f ".clusters/$cluster_name/install-config.yaml" ]; then
    cp ".clusters/$cluster_name/install-config.yaml" ".clusters/$cluster_name/install-config.yaml.bak"
  fi

  # Include the cluster dir in the path for disconnected installations
  export PATH="$PWD/.clusters/$cluster_name:$PATH"
  openshift-install version

  # Create installation files
  openshift-install create manifests --dir=".clusters/$cluster_name"
}

function apply () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  prepare "$cluster_name"

  # Create installation files
  openshift-install create ignition-configs --dir=".clusters/$cluster_name"

  # Provision the infrastructure and wait for bootstrap to complete
  terraform apply -var-file=".clusters/$cluster_name/terraform.tfvars" -state=".clusters/$cluster_name/terraform.tfstate" -auto-approve
  openshift-install --dir=".clusters/$cluster_name" wait-for bootstrap-complete --log-level=info

  # Destroy the bootstrap node
  sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 0/' ".clusters/$cluster_name/terraform.tfvars"
  terraform apply -var-file=".clusters/$cluster_name/terraform.tfvars" -state=".clusters/$cluster_name/terraform.tfstate" -auto-approve

  # Auto-approve all pending CSRs
  for i in {0..240}; do
    approve_csr "$cluster_name"
    sleep 15
  done &

  # Wait for the installation to complete
  openshift-install --dir=".clusters/$cluster_name" wait-for install-complete
}

function ping () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" whoami
}

function approve_csr () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" get csr --no-headers \
     | awk '/Pending/ {print $1}' \
     | xargs --no-run-if-empty oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" adm certificate approve
}

function start () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  ansible-playbook -i ".clusters/$cluster_name/inventory" ansible/start.yaml
}

function stop () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  ansible-playbook -i ".clusters/$cluster_name/inventory" ansible/stop.yaml
}

function post_install_nfs () {
  local cluster_name="${1:-}"

  oc apply --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" -f ".clusters/$cluster_name/registry-pv.yaml"
  oc patch --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type=json --patch-file=/dev/fd/0 <<EOF
[{"op": "remove", "path": "/spec/storage" },{"op": "add", "path": "/spec/storage", "value": {"pvc":{"claim": "registry-storage"}}}]
EOF
  oc apply --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" -f ".clusters/$cluster_name/nfs-provisioner.yaml"
  oc patch --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"managementState": "Managed"}}
EOF
}

function post_install_le () {
  local cluster_name="${1:-}"

  cert_dn="$(openssl x509 -noout -subject -in ".clusters/$cluster_name/cluster.crt")"
  cert_cn="${cert_dn#subject=CN = }"

  # Deploy certificate to ingress
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" create secret tls router-certs-$(date "+%Y-%m-%d") --cert=".clusters/$cluster_name/cluster.crt" --key=".clusters/$cluster_name/cluster.key" -n openshift-ingress --dry-run -o yaml > ".clusters/$cluster_name/router-certs.yaml"
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f ".clusters/$cluster_name/router-certs.yaml" -n openshift-ingress
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" patch ingresscontroller default -n openshift-ingress-operator --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec": { "defaultCertificate": { "name": "router-certs-$(date "+%Y-%m-%d")" }}}
EOF

  # Deploy certificate to api
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" create secret tls api-certs-$(date "+%Y-%m-%d") --cert=".clusters/$cluster_name/cluster.crt" --key=".clusters/$cluster_name/cluster.key" -n openshift-config --dry-run -o yaml > ".clusters/$cluster_name/api-certs.yaml"
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f ".clusters/$cluster_name/api-certs.yaml" -n openshift-config
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" patch apiserver cluster --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"servingCerts":{"namedCertificates":[{"names":["$cert_cn"],"servingCertificate":{"name": "api-certs-$(date "+%Y-%m-%d")"}}]}}}
EOF
}

function post_install_sso () {
  local cluster_name="${1:-}"

  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" create secret generic redhat-sso-client-secret -n openshift-config --from-literal="clientSecret=$GOOGLE_CLIENT_SECRET" --dry-run -o yaml > ".clusters/$cluster_name/sso-secret.yaml"
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f ".clusters/$cluster_name/sso-secret.yaml"
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: config.openshift.io/v1
kind: OAuth
metadata:
  name: cluster
spec:
  identityProviders:
  - google:
      clientID: "$GOOGLE_CLIENT_ID"
      clientSecret:
        name: redhat-sso-client-secret
      hostedDomain: redhat.com
    mappingMethod: claim
    name: RedHatSSO
    type: Google
EOF
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" adm policy add-cluster-role-to-user cluster-admin "$OCP_ADMIN"
}

function post_install () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"
  shift

  if [ $# -eq 0 ]; then
    set nfs sso le
  fi
  
  for i; do
    post_install_$i "$cluster_name"
  done
}

function install_addon_acmhub () {
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: v1
kind: Namespace
metadata:
  name: open-cluster-management
spec:
  finalizers:
  - kubernetes
EOF
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
  name: open-cluster-management
  namespace: open-cluster-management
spec:
  targetNamespaces:
  - open-cluster-management
EOF
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: acm-operator-subscription
  namespace: open-cluster-management
spec:
  sourceNamespace: openshift-marketplace
  source: redhat-operators
  channel: release-2.2
  installPlanApproval: Automatic
  name: advanced-cluster-management
EOF
  while ! oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" -n open-cluster-management get mch --all-namespaces -o yaml &>/dev/null; do
    echo "Waiting for the MultiClusterHub CRD to appear..."
    sleep 5
  done
  oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: operator.open-cluster-management.io/v1
kind: MultiClusterHub
metadata:
  name: multiclusterhub
  namespace: open-cluster-management
EOF
  echo
  echo "RH-ACM Current state is: $(oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" get mch multiclusterhub -n open-cluster-management -o=jsonpath='{.status.phase}')"
  echo
  echo "RH-ACM Console: $(oc --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" get route multicloud-console -n open-cluster-management -o jsonpath="https://{.spec.host}")"
  echo
}

function install_addon () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"
  local addon="${2:-}"

  install_addon_$addon "$cluster_name"
}

function shell () {
  assert_cluster_name "$@"
  local cluster_name="${1:-}"

  # Ansible
  export DEFAULT_HOST_LIST="$PWD/.clusters/$cluster_name"

  # Terraform
  export TF_CLI_ARGS_plan="-var-file=.clusters/$cluster_name/terraform.tfvars -state=.clusters/$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_apply="-var-file=.clusters/$cluster_name/terraform.tfvars -state=.clusters/$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_destroy="-var-file=.clusters/$cluster_name/terraform.tfvars -state=.clusters/$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_state_list="-state=.clusters/$cluster_name/terraform.tfstate"
  export TF_CLI_ARGS_state_rm="-state=.clusters/$cluster_name/terraform.tfstate"

  # Include the cluster dir in the path for disconnected installations
  export PATH="$PWD/.clusters/$cluster_name:$PATH"

  # OpenShift
  export KUBECONFIG="$PWD/.clusters/$cluster_name/auth/kubeconfig"
  export CLUSTER_NAME="$cluster_name"
  export PS1="[$CLUSTER_NAME:\w] "
  kubectl config set-cluster "$CLUSTER_NAME" --insecure-skip-tls-verify=true
  exec /bin/bash
}

if [ ! -e "local.env" ]; then
  echo "Please create local.env first!"
  exit 1
fi

source local.env
export LC_ALL=C.utf-8
export LANG=C.utf-8

case "${1:-}" in
init)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 init cluster-name"
    exit 1
  fi
  shift
  init "$@"
;;
prepare)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 prepare cluster-name"
    exit 1
  fi
  shift
  prepare "$@"
;;
start)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 start cluster-name"
    exit 1
  fi
  shift
  start "$@"
;;
stop)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 stop cluster-name"
    exit 1
  fi
  shift
  stop "$@"
;;
apply)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 apply cluster-name"
    exit 1
  fi
  shift
  apply "$@"
;;
approve-csr)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 approve-csr cluster-name"
    exit 1
  fi
  shift
  approve_csr "$@"
;;
ping)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 ping cluster-name"
    exit 1
  fi
  shift
  ping "$@"
;;
destroy)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 destroy cluster-name"
    exit 1
  fi
  shift
  destroy "$@"
;;
shell)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 shell cluster-name"
    exit 1
  fi
  shift
  shell "$@"
;;
post-install)
  if [ -z "${2:-}" ]; then
    echo "Usage: $0 post-install cluster-name"
    exit 1
  fi
  shift
  post_install "$@"
;;
install-addon)
  if [ -z "${2:-}" -o -z "${3:-}" ]; then
    echo "Usage: $0 install-addon cluster-name addon-name"
    exit 1
  fi
  shift
  install_addon "$@"
;;
*)
  echo "Usage: $0 {init|apply|approve-csr|post-install|destroy|shell|ping|start|stop} cluster-name"
  exit 1
;;
esac