Browse Source

install script and doc

standalone
Nicolas Massé 5 years ago
parent
commit
4b4e329ad9
  1. 2
      .gitignore
  2. 154
      README.md
  3. 195
      cluster
  4. 4
      install-config.yaml.sample
  5. 6
      local.env.sample
  6. 5
      terraform.tfvars.sample

2
.gitignore

@ -5,3 +5,5 @@
.vscode .vscode
terraform.tfvars terraform.tfvars
install-config.yaml install-config.yaml
.lego
local.env

154
README.md

@ -36,165 +36,59 @@ make
make install make install
``` ```
### On the hypervisor Install lego.
```sh
curl https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.7/4.7.0/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2.gz |gunzip -c > /var/lib/libvirt/images/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2
curl -Lo /var/lib/libvirt/images/centos-stream-8.qcow2 http://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2
```
## Install
Define the cluster name and the bastion.
```sh
cluster=ocp4
bastion=nicolas@hp-ml350.itix.fr
```
Install **openshift-installer** and **oc** on the bastion.
```sh
ssh -A "$bastion" curl -Lo /tmp/openshift-installer.tgz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.7/openshift-install-linux.tar.gz
ssh -A "$bastion" sudo tar zxvf /tmp/openshift-installer.tgz -C /usr/local/bin openshift-install
ssh -A "$bastion" curl -Lo /tmp/oc.tgz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.7/openshift-client-linux.tar.gz
ssh -A "$bastion" sudo tar zxvf /tmp/oc.tgz -C /usr/local/bin oc kubectl
```
Create the cluster configuration files.
```sh
mkdir "$cluster"
cp install-config.yaml.sample "$cluster/install-config.yaml"
openshift-install create manifests --dir="$cluster"
openshift-install create ignition-configs --dir="$cluster"
```
Customize the terraform variables.
```sh
cat > terraform.tfvars <<EOF
base_domain = "itix.xyz"
external_mac_address = "02:00:00:00:00:04"
public_cluster_ip = "90.79.1.247"
cluster_name = "$cluster"
EOF
```
Apply the terraform plan.
```sh
export GANDI_KEY="123...456"
terraform apply
```
Copy the cluster definition on the bastion and run the bootstrap process from there.
```sh
scp -r "$cluster" "$bastion:'$cluster'"
ssh -A "$bastion" openshift-install --dir="$cluster" wait-for bootstrap-complete --log-level=info
```
Delete the bootstrap node.
```sh
echo 'bootstrap_nodes = 0' >> terraform.tfvars
terraform apply
```
Approve the pending CSRs.
```sh
for i in {0..120}; do
ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" get csr --no-headers \
| awk '/Pending/ {print $1}' \
| xargs --no-run-if-empty ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" adm certificate approve
sleep 15
done &
```
Make sure all CSRs have been issued.
```sh
ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" get csr --no-headers
```
Provision storage for the registry.
```sh ```sh
ssh -A "$bastion" oc apply --kubeconfig="$cluster/auth/kubeconfig" -f - < "$cluster/registry-pv.yaml" curl -Lo /tmp/lego.tgz https://github.com/go-acme/lego/releases/download/v4.3.1/lego_v4.3.1_linux_amd64.tar.gz
sudo tar zxvf /tmp/lego.tgz -C /usr/local/bin lego
``` ```
Patch the registry to use the new storage. ### On the server
```sh
ssh -A "$bastion" oc patch --kubeconfig="$cluster/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type='json' --patch-file=/dev/fd/0 <<EOF
[{"op": "remove", "path": "/spec/storage" },{"op": "add", "path": "/spec/storage", "value": {"pvc":{"claim": "registry-storage"}}}]
EOF
```
Deploy the NFS provisioner. Install libvirt.
```sh ```sh
ssh -A "$bastion" oc apply --kubeconfig="$cluster/auth/kubeconfig" -f - < "$cluster/nfs-provisioner.yaml" sudo dnf install libvirt libvirt-daemon-kvm virt-install virt-viewer virt-top libguestfs-tools nmap-ncat
``` ```
Set image registry managementState from Removed to Managed. Configure NetworkManager to use dnsmasq. In **/etc/NetworkManager/NetworkManager.conf**:
```sh ```ini
ssh -A "$bastion" oc patch --kubeconfig="$cluster/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type merge --patch-file=/dev/fd/0 <<EOF [main]
{"spec":{"managementState": "Managed"}} dns=dnsmasq
EOF
``` ```
Wait for installation to finish. Download the required images.
```sh ```sh
ssh -A "$bastion" openshift-install --dir="$cluster" wait-for install-complete curl https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.7/4.7.0/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2.gz |gunzip -c > /var/lib/libvirt/images/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2
curl -Lo /var/lib/libvirt/images/centos-stream-8.qcow2 http://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2
``` ```
## Let's encrypt certificates ## Install
Install lego.
```sh
curl -Lo /tmp/lego.tgz https://github.com/go-acme/lego/releases/download/v4.3.1/lego_v4.3.1_linux_amd64.tar.gz
sudo tar zxvf /tmp/lego.tgz -C /usr/local/bin lego
```
Request a public certificate. Create the template files from their sample.
```sh ```sh
export GANDIV5_API_KEY="123...456" cp terraform.tfvars.sample terraform.tfvars
. "$cluster/dns.env" cp local.env.sample local.env
lego -m "nmasse@redhat.com" -d "$LE_API_HOSTNAME" -d "$LE_ROUTER_HOSTNAME" -a --dns gandi run --no-bundle cp install-config.yaml.sample install-config.yaml
``` ```
Create a secret containing the new router certificate. Initialize a new cluster.
```sh ```sh
oc create secret tls router-certs-$(date "+%Y-%m-%d") --cert=$HOME/.lego/certificates/$LE_API_HOSTNAME.crt --key=$HOME/.lego/certificates/$LE_API_HOSTNAME.key -n openshift-ingress --dry-run -o yaml > router.yaml ./cluster init my-cluster
ssh -A "$bastion" oc apply -f - -n openshift-ingress < router.yaml
``` ```
Update the ingress configuration. Deploy the cluster.
```sh ```sh
ssh -A "$bastion" oc patch ingresscontroller default -n openshift-ingress-operator --type=merge --patch-file=/dev/fd/0 <<EOF ./cluster apply my-cluster
{"spec": { "defaultCertificate": { "name": "$(date "+%Y-%m-%d")" }}}
EOF
``` ```
Create a secret containing the new certificate. Do the post-install on the cluster.
```sh ```sh
oc create secret tls api-certs-$(date "+%Y-%m-%d") --cert=$HOME/.lego/certificates/$LE_API_HOSTNAME.crt --key=$HOME/.lego/certificates/$LE_API_HOSTNAME.key -n openshift-config --dry-run -o yaml > api.yaml ./cluster post-install my-cluster
ssh -A "$bastion" oc apply -f - -n openshift-config < api.yaml
``` ```
Update the apiserver configuration.
```sh
oc patch apiserver cluster --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"servingCerts":{"namedCertificates":[{"names":["'$LE_API'"],"servingCertificate":{"name": "api-certs-$(date "+%Y-%m-%d")"}}]}}}
EOF

195
cluster

@ -0,0 +1,195 @@
#!/bin/bash
set -Eeuo pipefail
trap "exit" INT
function init () {
local cluster_name="${1:-}"
if [ -d "$cluster_name" ]; then
echo "Cluster '$cluster_name' already initialized !"
exit 1
fi
cluster_name="$1"
mkdir -p "$cluster_name"
sed "s/__CLUSTER_NAME__/$cluster_name/" install-config.yaml > "$cluster_name/install-config.yaml"
sed "s/__CLUSTER_NAME__/$cluster_name/" terraform.tfvars > "$cluster_name/terraform.tfvars"
echo "Cluster $cluster_name initialized successfully!"
echo
echo "Review and adjust the following files to your needs:"
echo "- $cluster_name/install-config.yaml"
echo "- $cluster_name/terraform.tfvars"
echo
exit 0
}
function destroy () {
local cluster_name="${1:-}"
if [ ! -d "$cluster_name" ]; then
echo "Cluster '$cluster_name' does not exist!"
exit 1
fi
terraform destroy -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate"
sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 1/' "$cluster_name/terraform.tfvars"
}
function apply () {
local cluster_name="${1:-}"
if [ ! -d "$cluster_name" ]; then
echo "Cluster '$cluster_name' does not exist!"
exit 1
fi
# Create installation files
openshift-install create manifests --dir="$cluster_name"
openshift-install create ignition-configs --dir="$cluster_name"
# Provision the infrastructure and wait for bootstrap to complete
terraform apply -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate" -auto-approve
openshift-install --dir="$cluster_name" wait-for bootstrap-complete --log-level=info
# Destroy the bootstrap node
sed -i.bak 's/^\s*bootstrap_nodes\s*=\s*.*$/bootstrap_nodes = 0/' "$cluster_name/terraform.tfvars"
terraform apply -var-file="$cluster_name/terraform.tfvars" -state="$cluster_name/terraform.tfstate" -auto-approve
# Auto-approve all pending CSRs
for i in {0..240}; do
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" get csr --no-headers \
| awk '/Pending/ {print $1}' \
| xargs --no-run-if-empty oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" adm certificate approve
sleep 15
done &
# Wait for the installation to complete
openshift-install --dir="$cluster_name" wait-for install-complete
}
function post_install_nfs () {
local cluster_name="${1:-}"
oc apply --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" -f "$cluster_name/registry-pv.yaml"
oc patch --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type=json --patch-file=/dev/fd/0 <<EOF
[{"op": "remove", "path": "/spec/storage" },{"op": "add", "path": "/spec/storage", "value": {"pvc":{"claim": "registry-storage"}}}]
EOF
oc apply --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" -f "$cluster_name/nfs-provisioner.yaml"
oc patch --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"managementState": "Managed"}}
EOF
}
function post_install_le () {
local cluster_name="${1:-}"
# Generated by terraform
source "$cluster_name/dns.env"
# Get a certificate from Let's Encrypt
lego -m "nmasse@redhat.com" -d "$LE_API_HOSTNAME" -d "$LE_ROUTER_HOSTNAME" -a --dns gandiv5 run --no-bundle
# Deploy certificate to ingress
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret tls router-certs-$(date "+%Y-%m-%d") --cert=.lego/certificates/$LE_API_HOSTNAME.crt --key=.lego/certificates/$LE_API_HOSTNAME.key -n openshift-ingress --dry-run -o yaml > "$cluster_name/router-certs.yaml"
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/router-certs.yaml" -n openshift-ingress
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" patch ingresscontroller default -n openshift-ingress-operator --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec": { "defaultCertificate": { "name": "router-certs-$(date "+%Y-%m-%d")" }}}
EOF
# Deploy certificate to api
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret tls api-certs-$(date "+%Y-%m-%d") --cert=.lego/certificates/$LE_API_HOSTNAME.crt --key=.lego/certificates/$LE_API_HOSTNAME.key -n openshift-config --dry-run -o yaml > "$cluster_name/api-certs.yaml"
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/router-certs.yaml" -n openshift-ingress
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" patch apiserver cluster --type=merge --patch-file=/dev/fd/0 <<EOF
{"spec":{"servingCerts":{"namedCertificates":[{"names":["$LE_API_HOSTNAME"],"servingCertificate":{"name": "api-certs-$(date "+%Y-%m-%d")"}}]}}}
EOF
}
function post_install_sso () {
local cluster_name="${1:-}"
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" create secret generic redhat-sso-client-secret -n openshift-config --from-literal="clientSecret=$GOOGLE_CLIENT_SECRET" --dry-run -o yaml > "$cluster_name/sso-secret.yaml"
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f "$cluster_name/sso-secret.yaml"
oc --insecure-skip-tls-verify --kubeconfig="$cluster_name/auth/kubeconfig" apply -f - <<EOF
apiVersion: config.openshift.io/v1
kind: OAuth
metadata:
name: cluster
spec:
identityProviders:
- google:
clientID: "$GOOGLE_CLIENT_ID"
clientSecret:
name: redhat-sso-client-secret
hostedDomain: redhat.com
mappingMethod: claim
name: RedHatSSO
type: Google
EOF
}
function post_install () {
local cluster_name="${1:-}"
shift
if [ ! -d "$cluster_name" ]; then
echo "Cluster '$cluster_name' does not exist!"
exit 1
fi
if [ $# -eq 0 ]; then
set nfs sso le
fi
for i; do
post_install_$i "$cluster_name"
done
}
if [ ! -e "local.env" ]; then
echo "Please create local.env first!"
exit 1
fi
source local.env
case "${1:-}" in
init)
if [ -z "${2:-}" ]; then
echo "Usage: $0 init cluster-name"
exit 1
fi
shift
bootstrap "$@"
;;
apply)
if [ -z "${2:-}" ]; then
echo "Usage: $0 apply cluster-name"
exit 1
fi
shift
apply "$@"
;;
destroy)
if [ -z "${2:-}" ]; then
echo "Usage: $0 destroy cluster-name"
exit 1
fi
shift
destroy "$@"
;;
post-install)
if [ -z "${2:-}" ]; then
echo "Usage: $0 post-install cluster-name"
exit 1
fi
shift
post_install "$@"
;;
*)
echo "Usage: $0 {init|apply|post-install|destroy} cluster-name"
exit 1
;;
esac

4
install-config.yaml.sample

@ -1,5 +1,5 @@
apiVersion: v1 apiVersion: v1
baseDomain: itix.xyz baseDomain: ocp.lab
compute: compute:
- name: worker - name: worker
hyperthreading: Enabled hyperthreading: Enabled
@ -9,7 +9,7 @@ controlPlane:
hyperthreading: Enabled hyperthreading: Enabled
replicas: 3 replicas: 3
metadata: metadata:
name: ocp4 name: __CLUSTER_NAME__
networking: networking:
clusterNetworks: clusterNetworks:
- cidr: 10.128.0.0/14 - cidr: 10.128.0.0/14

6
local.env.sample

@ -0,0 +1,6 @@
export BASTION=user@bastion.host
export GANDI_KEY="123...456"
export GANDIV5_API_KEY="123...456"
export GOOGLE_CLIENT_ID="client_id"
export GOOGLE_CLIENT_SECRET="client_secret"
export LE_EMAIL="user@redhat.com"

5
terraform.tfvars.sample

@ -0,0 +1,5 @@
base_domain = "ocp.lab"
external_mac_address = "02:00:00:00:00:04"
public_cluster_ip = "1.2.3.4"
cluster_name = "__CLUSTER_NAME__"
bootstrap_nodes = 1
Loading…
Cancel
Save