commit 3eb11f69fdd25abd1d711ac9c1fb2779f12a868f Author: Nicolas MASSE Date: Tue Mar 16 16:23:04 2021 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e57b9b5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.lock.hcl +*.tfstate +*.backup +.terraform +.vscode +terraform.tfvars +install-config.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000..cecf0bd --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# OpenShift 4 Installation + +## Pre-requisites + +### On your local machine + +Install Terraform. + +```sh +cat > hashicorp.repo <<"EOF" +[hashicorp] +name=Hashicorp Stable - $basearch +baseurl=https://rpm.releases.hashicorp.com/RHEL/8/$basearch/stable +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.releases.hashicorp.com/gpg +EOF +sudo dnf config-manager --add-repo hashicorp.repo +sudo dnf -y install terraform +``` + +Install the libvirt terraform provider. + +```sh +curl -Lo /tmp/libvirt-provider.tgz https://github.com/dmacvicar/terraform-provider-libvirt/releases/download/v0.6.3/terraform-provider-libvirt-0.6.3+git.1604843676.67f4f2aa.Fedora_32.x86_64.tar.gz +mkdir -p ~/.terraform.d/plugins/registry.terraform.io/dmacvicar/libvirt/0.6.3/linux_amd64 +tar xvf /tmp/libvirt-provider.tgz -C ~/.terraform.d/plugins/registry.terraform.io/dmacvicar/libvirt/0.6.3/linux_amd64 +``` + +Install the Gandi terraform provider. + +```sh +git clone https://github.com/go-gandi/terraform-provider-gandi +cd terraform-provider-gandi +make +make install +``` + +### On the hypervisor + +```sh +curl https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.7/4.7.0/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2.gz |gunzip -c > /var/lib/libvirt/images/rhcos-4.7.0-x86_64-qemu.x86_64.qcow2 +curl -Lo /var/lib/libvirt/images/centos-stream-8.qcow2 http://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2 +``` + +## Install + +Define the cluster name and the bastion. + +```sh +cluster=ocp4 +bastion=nicolas@hp-ml350.itix.fr +``` + +Install **openshift-installer** and **oc** on the bastion. + +```sh +ssh -A "$bastion" curl -Lo /tmp/openshift-installer.tgz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.7/openshift-install-linux.tar.gz +ssh -A "$bastion" sudo tar zxvf /tmp/openshift-installer.tgz -C /usr/local/bin openshift-install +ssh -A "$bastion" curl -Lo /tmp/oc.tgz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.7/openshift-client-linux.tar.gz +ssh -A "$bastion" sudo tar zxvf /tmp/oc.tgz -C /usr/local/bin oc kubectl +``` + +Create the cluster configuration files. + +```sh +mkdir "$cluster" +cp install-config.yaml.sample "$cluster/install-config.yaml" +openshift-install create manifests --dir="$cluster" +openshift-install create ignition-configs --dir="$cluster" +``` + +Customize the terraform variables. + +```sh +cat > terraform.tfvars <> terraform.tfvars +terraform apply +``` + +Approve the pending CSRs. + +```sh +for i in {0..120}; do + ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" get csr --no-headers \ + | awk '/Pending/ {print $1}' \ + | xargs --no-run-if-empty ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" adm certificate approve + sleep 15 +done & +``` + +Make sure all CSRs have been issued. + +```sh +ssh -A "$bastion" oc --kubeconfig="$cluster/auth/kubeconfig" get csr --no-headers +``` + +Provision storage for the registry. + +```sh +ssh -A "$bastion" oc apply --kubeconfig="$cluster/auth/kubeconfig" -f - < "$cluster/registry-pv.yaml" +``` + +Patch the registry to use the new storage. + +```sh +ssh -A "$bastion" oc patch --kubeconfig="$cluster/auth/kubeconfig" configs.imageregistry.operator.openshift.io cluster --type='json' --patch-file=/dev/fd/0 < router.yaml +ssh -A "$bastion" oc apply -f - -n openshift-ingress < router.yaml +``` + +Update the ingress configuration. + +```sh +ssh -A "$bastion" oc patch ingresscontroller default -n openshift-ingress-operator --type=merge --patch-file=/dev/fd/0 < api.yaml +ssh -A "$bastion" oc apply -f - -n openshift-config < api.yaml +``` + +Update the apiserver configuration. + +```sh +oc patch apiserver cluster --type=merge --patch-file=/dev/fd/0 < i.network_interface.0.addresses[0] }, + worker_nodes = { for i in libvirt_domain.worker : i.name => i.network_interface.0.addresses[0] }, + bootstrap_nodes = { for i in libvirt_domain.bootstrap : i.name => i.network_interface.0.addresses[0] } + }) + } +} + +data "template_file" "lb_network_config" { + template = file("${path.module}/templates/lb/network-config.cfg") + vars = { + ip = cidrhost(var.network_ip_range, 4) + dns = cidrhost(var.network_ip_range, 1) + gw = cidrhost(var.network_ip_range, 1) + } +} + +resource "libvirt_volume" "lb_disk" { + name = "${local.lb_name}.${var.volume_format}" + format = var.volume_format + pool = var.pool_name + base_volume_name = "${var.centos_image}.${var.volume_format}" + size = var.lb_disk_size +} + +resource "libvirt_domain" "lb" { + name = local.lb_name + vcpu = var.lb_vcpu + memory = var.lb_memory_size + cloudinit = libvirt_cloudinit_disk.lb_cloudinit.id + autostart = true + + disk { + volume_id = libvirt_volume.lb_disk.id + } + + # Makes the tty0 available via `virsh console` + console { + type = "pty" + target_port = "0" + } + + network_interface { + network_id = libvirt_network.ocp_net.id + addresses = [cidrhost(var.network_ip_range, 4)] + hostname = "lb" + wait_for_lease = false + } + + network_interface { + bridge = var.external_ifname + mac = var.external_mac_address + wait_for_lease = false + } +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..b4d0707 --- /dev/null +++ b/main.tf @@ -0,0 +1,35 @@ +terraform { + required_version = ">= 0.13" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = ">=0.6.3" + } + local = { + source = "hashicorp/local" + version = ">=2.0.0" + } + template = { + source = "hashicorp/template" + version = ">=2.2.0" + } + ignition = { + source = "community-terraform-providers/ignition" + version = "2.1.2" + } + gandi = { + version = "2.0.0" + source = "github/go-gandi/gandi" + } + } +} + +locals { + ocp_nodes = { for i in concat(libvirt_domain.bootstrap, libvirt_domain.master, libvirt_domain.worker) : i.name => i.network_interface.0.addresses[0] } + additional_nodes = { (libvirt_domain.lb.name) = cidrhost(var.network_ip_range, 4), (libvirt_domain.storage.name) = libvirt_domain.storage.network_interface.0.addresses[0] } + all_nodes = merge(local.ocp_nodes, local.additional_nodes) +} + +output "machines" { + value = local.all_nodes +} diff --git a/master.tf b/master.tf new file mode 100644 index 0000000..0ad793c --- /dev/null +++ b/master.tf @@ -0,0 +1,43 @@ +resource "libvirt_volume" "master_disk" { + name = "${format(local.master_format, count.index + 1)}.${var.volume_format}" + count = var.master_nodes + format = var.volume_format + pool = var.pool_name + base_volume_name = "${var.coreos_image}.${var.volume_format}" + size = var.master_disk_size +} + +resource "libvirt_ignition" "master_ignition" { + name = "${var.cluster_name}-master-ignition" + content = file("${path.module}/${var.cluster_name}/master.ign") +} + +resource "libvirt_domain" "master" { + count = var.master_nodes + name = format(local.master_format, count.index + 1) + vcpu = var.master_vcpu + memory = var.master_memory_size + coreos_ignition = libvirt_ignition.master_ignition.id + autostart = true + + disk { + volume_id = element(libvirt_volume.master_disk.*.id, count.index) + } + + # Makes the tty0 available via `virsh console` + console { + type = "pty" + target_port = "0" + } + + network_interface { + network_id = libvirt_network.ocp_net.id + addresses = [cidrhost(var.network_ip_range, 11 + count.index)] + hostname = format("master%d", count.index + 1) + + # When creating the domain resource, wait until the network interface gets + # a DHCP lease from libvirt, so that the computed IP addresses will be + # available when the domain is up and the plan applied. + wait_for_lease = true + } +} diff --git a/network.tf b/network.tf new file mode 100644 index 0000000..1458feb --- /dev/null +++ b/network.tf @@ -0,0 +1,36 @@ +resource "libvirt_network" "ocp_net" { + name = var.cluster_name + mode = "nat" + domain = local.network_domain + addresses = [var.network_ip_range] + autostart = true + + dns { + enabled = true + + hosts { + hostname = "host" + ip = cidrhost(var.network_ip_range, 1) + } + hosts { + hostname = "api" + ip = cidrhost(var.network_ip_range, 4) + } + hosts { + hostname = "api-int" + ip = cidrhost(var.network_ip_range, 4) + } + hosts { + hostname = "etcd" + ip = cidrhost(var.network_ip_range, 4) + } + } + + dhcp { + enabled = true + } + + xml { + xslt = templatefile("${path.module}/templates/network.xslt", { alias = "apps.${local.network_domain}", ip = cidrhost(var.network_ip_range, 4) }) + } +} diff --git a/post-install.tf b/post-install.tf new file mode 100644 index 0000000..c201202 --- /dev/null +++ b/post-install.tf @@ -0,0 +1,17 @@ +resource "local_file" "registry_pv" { + content = templatefile("${path.module}/templates/registry-pv.yaml", { nfs_server = libvirt_domain.storage.network_interface.0.addresses[0] }) + filename = "${var.cluster_name}/registry-pv.yaml" + file_permission = "0644" +} + +resource "local_file" "nfs_provisioner" { + content = templatefile("${path.module}/templates/nfs-provisioner.yaml", { nfs_server = libvirt_domain.storage.network_interface.0.addresses[0] }) + filename = "${var.cluster_name}/nfs-provisioner.yaml" + file_permission = "0644" +} + +resource "local_file" "dns_config" { + content = templatefile("${path.module}/templates/dns.env", { api_server = "api.${local.network_domain}", router = "*.apps.${local.network_domain}", dns_zone = var.base_domain, cluster_name = var.cluster_name }) + filename = "${var.cluster_name}/dns.env" + file_permission = "0644" +} diff --git a/provider.tf b/provider.tf new file mode 100644 index 0000000..75f2caa --- /dev/null +++ b/provider.tf @@ -0,0 +1,8 @@ +provider "libvirt" { + uri = "qemu:///system" +} + +provider "gandi" { + # key = "" + # sharing_id = "" +} diff --git a/public_dns.tf b/public_dns.tf new file mode 100644 index 0000000..87ee6c2 --- /dev/null +++ b/public_dns.tf @@ -0,0 +1,23 @@ +data "gandi_domain" "public_domain" { + name = var.base_domain +} + +resource "gandi_livedns_record" "api_record" { + zone = data.gandi_domain.public_domain.id + name = "api.ocp4" + type = "A" + ttl = 300 + values = [ + var.public_cluster_ip + ] +} + +resource "gandi_livedns_record" "router_record" { + zone = data.gandi_domain.public_domain.id + name = "*.apps.ocp4" + type = "A" + ttl = 300 + values = [ + var.public_cluster_ip + ] +} \ No newline at end of file diff --git a/storage.tf b/storage.tf new file mode 100644 index 0000000..8fc0c44 --- /dev/null +++ b/storage.tf @@ -0,0 +1,61 @@ +resource "libvirt_cloudinit_disk" "storage_cloudinit" { + name = "${local.storage_name}-cloudinit.iso" + user_data = data.template_file.storage_user_data.rendered + network_config = data.template_file.storage_network_config.rendered + pool = var.pool_name +} + +data "template_file" "storage_user_data" { + template = file("${path.module}/templates/storage/cloud-init.cfg") +} + +data "template_file" "storage_network_config" { + template = file("${path.module}/templates/storage/network-config.cfg") +} + +resource "libvirt_volume" "storage_os_disk" { + name = "${local.storage_name}-os.${var.volume_format}" + format = var.volume_format + pool = var.pool_name + base_volume_name = "${var.centos_image}.${var.volume_format}" +} + +resource "libvirt_volume" "storage_data_disk" { + name = "${local.storage_name}-data.${var.volume_format}" + format = var.volume_format + pool = var.pool_name + size = var.storage_disk_size +} + +resource "libvirt_domain" "storage" { + name = local.storage_name + vcpu = var.storage_vcpu + memory = var.storage_memory_size + cloudinit = libvirt_cloudinit_disk.storage_cloudinit.id + autostart = true + + disk { + volume_id = libvirt_volume.storage_os_disk.id + } + + disk { + volume_id = libvirt_volume.storage_data_disk.id + } + + # Makes the tty0 available via `virsh console` + console { + type = "pty" + target_port = "0" + } + + network_interface { + network_id = libvirt_network.ocp_net.id + addresses = [cidrhost(var.network_ip_range, 6)] + hostname = "storage" + + # When creating the domain resource, wait until the network interface gets + # a DHCP lease from libvirt, so that the computed IP addresses will be + # available when the domain is up and the plan applied. + wait_for_lease = true + } +} diff --git a/templates/dns.env b/templates/dns.env new file mode 100644 index 0000000..ec44b8c --- /dev/null +++ b/templates/dns.env @@ -0,0 +1,5 @@ +export LE_API_HOSTNAME="${api_server}" +export LE_ROUTER_HOSTNAME="${router}" +export DNS_ZONE="${dns_zone}" +export DNS_API_RECORD="api.${cluster_name}" +export DNS_ROUTER_RECORD="*.apps.${cluster_name}" diff --git a/templates/lb/cloud-init.cfg b/templates/lb/cloud-init.cfg new file mode 100644 index 0000000..2e0feba --- /dev/null +++ b/templates/lb/cloud-init.cfg @@ -0,0 +1,85 @@ +#cloud-config +# vim: syntax=yaml + +resize_rootfs: true + +users: +- name: nicolas + gecos: Nicolas MASSE + groups: wheel + lock_passwd: false + passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFW62WJXI1ZCMfNA4w0dMpL0fsldhbEfULNGIUB0nQui nmasse@localhost.localdomain + +packages: +# Useful tools +- net-tools +- hdparm +- iptraf +- iotop +- vim-enhanced +- tmux +- rsync +- tree +- unzip +- tar +- tcpdump +- telnet +- strace +- bind-utils +# Load Balancer +- haproxy +- firewalld + +runcmd: +# Enable KVM virsh console access +- [ "systemctl", "enable", "serial-getty@ttyS0.service" ] +- [ "systemctl", "start", "--no-block", "serial-getty@ttyS0.service" ] +# Disable SSH password authentication +- [ "sed", "-i.post-install", "-e", "s/PasswordAuthentication yes/PasswordAuthentication no/", "/etc/ssh/sshd_config" ] +- [ "systemctl", "restart", "sshd" ] +# Enable sudo without password +- [ "sed", "-i.post-install", "-e", "s/^%wheel\tALL=(ALL)\tALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers" ] +# Fix file permissions +- [ "chown", "-R", "nicolas:nicolas", "/home/nicolas" ] +# Configure HAProxy +- [ "systemctl", "enable", "firewalld" ] +- [ "systemctl", "start", "firewalld" ] +- [ "setsebool", "-P", "haproxy_connect_any=1" ] +- [ "systemctl", "enable", "haproxy" ] +- [ "systemctl", "restart", "haproxy" ] +- [ "firewall-cmd", "--add-service=http", "--permanent" ] +- [ "firewall-cmd", "--add-service=https", "--permanent" ] +- [ "firewall-cmd", "--add-port=6443/tcp", "--permanent" ] +- [ "firewall-cmd", "--add-port=22623/tcp", "--permanent" ] +- [ "firewall-cmd", "--reload" ] + +write_files: +- path: /root/.bashrc + # PS1='\[\033[01;31m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]# ' + content: | + UFMxPSdcW1wwMzNbMDE7MzFtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXSMgJwo= + encoding: base64 + append: true + +- path: /etc/skel/.bashrc + # PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' + content: | + UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXVwkICcK + encoding: base64 + append: true + +- path: /home/nicolas/.bashrc + # PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' + content: | + UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXVwkICcK + encoding: base64 + append: true + +- path: /etc/haproxy/haproxy.cfg + content: ${jsonencode(haproxy_cfg)} diff --git a/templates/lb/haproxy.cfg b/templates/lb/haproxy.cfg new file mode 100644 index 0000000..40a090b --- /dev/null +++ b/templates/lb/haproxy.cfg @@ -0,0 +1,74 @@ +global + log 127.0.0.1 local2 + + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + + + + +#--------------------------------------------------------------------- +# common defaults that all the 'listen' and 'backend' sections will +# use if not designated in their block +#--------------------------------------------------------------------- + +defaults + mode http + log global + option dontlognull + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + maxconn 3000 + +listen ingress-http + bind 0.0.0.0:80 + mode tcp +%{for name, ip in master_nodes~} + server ${name} ${ip}:80 check +%{endfor~} +%{for name, ip in worker_nodes~} + server ${name} ${ip}:80 check +%{endfor~} + +listen ingress-https + bind 0.0.0.0:443 + mode tcp +%{for name, ip in master_nodes~} + server ${name} ${ip}:443 check +%{endfor~} +%{for name, ip in worker_nodes~} + server ${name} ${ip}:443 check +%{endfor~} + +listen api + bind 0.0.0.0:6443 + mode tcp +%{for name, ip in master_nodes~} + server ${name} ${ip}:6443 check +%{endfor~} +%{for name, ip in bootstrap_nodes~} + server ${name} ${ip}:6443 check +%{endfor~} + +listen machine-config-server + bind 0.0.0.0:22623 + mode tcp +%{for name, ip in master_nodes~} + server ${name} ${ip}:22623 check +%{endfor~} +%{for name, ip in bootstrap_nodes~} + server ${name} ${ip}:22623 check +%{endfor~} diff --git a/templates/lb/network-config.cfg b/templates/lb/network-config.cfg new file mode 100644 index 0000000..e1d25c5 --- /dev/null +++ b/templates/lb/network-config.cfg @@ -0,0 +1,11 @@ +version: 2 +ethernets: + eth0: + addresses: + - ${ip}/24 + #gateway4: ${gw} + nameservers: + addresses: [${dns}] + eth1: + dhcp4: true + nameservers: {} diff --git a/templates/network.xslt b/templates/network.xslt new file mode 100644 index 0000000..3ae90ba --- /dev/null +++ b/templates/network.xslt @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/nfs-provisioner.yaml b/templates/nfs-provisioner.yaml new file mode 100644 index 0000000..69a7dcb --- /dev/null +++ b/templates/nfs-provisioner.yaml @@ -0,0 +1,115 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: "openshift-nfs-provisioner" +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: nfs-client-provisioner + namespace: "openshift-nfs-provisioner" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: nfs-client-provisioner + namespace: "openshift-nfs-provisioner" +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner + namespace: "openshift-nfs-provisioner" +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resourceNames: ["hostmount-anyuid"] + resources: ["securitycontextconstraints"] + verbs: ["use"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner + namespace: "openshift-nfs-provisioner" +subjects: + - kind: ServiceAccount + name: nfs-client-provisioner +roleRef: + kind: Role + name: nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nfs-client-provisioner + namespace: "openshift-nfs-provisioner" +spec: + replicas: 1 + selector: + matchLabels: + app: nfs-client-provisioner + strategy: + type: Recreate + template: + metadata: + labels: + app: nfs-client-provisioner + spec: + serviceAccountName: nfs-client-provisioner + containers: + - name: nfs-client-provisioner + image: quay.io/external_storage/nfs-client-provisioner:latest + volumeMounts: + - name: nfs-client-root + mountPath: /persistentvolumes + env: + - name: PROVISIONER_NAME + value: redhat-emea-ssa-team/hetzner-ocp4 + - name: NFS_SERVER + value: "${nfs_server}" + - name: NFS_PATH + value: "/srv/nfs/pv-user-pvs" + volumes: + - name: nfs-client-root + nfs: + server: "${nfs_server}" + path: "/srv/nfs/pv-user-pvs" +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: managed-nfs-storage + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: redhat-emea-ssa-team/hetzner-ocp4 +parameters: + archiveOnDelete: "false" diff --git a/templates/registry-pv.yaml b/templates/registry-pv.yaml new file mode 100644 index 0000000..74ce7d2 --- /dev/null +++ b/templates/registry-pv.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-registry-storage +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 100Gi + nfs: + path: "/srv/nfs/pv-infra-registry" + server: "${nfs_server}" + persistentVolumeReclaimPolicy: Recycle +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: registry-storage + namespace: openshift-image-registry +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi diff --git a/templates/storage/cloud-init.cfg b/templates/storage/cloud-init.cfg new file mode 100644 index 0000000..de02a70 --- /dev/null +++ b/templates/storage/cloud-init.cfg @@ -0,0 +1,104 @@ +#cloud-config +# vim: syntax=yaml + +disk_setup: + /dev/vdb: + table_type: mbr + layout: + - 100 + overwrite: false +fs_setup: +- label: storage + filesystem: xfs + device: /dev/vdb + partition: 1 +resize_rootfs: true + +mounts: +- [ "/dev/vdb1", "/srv", "xfs", "defaults", "0", "0" ] + +users: +- name: nicolas + gecos: Nicolas MASSE + groups: wheel + lock_passwd: false + passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFW62WJXI1ZCMfNA4w0dMpL0fsldhbEfULNGIUB0nQui nmasse@localhost.localdomain + +packages: +# Useful tools +- net-tools +- hdparm +- iptraf +- iotop +- vim-enhanced +- tmux +- rsync +- tree +- unzip +- tar +- tcpdump +- telnet +- strace +- bind-utils +# NFS +- firewalld +- nfs-utils + +runcmd: +# Enable KVM virsh console access +- [ "systemctl", "enable", "serial-getty@ttyS0.service" ] +- [ "systemctl", "start", "--no-block", "serial-getty@ttyS0.service" ] +# Disable SSH password authentication +- [ "sed", "-i.post-install", "-e", "s/PasswordAuthentication yes/PasswordAuthentication no/", "/etc/ssh/sshd_config" ] +- [ "systemctl", "restart", "sshd" ] +# Enable sudo without password +- [ "sed", "-i.post-install", "-e", "s/^%wheel\tALL=(ALL)\tALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers" ] +# Fix file permissions +- [ "chown", "-R", "nicolas:nicolas", "/home/nicolas" ] +# Enable NFS +- [ "mount", "/srv" ] +- [ "systemctl", "enable", "rpcbind" ] +- [ "systemctl", "start", "rpcbind" ] +- [ "systemctl", "enable", "nfs-server" ] +- [ "systemctl", "start", "nfs-server" ] +- [ "systemctl", "enable", "firewalld" ] +- [ "systemctl", "start", "firewalld" ] +- [ "setsebool", "-P", "nfs_export_all_rw", "1" ] +- [ "mkdir", "-p", "/srv/nfs" ] +- [ "exportfs", "-rav" ] +#- [ "/bin/bash", "-c", "for i in {0..999}; do pv=$(printf '/srv/nfs/pv-%03d\n' $i); mkdir $pv; chmod 777 $pv; done" ] +- [ "/bin/bash", "-c", "for pv in pv-infra-registry pv-user-pvs; do mkdir -p /srv/nfs/$pv; chmod 770 /srv/nfs/$pv; done" ] +- [ "firewall-cmd", "--add-service=nfs", "--permanent" ] +- [ "firewall-cmd", "--reload" ] + +write_files: +- path: /root/.bashrc + # PS1='\[\033[01;31m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]# ' + content: | + UFMxPSdcW1wwMzNbMDE7MzFtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXSMgJwo= + encoding: base64 + append: true + +- path: /etc/skel/.bashrc + # PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' + content: | + UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXVwkICcK + encoding: base64 + append: true + +- path: /home/nicolas/.bashrc + # PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' + content: | + UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb + XDAzM1swMG1cXVwkICcK + encoding: base64 + append: true + +- path: /etc/exports + content: | + /srv/nfs *(rw,no_root_squash) diff --git a/templates/storage/network-config.cfg b/templates/storage/network-config.cfg new file mode 100644 index 0000000..39ca322 --- /dev/null +++ b/templates/storage/network-config.cfg @@ -0,0 +1,4 @@ +version: 2 +ethernets: + eth0: + dhcp4: true \ No newline at end of file diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..8e8b42c --- /dev/null +++ b/variables.tf @@ -0,0 +1,146 @@ +variable "master_nodes" { + type = number + default = 3 +} + +variable "worker_nodes" { + type = number + default = 3 +} + +variable "bootstrap_nodes" { + type = number + default = 1 +} + +variable "pool_name" { + type = string + default = "default" +} + +variable "volume_format" { + type = string + default = "qcow2" +} + +variable "centos_image" { + type = string + default = "centos-stream-8" +} + +variable "coreos_image" { + type = string + default = "rhcos-4.7.0-x86_64-qemu.x86_64" +} + +variable "cluster_name" { + type = string + default = "ocp4" +} + +variable "external_ifname" { + type = string + default = "virbr1" +} + +variable "external_mac_address" { + type = string +} + +variable "base_domain" { + type = string + default = "ocp.lab" +} + +variable "network_ip_range" { + type = string + default = "10.10.3.0/24" +} + +variable "public_cluster_ip" { + type = string +} + +variable "master_disk_size" { + type = number + default = 120 * 1024 * 1024 * 1024 +} + +variable "master_vcpu" { + type = number + default = 4 +} + +variable "master_memory_size" { + type = number + default = 16 * 1024 +} + +variable "lb_disk_size" { + type = number + default = 10 * 1024 * 1024 * 1024 +} + +variable "lb_vcpu" { + type = number + default = 2 +} + +variable "lb_memory_size" { + type = number + default = 4 * 1024 +} + +variable "storage_disk_size" { + type = number + default = 120 * 1024 * 1024 * 1024 +} + +variable "storage_vcpu" { + type = number + default = 2 +} + +variable "storage_memory_size" { + type = number + default = 8 * 1024 +} + +variable "worker_disk_size" { + type = number + default = 120 * 1024 * 1024 * 1024 +} + +variable "worker_vcpu" { + type = number + default = 2 +} + +variable "worker_memory_size" { + type = number + default = 8 * 1024 +} + +variable "bootstrap_disk_size" { + type = number + default = 120 * 1024 * 1024 * 1024 +} + +variable "bootstrap_vcpu" { + type = number + default = 4 +} + +variable "bootstrap_memory_size" { + type = number + default = 16 * 1024 +} + +locals { + master_format = "${var.cluster_name}-master-%02d" + worker_format = "${var.cluster_name}-worker-%02d" + bootstrap_name = "${var.cluster_name}-bootstrap" + storage_name = "${var.cluster_name}-storage" + lb_name = "${var.cluster_name}-lb" + network_domain = "${var.cluster_name}.${var.base_domain}" +} diff --git a/worker.tf b/worker.tf new file mode 100644 index 0000000..706c355 --- /dev/null +++ b/worker.tf @@ -0,0 +1,43 @@ +resource "libvirt_volume" "worker_disk" { + name = "${format(local.worker_format, count.index + 1)}.${var.volume_format}" + count = var.worker_nodes + format = var.volume_format + pool = var.pool_name + base_volume_name = "${var.coreos_image}.${var.volume_format}" + size = var.worker_disk_size +} + +resource "libvirt_ignition" "worker_ignition" { + name = "${var.cluster_name}-worker-ignition" + content = file("${path.module}/${var.cluster_name}/worker.ign") +} + +resource "libvirt_domain" "worker" { + count = var.worker_nodes + name = format(local.worker_format, count.index + 1) + vcpu = var.worker_vcpu + memory = var.worker_memory_size + coreos_ignition = libvirt_ignition.worker_ignition.id + autostart = true + + disk { + volume_id = element(libvirt_volume.worker_disk.*.id, count.index) + } + + # Makes the tty0 available via `virsh console` + console { + type = "pty" + target_port = "0" + } + + network_interface { + network_id = libvirt_network.ocp_net.id + addresses = [cidrhost(var.network_ip_range, 21 + count.index)] + hostname = format("worker%d", count.index + 1) + + # When creating the domain resource, wait until the network interface gets + # a DHCP lease from libvirt, so that the computed IP addresses will be + # available when the domain is up and the plan applied. + wait_for_lease = true + } +}