commit
fda1a57708
21 changed files with 935 additions and 0 deletions
@ -0,0 +1,7 @@ |
|||
.terraform |
|||
*.tfstate |
|||
*.tfstate.backup |
|||
.vscode |
|||
*.lock.hcl |
|||
terraform.tfvars |
|||
|
|||
@ -0,0 +1,4 @@ |
|||
[submodule "kubespray"] |
|||
path = kubespray |
|||
url = https://github.com/kubernetes-sigs/kubespray.git |
|||
branch = release-2.15 |
|||
@ -0,0 +1,132 @@ |
|||
# Kubernetes installation |
|||
|
|||
## Pre-requisites |
|||
|
|||
### On your local machine |
|||
|
|||
Install Terraform. |
|||
|
|||
```sh |
|||
cat > hashicorp.repo <<"EOF" |
|||
[hashicorp] |
|||
name=Hashicorp Stable - $basearch |
|||
baseurl=https://rpm.releases.hashicorp.com/RHEL/8/$basearch/stable |
|||
enabled=1 |
|||
gpgcheck=1 |
|||
gpgkey=https://rpm.releases.hashicorp.com/gpg |
|||
EOF |
|||
sudo dnf config-manager --add-repo hashicorp.repo |
|||
sudo dnf -y install terraform |
|||
``` |
|||
|
|||
Install the libvirt terraform provider. |
|||
|
|||
```sh |
|||
curl -Lo /tmp/libvirt-provider.tgz https://github.com/dmacvicar/terraform-provider-libvirt/releases/download/v0.6.3/terraform-provider-libvirt-0.6.3+git.1604843676.67f4f2aa.Fedora_32.x86_64.tar.gz |
|||
mkdir -p ~/.terraform.d/plugins/registry.terraform.io/dmacvicar/libvirt/0.6.3/linux_amd64 |
|||
tar xvf /tmp/libvirt-provider.tgz -C ~/.terraform.d/plugins/registry.terraform.io/dmacvicar/libvirt/0.6.3/linux_amd64 |
|||
``` |
|||
|
|||
Initialize Terraform. |
|||
|
|||
```sh |
|||
cd terraform |
|||
terraform init |
|||
``` |
|||
|
|||
Install kubespray dependencies. |
|||
|
|||
```sh |
|||
sudo dnf install ansible python3-netaddr python3-pbr python3-ruamel-yaml python3-jmespath |
|||
``` |
|||
|
|||
### On the hypervisor |
|||
|
|||
Install libvirt. |
|||
|
|||
```sh |
|||
sudo dnf install libvirt libvirt-daemon-kvm virt-install virt-viewer virt-top libguestfs-tools nmap-ncat |
|||
``` |
|||
|
|||
Fetch the latest CentOS Stream 8 cloud image. |
|||
|
|||
```sh |
|||
sudo curl -Lo /var/lib/libvirt/images/centos-stream-8.qcow2 http://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20201217.0.x86_64.qcow2 |
|||
``` |
|||
|
|||
## Install |
|||
|
|||
Find a name for the cluster. |
|||
|
|||
```sh |
|||
export CLUSTER_NAME=kube |
|||
``` |
|||
|
|||
Add the DNS entries to your DNS server (dnsmasq in the following example). |
|||
|
|||
```sh |
|||
# Hosts |
|||
host-record=lb.kube.itix.lab,192.168.16.4,24h |
|||
host-record=storage.kube.itix.lab,192.168.16.6,24h |
|||
host-record=master1.kube.itix.lab,192.168.16.11,24h |
|||
host-record=master2.kube.itix.lab,192.168.16.12,24h |
|||
host-record=master3.kube.itix.lab,192.168.16.13,24h |
|||
host-record=worker1.kube.itix.lab,192.168.16.21,24h |
|||
host-record=worker2.kube.itix.lab,192.168.16.22,24h |
|||
|
|||
# Services |
|||
host-record=api.kube.itix.lab,192.168.16.4,24h |
|||
cname=*.apps.kube.itix.lab,lb.kube.itix.lab |
|||
``` |
|||
|
|||
Deploy the Virtual Machines. |
|||
|
|||
```sh |
|||
export LIBVIRT_DEFAULT_URI="qemu+ssh://$LIBVIRT_USER@$LIBVIRT_SERVER/system" |
|||
cd terraform |
|||
terraform init |
|||
terraform apply -var cluster_name=$CLUSTER_NAME |
|||
``` |
|||
|
|||
Set the default cluster variables. |
|||
|
|||
```sh |
|||
cp -r inventory/sample/group_vars inventory/$CLUSTER_NAME/group_vars |
|||
``` |
|||
|
|||
Install Kubernetes. |
|||
|
|||
```sh |
|||
cd ../kubespray |
|||
ansible -i inventory/$CLUSTER_NAME/inventory.ini all -m wait_for -a "port=22" |
|||
ansible-playbook -i inventory/$CLUSTER_NAME/inventory.ini cluster.yml |
|||
sudo chown -R $USER inventory/$CLUSTER_NAME/artifacts/ |
|||
``` |
|||
|
|||
Ensure the cluster is up and running. |
|||
|
|||
```sh |
|||
KUBECONFIG=inventory/$CLUSTER_NAME/artifacts/admin.conf kubectl get nodes |
|||
``` |
|||
|
|||
## Post-Install |
|||
|
|||
Expose the dashboard. |
|||
|
|||
```sh |
|||
KUBECONFIG=inventory/$CLUSTER_NAME/artifacts/admin.conf kubectl create ingress dashboard -n kube-system --rule "dashboard.apps.kube.itix.lab/*=kubernetes-dashboard:443,tls" --annotation=ingress.kubernetes.io/ssl-passthrough=true --annotation=nginx.ingress.kubernetes.io/backend-protocol=HTTPS --annotation=kubernetes.io/ingress.allow-http=false |
|||
``` |
|||
|
|||
Create the admin account. |
|||
|
|||
```sh |
|||
export KUBECONFIG=inventory/$CLUSTER_NAME/artifacts/admin.conf |
|||
kubectl create sa admin -n kube-system |
|||
kubectl create clusterrolebinding admin --clusterrole=cluster-admin --serviceaccount=kube-system:admin -n kube-system |
|||
``` |
|||
|
|||
Fetch the admin password. |
|||
|
|||
```sh |
|||
kubectl -n kube-system get secret $(kubectl -n kube-system get sa/admin -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}" |
|||
``` |
|||
@ -0,0 +1,66 @@ |
|||
resource "libvirt_cloudinit_disk" "lb_cloudinit" { |
|||
name = "${local.lb_name}-cloudinit.iso" |
|||
user_data = data.template_file.lb_user_data.rendered |
|||
network_config = file("${path.module}/templates/lb/network-config.cfg") |
|||
pool = var.pool_name |
|||
} |
|||
|
|||
data "template_file" "lb_user_data" { |
|||
template = file("${path.module}/templates/lb/cloud-init.cfg") |
|||
vars = { |
|||
haproxy_cfg = templatefile("${path.module}/templates/lb/haproxy.cfg", { |
|||
master_nodes = { for i in local.master_nodes : i.name => i.ip }, |
|||
worker_nodes = { for i in local.worker_nodes : i.name => i.ip } |
|||
}) |
|||
} |
|||
} |
|||
|
|||
resource "libvirt_volume" "lb_disk" { |
|||
name = "${local.lb_name}.${var.volume_format}" |
|||
format = var.volume_format |
|||
pool = var.pool_name |
|||
base_volume_name = "${var.os_image}.${var.volume_format}" |
|||
size = var.lb_disk_size |
|||
} |
|||
|
|||
locals { |
|||
lb_node = { |
|||
hostname = local.lb_hostname |
|||
name = local.lb_name |
|||
ip = cidrhost(var.network_ip_range, 4) |
|||
mac = format(var.network_mac_format, 4) |
|||
role = "lb" |
|||
} |
|||
} |
|||
|
|||
resource "libvirt_domain" "lb" { |
|||
name = local.lb_name |
|||
vcpu = var.lb_vcpu |
|||
memory = var.lb_memory_size |
|||
cloudinit = libvirt_cloudinit_disk.lb_cloudinit.id |
|||
autostart = false |
|||
|
|||
cpu = { |
|||
mode = "host-passthrough" |
|||
} |
|||
|
|||
disk { |
|||
volume_id = libvirt_volume.lb_disk.id |
|||
} |
|||
|
|||
# Makes the tty0 available via `virsh console` |
|||
console { |
|||
type = "pty" |
|||
target_port = "0" |
|||
} |
|||
|
|||
network_interface { |
|||
network_name = var.network_name |
|||
mac = local.lb_node.mac |
|||
wait_for_lease = false |
|||
} |
|||
|
|||
xml { |
|||
xslt = file("${path.module}/network.xslt") |
|||
} |
|||
} |
|||
@ -0,0 +1,26 @@ |
|||
terraform { |
|||
required_version = ">= 0.13" |
|||
required_providers { |
|||
libvirt = { |
|||
source = "dmacvicar/libvirt" |
|||
version = ">=0.6.3" |
|||
} |
|||
local = { |
|||
source = "hashicorp/local" |
|||
version = ">=2.0.0" |
|||
} |
|||
template = { |
|||
source = "hashicorp/template" |
|||
version = ">=2.2.0" |
|||
} |
|||
} |
|||
} |
|||
|
|||
locals { |
|||
additional_nodes = [local.lb_node, local.storage_node] |
|||
all_nodes = concat(local.additional_nodes, local.master_nodes, local.worker_nodes) |
|||
} |
|||
|
|||
output "machines" { |
|||
value = local.all_nodes |
|||
} |
|||
@ -0,0 +1,58 @@ |
|||
resource "libvirt_cloudinit_disk" "master_cloudinit" { |
|||
name = "${var.cluster_name}-master-cloudinit.iso" |
|||
user_data = file("${path.module}/templates/main/cloud-init.cfg") |
|||
network_config = file("${path.module}/templates/main/network-config.cfg") |
|||
pool = var.pool_name |
|||
} |
|||
|
|||
resource "libvirt_volume" "master_disk" { |
|||
name = "${format(local.master_format, count.index + 1)}.${var.volume_format}" |
|||
count = var.master_nodes |
|||
format = var.volume_format |
|||
pool = var.pool_name |
|||
base_volume_name = "${var.os_image}.${var.volume_format}" |
|||
size = var.master_disk_size |
|||
} |
|||
|
|||
locals { |
|||
master_nodes = [for i in range(var.master_nodes) : { |
|||
hostname = format(local.master_hostname_format, i + 1) |
|||
name = format(local.master_format, i + 1) |
|||
ip = cidrhost(var.network_ip_range, 11 + i) |
|||
mac = format(var.network_mac_format, 11 + i) |
|||
role = "master" |
|||
}] |
|||
} |
|||
|
|||
resource "libvirt_domain" "master" { |
|||
count = var.master_nodes |
|||
name = format(local.master_format, count.index + 1) |
|||
vcpu = var.master_vcpu |
|||
memory = var.master_memory_size |
|||
cloudinit = libvirt_cloudinit_disk.master_cloudinit.id |
|||
autostart = false |
|||
|
|||
cpu = { |
|||
mode = "host-passthrough" |
|||
} |
|||
|
|||
disk { |
|||
volume_id = element(libvirt_volume.master_disk.*.id, count.index) |
|||
} |
|||
|
|||
# Makes the tty0 available via `virsh console` |
|||
console { |
|||
type = "pty" |
|||
target_port = "0" |
|||
} |
|||
|
|||
network_interface { |
|||
network_name = var.network_name |
|||
mac = element(local.master_nodes.*.mac, count.index) |
|||
wait_for_lease = false |
|||
} |
|||
|
|||
xml { |
|||
xslt = file("${path.module}/network.xslt") |
|||
} |
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
<?xml version="1.0" ?> |
|||
<xsl:stylesheet version="1.0" |
|||
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> |
|||
<xsl:output omit-xml-declaration="yes" indent="yes"/> |
|||
|
|||
<!-- Target portgroup --> |
|||
<xsl:param name="portgroup" select="'lab16'"/> |
|||
|
|||
<!-- XSLT Identity template --> |
|||
<xsl:template match="node()|@*"> |
|||
<xsl:copy> |
|||
<xsl:apply-templates select="node()|@*"/> |
|||
</xsl:copy> |
|||
</xsl:template> |
|||
|
|||
<!-- Put the NIC in the desired portgroup --> |
|||
<xsl:template match="/domain/devices/interface/source"> |
|||
<xsl:copy> |
|||
<xsl:apply-templates select="@*|node()"/> |
|||
<xsl:attribute name="portgroup"> |
|||
<xsl:value-of select="$portgroup"/> |
|||
</xsl:attribute> |
|||
</xsl:copy> |
|||
</xsl:template> |
|||
</xsl:stylesheet> |
|||
@ -0,0 +1,5 @@ |
|||
resource "local_file" "ansible_inventory" { |
|||
content = templatefile("${path.module}/templates/inventory", { masters = local.master_nodes, workers = local.worker_nodes, lb_node = local.lb_node, api_endpoint = "api.${local.network_domain}" }) |
|||
filename = "../kubespray/inventory/${var.cluster_name}/inventory.ini" |
|||
file_permission = "0644" |
|||
} |
|||
@ -0,0 +1,2 @@ |
|||
provider "libvirt" { |
|||
} |
|||
@ -0,0 +1,66 @@ |
|||
resource "libvirt_cloudinit_disk" "storage_cloudinit" { |
|||
name = "${local.storage_name}-cloudinit.iso" |
|||
user_data = file("${path.module}/templates/storage/cloud-init.cfg") |
|||
network_config = file("${path.module}/templates/storage/network-config.cfg") |
|||
pool = var.pool_name |
|||
} |
|||
|
|||
resource "libvirt_volume" "storage_os_disk" { |
|||
name = "${local.storage_name}-os.${var.volume_format}" |
|||
format = var.volume_format |
|||
pool = var.pool_name |
|||
base_volume_name = "${var.os_image}.${var.volume_format}" |
|||
} |
|||
|
|||
resource "libvirt_volume" "storage_data_disk" { |
|||
name = "${local.storage_name}-data.${var.volume_format}" |
|||
format = var.volume_format |
|||
pool = var.pool_name |
|||
size = var.storage_disk_size |
|||
} |
|||
|
|||
locals { |
|||
storage_node = { |
|||
hostname = local.storage_hostname |
|||
name = local.storage_name |
|||
ip = cidrhost(var.network_ip_range, 6) |
|||
mac = format(var.network_mac_format, 6) |
|||
role = "storage" |
|||
} |
|||
} |
|||
|
|||
resource "libvirt_domain" "storage" { |
|||
name = local.storage_name |
|||
vcpu = var.storage_vcpu |
|||
memory = var.storage_memory_size |
|||
cloudinit = libvirt_cloudinit_disk.storage_cloudinit.id |
|||
autostart = false |
|||
|
|||
cpu = { |
|||
mode = "host-passthrough" |
|||
} |
|||
|
|||
disk { |
|||
volume_id = libvirt_volume.storage_os_disk.id |
|||
} |
|||
|
|||
disk { |
|||
volume_id = libvirt_volume.storage_data_disk.id |
|||
} |
|||
|
|||
# Makes the tty0 available via `virsh console` |
|||
console { |
|||
type = "pty" |
|||
target_port = "0" |
|||
} |
|||
|
|||
network_interface { |
|||
network_name = var.network_name |
|||
mac = local.storage_node.mac |
|||
wait_for_lease = false |
|||
} |
|||
|
|||
xml { |
|||
xslt = file("${path.module}/network.xslt") |
|||
} |
|||
} |
|||
@ -0,0 +1,64 @@ |
|||
[all] |
|||
%{for host in masters~} |
|||
${host.hostname} ansible_host=${host.ip} etcd_member_name=${host.name} |
|||
%{endfor~} |
|||
%{for host in workers~} |
|||
${host.hostname} ansible_host=${host.ip} |
|||
%{endfor~} |
|||
|
|||
[all:vars] |
|||
ansible_become=yes |
|||
ansible_user=nicolas |
|||
ansible_ssh_extra_args='-o StrictHostKeyChecking=no' |
|||
|
|||
# Store the .kube/config file locally |
|||
kubeconfig_localhost=true |
|||
|
|||
# Configure the Load Balancer in front of the api-server |
|||
loadbalancer_apiserver={"address":"${lb_node.ip}","port":6443} |
|||
apiserver_loadbalancer_domain_name=${api_endpoint} |
|||
|
|||
[kube-node:vars] |
|||
# Add a label to all worker nodes |
|||
node_labels={"worker": "true"} |
|||
|
|||
[k8s-cluster:vars] |
|||
# Enable the Nginx ingress controller |
|||
ingress_nginx_enabled=true |
|||
ingress_nginx_nodeselector={"worker":"true"} |
|||
|
|||
# Enable the Kubernetes dashboard |
|||
dashboard_enabled=true |
|||
|
|||
# ## configure a bastion host if your nodes are not directly reachable |
|||
# [bastion] |
|||
# bastion ansible_host=x.x.x.x ansible_user=some_user |
|||
|
|||
# When kube-node contains etcd, you define your etcd cluster to be as well |
|||
# schedulable for Kubernetes workloads. If you want it a standalone, make |
|||
# sure those groups do not intersect. If you want the server to act both as |
|||
# master and node, the server must be defined on both groups kube-master and |
|||
# kube-node. If you want a standalone and unschedulable master, the server |
|||
# must be defined only in the kube-master and not kube-node. |
|||
|
|||
[kube-master] |
|||
%{for host in masters~} |
|||
${host.hostname} |
|||
%{endfor~} |
|||
|
|||
[etcd] |
|||
%{for host in masters~} |
|||
${host.hostname} |
|||
%{endfor~} |
|||
|
|||
[kube-node] |
|||
%{for host in workers~} |
|||
${host.hostname} |
|||
%{endfor~} |
|||
|
|||
[calico-rr] |
|||
|
|||
[k8s-cluster:children] |
|||
kube-master |
|||
kube-node |
|||
calico-rr |
|||
@ -0,0 +1,84 @@ |
|||
#cloud-config |
|||
# vim: syntax=yaml |
|||
|
|||
resize_rootfs: true |
|||
|
|||
users: |
|||
- name: nicolas |
|||
gecos: Nicolas MASSE |
|||
groups: wheel |
|||
lock_passwd: false |
|||
passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 |
|||
ssh_authorized_keys: |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFW62WJXI1ZCMfNA4w0dMpL0fsldhbEfULNGIUB0nQui nmasse@localhost.localdomain |
|||
|
|||
packages: |
|||
# Useful tools |
|||
- net-tools |
|||
- hdparm |
|||
- iptraf |
|||
- iotop |
|||
- vim-enhanced |
|||
- tmux |
|||
- rsync |
|||
- tree |
|||
- unzip |
|||
- tar |
|||
- tcpdump |
|||
- telnet |
|||
- strace |
|||
- bind-utils |
|||
# Load Balancer |
|||
- haproxy |
|||
- firewalld |
|||
|
|||
runcmd: |
|||
# Enable KVM virsh console access |
|||
- [ "systemctl", "enable", "serial-getty@ttyS0.service" ] |
|||
- [ "systemctl", "start", "--no-block", "serial-getty@ttyS0.service" ] |
|||
# Disable SSH password authentication |
|||
- [ "sed", "-i.post-install", "-e", "s/PasswordAuthentication yes/PasswordAuthentication no/", "/etc/ssh/sshd_config" ] |
|||
- [ "systemctl", "restart", "sshd" ] |
|||
# Enable sudo without password |
|||
- [ "sed", "-i.post-install", "-e", "s/^%wheel\tALL=(ALL)\tALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers" ] |
|||
# Fix file permissions |
|||
- [ "chown", "-R", "nicolas:nicolas", "/home/nicolas" ] |
|||
# Configure HAProxy |
|||
- [ "setsebool", "-P", "haproxy_connect_any=1" ] |
|||
- [ "systemctl", "enable", "haproxy" ] |
|||
- [ "systemctl", "restart", "haproxy" ] |
|||
- [ "firewall-offline-cmd", "--add-service=http" ] |
|||
- [ "firewall-offline-cmd", "--add-service=https" ] |
|||
- [ "firewall-offline-cmd", "--add-port=6443/tcp" ] |
|||
- [ "firewall-offline-cmd", "--add-port=22623/tcp" ] |
|||
- [ "systemctl", "enable", "firewalld" ] |
|||
- [ "systemctl", "start", "firewalld" ] |
|||
|
|||
write_files: |
|||
- path: /root/.bashrc |
|||
# PS1='\[\033[01;31m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]# ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzFtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXSMgJwo= |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /etc/skel/.bashrc |
|||
# PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXVwkICcK |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /home/nicolas/.bashrc |
|||
# PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXVwkICcK |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /etc/haproxy/haproxy.cfg |
|||
content: ${jsonencode(haproxy_cfg)} |
|||
@ -0,0 +1,61 @@ |
|||
global |
|||
log 127.0.0.1 local2 |
|||
|
|||
chroot /var/lib/haproxy |
|||
pidfile /var/run/haproxy.pid |
|||
maxconn 4000 |
|||
user haproxy |
|||
group haproxy |
|||
|
|||
# turn on stats unix socket |
|||
stats socket /var/lib/haproxy/stats |
|||
|
|||
|
|||
|
|||
|
|||
#--------------------------------------------------------------------- |
|||
# common defaults that all the 'listen' and 'backend' sections will |
|||
# use if not designated in their block |
|||
#--------------------------------------------------------------------- |
|||
|
|||
defaults |
|||
mode http |
|||
log global |
|||
option dontlognull |
|||
option redispatch |
|||
retries 3 |
|||
timeout http-request 10s |
|||
timeout queue 1m |
|||
timeout connect 10s |
|||
timeout client 1m |
|||
timeout server 1m |
|||
timeout http-keep-alive 10s |
|||
timeout check 10s |
|||
maxconn 3000 |
|||
|
|||
listen ingress-http |
|||
bind 0.0.0.0:80 |
|||
mode tcp |
|||
%{for name, ip in master_nodes~} |
|||
server ${name} ${ip}:80 check |
|||
%{endfor~} |
|||
%{for name, ip in worker_nodes~} |
|||
server ${name} ${ip}:80 check |
|||
%{endfor~} |
|||
|
|||
listen ingress-https |
|||
bind 0.0.0.0:443 |
|||
mode tcp |
|||
%{for name, ip in master_nodes~} |
|||
server ${name} ${ip}:443 check |
|||
%{endfor~} |
|||
%{for name, ip in worker_nodes~} |
|||
server ${name} ${ip}:443 check |
|||
%{endfor~} |
|||
|
|||
listen api |
|||
bind 0.0.0.0:6443 |
|||
mode tcp |
|||
%{for name, ip in master_nodes~} |
|||
server ${name} ${ip}:6443 check |
|||
%{endfor~} |
|||
@ -0,0 +1,4 @@ |
|||
version: 2 |
|||
ethernets: |
|||
eth0: |
|||
dhcp4: true |
|||
@ -0,0 +1,41 @@ |
|||
#cloud-config |
|||
# vim: syntax=yaml |
|||
|
|||
resize_rootfs: true |
|||
|
|||
users: |
|||
- name: nicolas |
|||
gecos: Nicolas MASSE |
|||
groups: wheel |
|||
lock_passwd: false |
|||
passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 |
|||
ssh_authorized_keys: |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFW62WJXI1ZCMfNA4w0dMpL0fsldhbEfULNGIUB0nQui nmasse@localhost.localdomain |
|||
|
|||
packages: |
|||
# Useful tools |
|||
- net-tools |
|||
- hdparm |
|||
- iptraf |
|||
- iotop |
|||
- vim-enhanced |
|||
- tmux |
|||
- rsync |
|||
- tree |
|||
- unzip |
|||
- tar |
|||
- tcpdump |
|||
- telnet |
|||
- strace |
|||
- bind-utils |
|||
|
|||
runcmd: |
|||
# Enable KVM virsh console access |
|||
- [ "systemctl", "enable", "serial-getty@ttyS0.service" ] |
|||
- [ "systemctl", "start", "--no-block", "serial-getty@ttyS0.service" ] |
|||
# Disable SSH password authentication |
|||
- [ "sed", "-i.post-install", "-e", "s/PasswordAuthentication yes/PasswordAuthentication no/", "/etc/ssh/sshd_config" ] |
|||
- [ "systemctl", "restart", "sshd" ] |
|||
# Enable sudo without password |
|||
- [ "sed", "-i.post-install", "-e", "s/^%wheel\tALL=(ALL)\tALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers" ] |
|||
@ -0,0 +1,4 @@ |
|||
version: 2 |
|||
ethernets: |
|||
eth0: |
|||
dhcp4: true |
|||
@ -0,0 +1,102 @@ |
|||
#cloud-config |
|||
# vim: syntax=yaml |
|||
|
|||
disk_setup: |
|||
/dev/vdb: |
|||
table_type: mbr |
|||
layout: |
|||
- 100 |
|||
overwrite: false |
|||
fs_setup: |
|||
- label: storage |
|||
filesystem: xfs |
|||
device: /dev/vdb |
|||
partition: 1 |
|||
resize_rootfs: true |
|||
|
|||
mounts: |
|||
- [ "/dev/vdb1", "/srv", "xfs", "defaults", "0", "0" ] |
|||
|
|||
users: |
|||
- name: nicolas |
|||
gecos: Nicolas MASSE |
|||
groups: wheel |
|||
lock_passwd: false |
|||
passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 |
|||
ssh_authorized_keys: |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain |
|||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFW62WJXI1ZCMfNA4w0dMpL0fsldhbEfULNGIUB0nQui nmasse@localhost.localdomain |
|||
|
|||
packages: |
|||
# Useful tools |
|||
- net-tools |
|||
- hdparm |
|||
- iptraf |
|||
- iotop |
|||
- vim-enhanced |
|||
- tmux |
|||
- rsync |
|||
- tree |
|||
- unzip |
|||
- tar |
|||
- tcpdump |
|||
- telnet |
|||
- strace |
|||
- bind-utils |
|||
# NFS |
|||
- firewalld |
|||
- nfs-utils |
|||
|
|||
runcmd: |
|||
# Enable KVM virsh console access |
|||
- [ "systemctl", "enable", "serial-getty@ttyS0.service" ] |
|||
- [ "systemctl", "start", "--no-block", "serial-getty@ttyS0.service" ] |
|||
# Disable SSH password authentication |
|||
- [ "sed", "-i.post-install", "-e", "s/PasswordAuthentication yes/PasswordAuthentication no/", "/etc/ssh/sshd_config" ] |
|||
- [ "systemctl", "restart", "sshd" ] |
|||
# Enable sudo without password |
|||
- [ "sed", "-i.post-install", "-e", "s/^%wheel\tALL=(ALL)\tALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers" ] |
|||
# Fix file permissions |
|||
- [ "chown", "-R", "nicolas:nicolas", "/home/nicolas" ] |
|||
# Enable NFS |
|||
- [ "mount", "/srv" ] |
|||
- [ "systemctl", "enable", "rpcbind" ] |
|||
- [ "systemctl", "start", "rpcbind" ] |
|||
- [ "systemctl", "enable", "nfs-server" ] |
|||
- [ "systemctl", "start", "nfs-server" ] |
|||
- [ "setsebool", "-P", "nfs_export_all_rw", "1" ] |
|||
- [ "mkdir", "-p", "/srv/nfs" ] |
|||
- [ "exportfs", "-rav" ] |
|||
- [ "/bin/bash", "-c", "for pv in pv-infra-registry pv-user-pvs; do mkdir -p /srv/nfs/$pv; chmod 770 /srv/nfs/$pv; done" ] |
|||
- [ "firewall-offline-cmd", "--add-service=nfs" ] |
|||
- [ "systemctl", "enable", "firewalld" ] |
|||
- [ "systemctl", "start", "firewalld" ] |
|||
|
|||
write_files: |
|||
- path: /root/.bashrc |
|||
# PS1='\[\033[01;31m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]# ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzFtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXSMgJwo= |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /etc/skel/.bashrc |
|||
# PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXVwkICcK |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /home/nicolas/.bashrc |
|||
# PS1='\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' |
|||
content: | |
|||
UFMxPSdcW1wwMzNbMDE7MzJtXF1cdUBcaFxbXDAzM1swMG1cXTpcW1wwMzNbMDE7MzRtXF1cd1xb |
|||
XDAzM1swMG1cXVwkICcK |
|||
encoding: base64 |
|||
append: true |
|||
|
|||
- path: /etc/exports |
|||
content: | |
|||
/srv/nfs *(rw,no_root_squash) |
|||
@ -0,0 +1,4 @@ |
|||
version: 2 |
|||
ethernets: |
|||
eth0: |
|||
dhcp4: true |
|||
@ -0,0 +1,121 @@ |
|||
variable "master_nodes" { |
|||
type = number |
|||
default = 3 |
|||
} |
|||
|
|||
variable "worker_nodes" { |
|||
type = number |
|||
default = 2 |
|||
} |
|||
|
|||
variable "pool_name" { |
|||
type = string |
|||
default = "default" |
|||
} |
|||
|
|||
variable "volume_format" { |
|||
type = string |
|||
default = "qcow2" |
|||
} |
|||
|
|||
variable "os_image" { |
|||
type = string |
|||
default = "centos-stream-8" |
|||
} |
|||
|
|||
variable "cluster_name" { |
|||
type = string |
|||
default = "kube" |
|||
} |
|||
|
|||
variable "base_domain" { |
|||
type = string |
|||
default = "itix.lab" |
|||
} |
|||
|
|||
variable "network_name" { |
|||
type = string |
|||
default = "lab" |
|||
} |
|||
|
|||
variable "network_ip_range" { |
|||
type = string |
|||
default = "192.168.16.0/24" |
|||
} |
|||
|
|||
variable "network_mac_format" { |
|||
type = string |
|||
default = "02:01:10:00:10:%02x" |
|||
} |
|||
|
|||
variable "master_disk_size" { |
|||
type = number |
|||
default = 120 * 1024 * 1024 * 1024 |
|||
} |
|||
|
|||
variable "master_vcpu" { |
|||
type = number |
|||
default = 4 |
|||
} |
|||
|
|||
variable "master_memory_size" { |
|||
type = number |
|||
default = 16 * 1024 |
|||
} |
|||
|
|||
variable "lb_disk_size" { |
|||
type = number |
|||
default = 10 * 1024 * 1024 * 1024 |
|||
} |
|||
|
|||
variable "lb_vcpu" { |
|||
type = number |
|||
default = 2 |
|||
} |
|||
|
|||
variable "lb_memory_size" { |
|||
type = number |
|||
default = 4 * 1024 |
|||
} |
|||
|
|||
variable "storage_disk_size" { |
|||
type = number |
|||
default = 120 * 1024 * 1024 * 1024 |
|||
} |
|||
|
|||
variable "storage_vcpu" { |
|||
type = number |
|||
default = 2 |
|||
} |
|||
|
|||
variable "storage_memory_size" { |
|||
type = number |
|||
default = 8 * 1024 |
|||
} |
|||
|
|||
variable "worker_disk_size" { |
|||
type = number |
|||
default = 120 * 1024 * 1024 * 1024 |
|||
} |
|||
|
|||
variable "worker_vcpu" { |
|||
type = number |
|||
default = 4 |
|||
} |
|||
|
|||
variable "worker_memory_size" { |
|||
type = number |
|||
default = 8 * 1024 |
|||
} |
|||
|
|||
locals { |
|||
master_format = "${var.cluster_name}-master-%02d" |
|||
master_hostname_format = "master%d.${local.network_domain}" |
|||
worker_format = "${var.cluster_name}-worker-%02d" |
|||
worker_hostname_format = "worker%d.${local.network_domain}" |
|||
storage_name = "${var.cluster_name}-storage" |
|||
storage_hostname = "storage.${local.network_domain}" |
|||
lb_name = "${var.cluster_name}-lb" |
|||
lb_hostname = "lb.${local.network_domain}" |
|||
network_domain = "${var.cluster_name}.${var.base_domain}" |
|||
} |
|||
@ -0,0 +1,58 @@ |
|||
resource "libvirt_cloudinit_disk" "worker_cloudinit" { |
|||
name = "${var.cluster_name}-worker-cloudinit.iso" |
|||
user_data = file("${path.module}/templates/main/cloud-init.cfg") |
|||
network_config = file("${path.module}/templates/main/network-config.cfg") |
|||
pool = var.pool_name |
|||
} |
|||
|
|||
resource "libvirt_volume" "worker_disk" { |
|||
name = "${format(local.worker_format, count.index + 1)}.${var.volume_format}" |
|||
count = var.worker_nodes |
|||
format = var.volume_format |
|||
pool = var.pool_name |
|||
base_volume_name = "${var.os_image}.${var.volume_format}" |
|||
size = var.worker_disk_size |
|||
} |
|||
|
|||
locals { |
|||
worker_nodes = [for i in range(var.worker_nodes) : { |
|||
hostname = format(local.worker_hostname_format, i + 1) |
|||
name = format(local.worker_format, i + 1) |
|||
ip = cidrhost(var.network_ip_range, 21 + i) |
|||
mac = format(var.network_mac_format, 21 + i) |
|||
role = "worker" |
|||
}] |
|||
} |
|||
|
|||
resource "libvirt_domain" "worker" { |
|||
count = var.worker_nodes |
|||
name = format(local.worker_format, count.index + 1) |
|||
vcpu = var.worker_vcpu |
|||
memory = var.worker_memory_size |
|||
cloudinit = libvirt_cloudinit_disk.worker_cloudinit.id |
|||
autostart = false |
|||
|
|||
cpu = { |
|||
mode = "host-passthrough" |
|||
} |
|||
|
|||
disk { |
|||
volume_id = element(libvirt_volume.worker_disk.*.id, count.index) |
|||
} |
|||
|
|||
# Makes the tty0 available via `virsh console` |
|||
console { |
|||
type = "pty" |
|||
target_port = "0" |
|||
} |
|||
|
|||
network_interface { |
|||
network_name = var.network_name |
|||
mac = element(local.worker_nodes.*.mac, count.index) |
|||
wait_for_lease = false |
|||
} |
|||
|
|||
xml { |
|||
xslt = file("${path.module}/network.xslt") |
|||
} |
|||
} |
|||
Loading…
Reference in new issue