diff --git a/.gitignore b/.gitignore index ce1fbb6..817da35 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ install-config.yaml .lego local.env .clusters +ansible/ansible-venv +kubespray.yaml + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e15ccd0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "kubespray"] + path = ansible/kubespray + url = https://github.com/kubernetes-sigs/kubespray + branch = release-2.18 diff --git a/ansible/kubespray b/ansible/kubespray new file mode 160000 index 0000000..2cc5f04 --- /dev/null +++ b/ansible/kubespray @@ -0,0 +1 @@ +Subproject commit 2cc5f04bada0938caedab680872aefcb69c9926c diff --git a/ansible/start.yaml b/ansible/start.yaml index e596510..7acfd3e 100644 --- a/ansible/start.yaml +++ b/ansible/start.yaml @@ -1,5 +1,5 @@ - name: Start the Kubernetes cluster - hosts: bastion + hosts: jumphost gather_facts: no become: no vars: diff --git a/ansible/stop.yaml b/ansible/stop.yaml index 1147907..2e662e5 100644 --- a/ansible/stop.yaml +++ b/ansible/stop.yaml @@ -1,5 +1,5 @@ - name: Stop the Kubernetes cluster - hosts: bastion + hosts: jumphost gather_facts: no become: no vars: diff --git a/clusterctl b/clusterctl index fc04864..a578cd1 100755 --- a/clusterctl +++ b/clusterctl @@ -26,11 +26,14 @@ function init () { mkdir -p ".clusters/$cluster_name" sed "s/__CLUSTER_NAME__/$cluster_name/" terraform.tfvars > ".clusters/$cluster_name/terraform.tfvars" + cp kubespray.yaml ".clusters/$cluster_name/kubespray.yaml" + ln -sf ../../ansible/kubespray/inventory/sample/group_vars/ ".clusters/$cluster_name/group_vars" echo "Cluster $cluster_name initialized successfully!" echo echo "Review and adjust the following files to your needs:" echo "- .clusters/$cluster_name/terraform.tfvars" + echo "- .clusters/$cluster_name/kubespray.yaml" echo exit 0 } @@ -54,21 +57,20 @@ function start () { assert_cluster_name "$@" local cluster_name="${1:-}" - ansible-playbook -i ".clusters/$cluster_name/inventory" ansible/start.yaml + ansible-playbook -i ".clusters/$cluster_name/inventory.yaml" ansible/start.yaml } function stop () { assert_cluster_name "$@" local cluster_name="${1:-}" - ansible-playbook -i ".clusters/$cluster_name/inventory" ansible/stop.yaml + ansible-playbook -i ".clusters/$cluster_name/inventory.yaml" ansible/stop.yaml } function post_install_nfs () { local cluster_name="${1:-}" - kubectl apply --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" -f ".clusters/$cluster_name/registry-pv.yaml" - kubectl apply --insecure-skip-tls-verify --kubeconfig=".clusters/$cluster_name/auth/kubeconfig" -f ".clusters/$cluster_name/nfs-provisioner.yaml" + kubectl apply --insecure-skip-tls-verify -f ".clusters/$cluster_name/nfs-provisioner.yaml" } function post_install () { @@ -79,12 +81,49 @@ function post_install () { if [ $# -eq 0 ]; then set nfs fi - + + export KUBECONFIG="$PWD/.clusters/$cluster_name/kube.config" for i; do post_install_$i "$cluster_name" done } +function ensure_kubespray_venv_ready () { + if [ ! -d "ansible/ansible-venv" ]; then + python -m venv ansible/ansible-venv + ansible-venv/bin/pip install -r ansible/kubespray/requirements.txt + ansible-venv/bin/pip install selinux + fi +} + +function kubespray_install () { + local cluster_name="${1:-}" + + ensure_kubespray_venv_ready + ansible/ansible-venv/bin/ansible-playbook -i ."clusters/$cluster_name/inventory.yaml" ansible/kubespray/cluster.yml -e "@.clusters/$cluster_name/kubespray.yaml" + ln -sf artifacts/admin.conf ".clusters/$cluster_name/kube.config" +} + +function kubespray_remove () { + local cluster_name="${1:-}" + + ensure_kubespray_venv_ready + ansible/ansible-venv/bin/ansible-playbook -i ."clusters/$cluster_name/inventory.yaml" ansible/kubespray/reset.yml -e "@.clusters/$cluster_name/kubespray.yaml" + rm -f ".clusters/$cluster_name/kube.config" +} + +function kubespray () { + assert_cluster_name "$@" + local cluster_name="${1:-}" + shift + + if [ $# -eq 0 ]; then + set install + fi + + kubespray_$1 "$cluster_name" +} + function install_addon () { assert_cluster_name "$@" local cluster_name="${1:-}" @@ -98,7 +137,7 @@ function shell () { local cluster_name="${1:-}" # Ansible - export DEFAULT_HOST_LIST="$PWD/.clusters/$cluster_name" + export DEFAULT_HOST_LIST="$PWD/.clusters/$cluster_name/inventory.yaml" # Terraform export TF_CLI_ARGS_plan="-var-file=.clusters/$cluster_name/terraform.tfvars -state=.clusters/$cluster_name/terraform.tfstate" @@ -108,7 +147,7 @@ function shell () { export TF_CLI_ARGS_state_rm="-state=.clusters/$cluster_name/terraform.tfstate" # Kubernetes - export KUBECONFIG="$PWD/.clusters/$cluster_name/auth/kubeconfig" + export KUBECONFIG="$PWD/.clusters/$cluster_name/kube.config" export KUBECTL_BINARY="$(which kubectl)" export CLUSTER_NAME="$cluster_name" export PS1="[$CLUSTER_NAME:\w] " @@ -161,6 +200,14 @@ apply) shift apply "$@" ;; +kubespray) + if [ -z "${2:-}" ]; then + echo "Usage: $0 kubespray cluster-name {install|remove}" + exit 1 + fi + shift + kubespray "$@" +;; destroy) if [ -z "${2:-}" ]; then echo "Usage: $0 destroy cluster-name" diff --git a/kubespray.yaml.sample b/kubespray.yaml.sample new file mode 100644 index 0000000..4ad1aae --- /dev/null +++ b/kubespray.yaml.sample @@ -0,0 +1,6 @@ +container_manager: crio +download_container: 'false' +etcd_kubeadm_enabled: 'true' +loadbalancer_apiserver_type: nginx +credentials_dir: "{{ inventory_dir }}" +kubeconfig_localhost: true diff --git a/post-install.tf b/post-install.tf index eefcec1..7ff3542 100644 --- a/post-install.tf +++ b/post-install.tf @@ -1,9 +1,3 @@ -resource "local_file" "registry_pv" { - content = templatefile("${path.module}/templates/registry-pv.yaml", { nfs_server = local.storage_node.ip }) - filename = ".clusters/${var.cluster_name}/registry-pv.yaml" - file_permission = "0644" -} - resource "local_file" "nfs_provisioner" { content = templatefile("${path.module}/templates/nfs-provisioner.yaml", { nfs_server = local.storage_node.ip }) filename = ".clusters/${var.cluster_name}/nfs-provisioner.yaml" @@ -11,7 +5,7 @@ resource "local_file" "nfs_provisioner" { } resource "local_file" "ansible_inventory" { - content = templatefile("${path.module}/templates/inventory", { nodes = local.all_nodes }) - filename = ".clusters/${var.cluster_name}/inventory" + content = templatefile("${path.module}/templates/inventory.yaml", { nodes = local.all_nodes, lb_ip = local.lb_node.ip }) + filename = ".clusters/${var.cluster_name}/inventory.yaml" file_permission = "0644" } diff --git a/templates/base/cloud-init.cfg b/templates/base/cloud-init.cfg index 859eb2f..ece39e3 100644 --- a/templates/base/cloud-init.cfg +++ b/templates/base/cloud-init.cfg @@ -8,6 +8,7 @@ users: gecos: Nicolas MASSE groups: sudo lock_passwd: false + shell: /bin/bash passwd: $6$XUTB20jVVXIqh78k$L1A9Lft5JlbOtNbeDP.fOZ5giLl09LfJGGCon5uwtsIhPJoNkj4SIk08Rb6vSowOps2ik5tlUwT2ZOZ6jjr7.0 ssh_authorized_keys: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPR1tt58X0+vbvsCR12gMAqr+g7vjt1Fx/qqz9EiboIs nicolas@localhost.localdomain @@ -19,3 +20,7 @@ runcmd: - [ "systemctl", "restart", "sshd" ] # Enable sudo without password - [ "sed", "-i.post-install", "-e", "s/^%sudo\tALL=(ALL:ALL) ALL/%sudo ALL=(ALL:ALL) NOPASSWD: ALL/", "/etc/sudoers" ] + +packages: +# Needed to mount NFS Persistent Volumes +- nfs-client diff --git a/templates/inventory b/templates/inventory deleted file mode 100644 index f2c9f30..0000000 --- a/templates/inventory +++ /dev/null @@ -1,5 +0,0 @@ -[all:vars] -nodes=${jsonencode(nodes)} - -[bastion] -admin.itix.lab ansible_user=nicolas diff --git a/templates/inventory.yaml b/templates/inventory.yaml new file mode 100644 index 0000000..c50330a --- /dev/null +++ b/templates/inventory.yaml @@ -0,0 +1,47 @@ +all: + vars: + ansible_user: nicolas + children: + jumphost: + hosts: + admin.itix.lab: + ansible_user: nicolas + nodes: ${jsonencode(nodes)} + etcd: + vars: + ansible_become: yes + ansible_ssh_common_args: -o StrictHostKeyChecking=no + hosts: +%{for node in nodes} +%{if node.role == "master"} + ${jsonencode(node.name)}: + ansible_host: ${jsonencode(node.ip)} + etcd_member_name: ${jsonencode(node.name)} +%{endif} +%{endfor} + k8s_cluster: + vars: + ansible_become: yes + ansible_ssh_common_args: -o StrictHostKeyChecking=no + loadbalancer_apiserver: + address: ${lb_ip} + port: 6443 + children: + calico_rr: {} + kube_control_plane: + hosts: +%{for node in nodes} +%{if node.role == "master"} + ${jsonencode(node.name)}: {} +%{endif} +%{endfor} + kube_node: + hosts: +%{for node in nodes} +%{if node.role == "worker"} + ${jsonencode(node.name)}: + ansible_host: ${jsonencode(node.ip)} +%{endif} +%{endfor} + ungrouped: {} + diff --git a/templates/nfs-provisioner.yaml b/templates/nfs-provisioner.yaml index a1890a1..f993c7a 100644 --- a/templates/nfs-provisioner.yaml +++ b/templates/nfs-provisioner.yaml @@ -1,13 +1,13 @@ kind: Namespace apiVersion: v1 metadata: - name: "openshift-nfs-provisioner" + name: "nfs-provisioner" --- kind: ServiceAccount apiVersion: v1 metadata: name: nfs-client-provisioner - namespace: "openshift-nfs-provisioner" + namespace: "nfs-provisioner" --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -34,7 +34,7 @@ metadata: subjects: - kind: ServiceAccount name: nfs-client-provisioner - namespace: "openshift-nfs-provisioner" + namespace: "nfs-provisioner" roleRef: kind: ClusterRole name: nfs-client-provisioner-runner @@ -44,21 +44,17 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nfs-client-provisioner - namespace: "openshift-nfs-provisioner" + namespace: "nfs-provisioner" rules: - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: ["security.openshift.io"] - resourceNames: ["hostmount-anyuid"] - resources: ["securitycontextconstraints"] - verbs: ["use"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nfs-client-provisioner - namespace: "openshift-nfs-provisioner" + namespace: "nfs-provisioner" subjects: - kind: ServiceAccount name: nfs-client-provisioner @@ -71,7 +67,7 @@ kind: Deployment apiVersion: apps/v1 metadata: name: nfs-client-provisioner - namespace: "openshift-nfs-provisioner" + namespace: "nfs-provisioner" spec: replicas: 1 selector: @@ -93,7 +89,7 @@ spec: mountPath: /persistentvolumes env: - name: PROVISIONER_NAME - value: redhat-emea-ssa-team/hetzner-ocp4 + value: nfs-subdir-external-provisioner - name: NFS_SERVER value: "${nfs_server}" - name: NFS_PATH @@ -110,6 +106,6 @@ metadata: name: managed-nfs-storage annotations: storageclass.kubernetes.io/is-default-class: "true" -provisioner: redhat-emea-ssa-team/hetzner-ocp4 +provisioner: nfs-subdir-external-provisioner parameters: archiveOnDelete: "false" diff --git a/templates/registry-pv.yaml b/templates/registry-pv.yaml deleted file mode 100644 index 74ce7d2..0000000 --- a/templates/registry-pv.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nfs-registry-storage -spec: - accessModes: - - ReadWriteMany - capacity: - storage: 100Gi - nfs: - path: "/srv/nfs/pv-infra-registry" - server: "${nfs_server}" - persistentVolumeReclaimPolicy: Recycle ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: registry-storage - namespace: openshift-image-registry -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi