diff --git a/OpenShift-Lab-Ansible-Playbook/ansible b/OpenShift-Lab-Ansible-Playbook/ansible index 8ce1b73..52f5120 100755 --- a/OpenShift-Lab-Ansible-Playbook/ansible +++ b/OpenShift-Lab-Ansible-Playbook/ansible @@ -38,6 +38,11 @@ case "$target" in read -s rhn_password export RHN_PASSWORD="$rhn_password" fi + if [ -z "$RHN_POOLID" ]; then + echo -n "Please enter your RHN Pool ID: " + read rhn_poolid + export RHN_POOLID="$rhn_poolid" + fi echo echo for host; do diff --git a/OpenShift-Lab-Ansible-Playbook/group_vars/lab b/OpenShift-Lab-Ansible-Playbook/group_vars/lab index 2ea7b95..fcc2605 100644 --- a/OpenShift-Lab-Ansible-Playbook/group_vars/lab +++ b/OpenShift-Lab-Ansible-Playbook/group_vars/lab @@ -1,3 +1,5 @@ --- openshift_cluster_dns: app.openshift.test dns_suffix: openshift.test + openshift_version: 3.4 + router_stats_password: redhat diff --git a/OpenShift-Lab-Ansible-Playbook/hosts-lab b/OpenShift-Lab-Ansible-Playbook/hosts-lab index 1723303..84ea83e 100644 --- a/OpenShift-Lab-Ansible-Playbook/hosts-lab +++ b/OpenShift-Lab-Ansible-Playbook/hosts-lab @@ -9,9 +9,9 @@ admin.openshift.test admin.openshift.test [nodes] -nodeinfra1.openshift.test onlyforinfra=1 -node1.openshift.test -node2.openshift.test +nodeinfra1.openshift.test onlyforinfra=1 region=infra zone=infranodes +node1.openshift.test region=primary zone=east +node2.openshift.test region=primary zone=west [masters] -master1.openshift.test +master1.openshift.test region=infra diff --git a/OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts b/OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts index d23ed95..c2d3458 100644 --- a/OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts +++ b/OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts @@ -4,6 +4,10 @@ # # -127.0.0.1 {{ inventory_hostname }} {{ inventory_hostname_short }} localhost.localdomain localhost +127.0.0.1 localhost.localdomain localhost + +# This is a requirement from the OpenShift installer +{{ ansible_default_ipv4['address'] }} {{ inventory_hostname }} {{ inventory_hostname_short }} + # End of file diff --git a/OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml index 68fd21f..8f820f4 100644 --- a/OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml +++ b/OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml @@ -1,13 +1,5 @@ --- - - name: Make sure optional repo is enabled - command: subscription-manager repos --enable rhel-7-server-optional-rpms - tags: rpm - - - name: Make sure extras repo is enabled - command: subscription-manager repos --enable rhel-7-server-extras-rpms - tags: rpm - - name: Install Docker yum: name=docker state=installed tags: rpm @@ -15,6 +7,10 @@ - name: Enable insecure registries lineinfile: state=present dest=/etc/sysconfig/docker regexp="^INSECURE_REGISTRY=" line="INSECURE_REGISTRY='--insecure-registry 172.30.0.0/16'" insertafter="^# INSECURE_REGISTRY=" +# +# TODO : On the master nodes only ? +# + - name: Check if sdb is empty command: sfdisk -d /dev/sdb register: sfdisk @@ -31,4 +27,3 @@ - name: Start Docker service: name=docker state=started enabled=yes - tags: wip diff --git a/OpenShift-Lab-Ansible-Playbook/roles/nfs/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/nfs/tasks/main.yml new file mode 100644 index 0000000..0b10c39 --- /dev/null +++ b/OpenShift-Lab-Ansible-Playbook/roles/nfs/tasks/main.yml @@ -0,0 +1,36 @@ +--- + + - name: Make sure nfs-utils is installed + yum: name=nfs-utils state=installed + + - name: set virt_use_nfs to 1 + command: setsebool -P virt_use_nfs 1 + + - name: Create a directory for NFS storage + file: path=/openshift-storage state=directory owner=nfsnobody group=nfsnobody mode=0777 + + - name: Fill /etc/exports + template: dest=/etc/exports src=exports + + - name: Start nfs-server + service: name=nfs-server state=started enabled=yes + + - name: Add an iptable rule to allow port 2049 (tcp) from other hosts + lineinfile: dest=/etc/sysconfig/iptables regexp="^-A INPUT -p tcp .*--dport 2049" line="-A INPUT -p tcp -m state --state NEW -m tcp --dport 2049 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT" + tags: iptables + + - name: Add an iptable rule to allow port 2049 (udp) from other hosts + lineinfile: dest=/etc/sysconfig/iptables line="-A INPUT -p udp --dport 2049 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT" + tags: iptables + + - name: Add an iptable rule to allow port 111 (tcp) from other hosts + lineinfile: dest=/etc/sysconfig/iptables regexp="^-A INPUT -p tcp .*--dport 111" line="-A INPUT -p tcp -m state --state NEW -m tcp --dport 111 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT" + tags: iptables + + - name: Add an iptable rule to allow port 111 (udp) from other hosts + lineinfile: dest=/etc/sysconfig/iptables line="-A INPUT -p udp --dport 111 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT" + tags: iptables + + - name: Restart iptables + service: name=iptables enabled=yes state=restarted + tags: iptables diff --git a/OpenShift-Lab-Ansible-Playbook/roles/nfs/templates/exports b/OpenShift-Lab-Ansible-Playbook/roles/nfs/templates/exports new file mode 100644 index 0000000..715e2d7 --- /dev/null +++ b/OpenShift-Lab-Ansible-Playbook/roles/nfs/templates/exports @@ -0,0 +1 @@ +/openshift-storage *(rw,all_squash) diff --git a/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml index 5147ce4..7b7a7c1 100644 --- a/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml +++ b/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml @@ -1,59 +1,7 @@ --- - # install atomic-openshift-utils - # run atomic-openshift-installer install - # check answer file in ~/.config/openshift/installer.cfg.yml + - name: Fill-in the ansible inventory file on the admin server + template: src=ansible-hosts dest=/etc/ansible/hosts -# oc label node master1.example.com region="infra" zone="na" -# oc label node infranode1.example.com region="infra" zone="infranodes" -# oc label node node1.example.com region="primary" zone="east" -# oc label node node2.example.com region="primary" zone="west" - -#oadm registry --config=/etc/origin/master/admin.kubeconfig \ -# --service-account=registry \ -# --selector='region=infra' -# --mount-host= - -# deploy openshift3/ose-haproxy-router - -#oc create|delete -f \ -# examples/image-streams/image-streams-rhel7.json \ -# -n openshift - -#oc create|delete -f \ -# examples/xpaas-streams/jboss-image-streams.json -# -n openshift - -#oc create -f \ -# examples/db-templates -n openshift - -#oc create|delete -f \ -# examples/quickstart-templates -n openshift - -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "metadata": { - "name": "pv0001" - }, - "spec": { - "capacity": { - "storage": "5Gi" - }, - "accessModes": [ "ReadWriteOnce" ], - "nfs": { - "path": "/tmp", - "server": "172.17.0.2" - }, - "persistentVolumeReclaimPolicy": "Recycle" - } -} - -# setsebool -P virt_use_nfs 1 - -# /example_fs *(rw,all_squash) (in /etc/exports) - -# chown -R nfsnobody:nfsnobody /example_fs -# chmod 777 - -# see https://github.com/openshift/openshift-ansible/tree/master/roles/kube_nfs_volumes + - name: Run the OpenShift installation playbook + fail: msg="run 'ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml' on the admin node" diff --git a/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/templates/ansible-hosts b/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/templates/ansible-hosts new file mode 100644 index 0000000..5984d18 --- /dev/null +++ b/OpenShift-Lab-Ansible-Playbook/roles/openshift-install/templates/ansible-hosts @@ -0,0 +1,55 @@ +[OSEv3:children] +masters +nodes +nfs + +[OSEv3:vars] +ansible_ssh_user=redhat +ansible_become=yes # Use SUDO +deployment_type=openshift-enterprise +openshift_release={{ openshift_version }} + +openshift_master_cluster_method=native +openshift_master_cluster_hostname={{ hostvars[groups['masters'][0]]['inventory_hostname'] }} +openshift_master_cluster_public_hostname={{ hostvars[groups['masters'][0]]['inventory_hostname'] }} + +os_sdn_network_plugin_name='redhat/openshift-ovs-multitenant' + +openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}] +#openshift_master_htpasswd_users={'andrew': '$apr1$cHkRDw5u$eU/ENgeCdo/ADmHF7SZhP/', 'marina': '$apr1$cHkRDw5u$eU/ENgeCdo/ADmHF7SZhP/' + +# default project node selector +osm_default_node_selector='region=primary' +openshift_hosted_router_selector='region=infra' +openshift_hosted_router_replicas=1 +#openshift_hosted_router_certificate={"certfile": "/path/to/router.crt", "keyfile": "/path/to/router.key", "cafile": "/path/to/router-ca.crt"} +openshift_hosted_registry_selector='region=infra' +openshift_hosted_registry_replicas=1 + +openshift_master_default_subdomain={{ openshift_cluster_dns }} + +#openshift_use_dnsmasq=False +#openshift_node_dnsmasq_additional_config_file=/home/bob/ose-dnsmasq.conf + +openshift_hosted_registry_storage_kind=nfs +openshift_hosted_registry_storage_access_modes=['ReadWriteMany'] +openshift_hosted_registry_storage_host={{ hostvars[groups['admin'][0]]['inventory_hostname'] }} +openshift_hosted_registry_storage_nfs_directory=/openshift-storage +openshift_hosted_registry_storage_volume_name=registry +openshift_hosted_registry_storage_volume_size=5Gi + +[nfs] +{{ hostvars[groups['admin'][0]]['inventory_hostname'] }} + +[masters] +{% for item in groups['masters'] %} +{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }} +{% endfor %} + +[nodes] +{% for item in groups['masters'] %} +{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_node_labels="{'region': '{{ hostvars[item]['region'] }}'}" +{% endfor %} +{% for item in groups['nodes'] %} +{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_node_labels="{'region': '{{ hostvars[item]['region'] }}', 'zone': '{{ hostvars[item]['zone'] }}'}" +{% endfor %} diff --git a/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/tasks/main.yml new file mode 100644 index 0000000..623e583 --- /dev/null +++ b/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/tasks/main.yml @@ -0,0 +1,36 @@ +--- + + - name: Make sure infra pods runs in the infra region + become: no + command: oc annotate namespace default openshift.io/node-selector='region=infra' --overwrite + + - name: Make sure the httpd-tools package is installed (we need htpasswd) + yum: name=httpd-tools state=installed + + - name: Create a few test users + command: htpasswd -b /etc/origin/master/htpasswd {{ item.login }} {{ item.password }} + with_items: + - { login: andrew, password: andrew } + - { login: marina, password: marina } + + - name: Recreate the default router certificate + command: oadm ca create-server-cert --signer-cert=/etc/origin/master/ca.crt --signer-key=/etc/origin/master/ca.key --signer-serial=/etc/origin/master/ca.serial.txt --hostnames='*.{{ openshift_cluster_dns }}' --cert=/home/{{ ansible_ssh_user }}/cloudapps.crt --key=/home/{{ ansible_ssh_user }}/cloudapps.key + + - name: Build the certificate + key bundle + shell: cat /home/{{ ansible_ssh_user }}/cloudapps.crt /home/{{ ansible_ssh_user }}/cloudapps.key /etc/origin/master/ca.crt > /home/{{ ansible_ssh_user }}/cloudapps.pem + + - name: Destroy the existing router + command: oc delete dc/router svc/router + become: no + + - name: Deploy the new router + command: oadm router customrouter --replicas=1 --default-cert=/home/{{ ansible_ssh_user }}/cloudapps.pem --service-account=router --stats-password='{{ router_stats_password }}' + become: no + + - name: Ship the PV creation script + template: src=create-pv.sh dest=/home/{{ ansible_ssh_user }}/create-pv.sh mode=0755 + become: no + + - name: Run the PV creation script + command: /home/{{ ansible_ssh_user }}/create-pv.sh + become: no diff --git a/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/templates/create-pv.sh b/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/templates/create-pv.sh new file mode 100644 index 0000000..5204af4 --- /dev/null +++ b/OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/templates/create-pv.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +mkdir -p "$HOME/pvs/" + +for size in 1Gi 5Gi 10Gi; do + for volume in pv{1..25} ; do + cat << EOF > $HOME/pvs/pv-${size}-${volume}.json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": { + "name": "${volume}" + }, + "spec": { + "capacity": { + "storage": "${size}" + }, + "accessModes": [ "ReadWriteOnce" ], + "nfs": { + "path": "/openshift-storage/pv-${size}-${volume}", + "server": "{{ hostvars[groups['admin'][0]]['ansible_default_ipv4']['address'] }}" + }, + "persistentVolumeReclaimPolicy": "Recycle" + } +} +EOF + done +done diff --git a/OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml index edf2c93..9863de9 100644 --- a/OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml +++ b/OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml @@ -7,20 +7,45 @@ - "ansible_os_family == 'RedHat'" - "ansible_distribution_version == '7.3'" + - name: First, disable any repos (using subscription-manager) + command: subscription-manager repos --disable="*" + tags: rpm + + - name: Make sure mandatory repos are enabled + command: subscription-manager repos --enable {{ item }} + with_items: + - rhel-7-server-rpms + - rhel-7-server-optional-rpms + - rhel-7-server-extras-rpms + - rhel-7-server-ose-{{ openshift_version }}-rpms + tags: rpm + - name: Install wget yum: name=wget state=installed + when: "'admin' in group_names" # Only on admin server tags: rpm - name: Install bridge-utils yum: name=bridge-utils state=installed + when: "'admin' in group_names" # Only on admin server + tags: rpm + + - name: Install nfs-utils + yum: name=nfs-utils state=installed tags: rpm - name: Install bash-completion yum: name=bash-completion state=installed + when: "'admin' in group_names or 'masters' in group_names" # Only on admin or master server + tags: rpm + + - name: Install NetworkManager + yum: name=NetworkManager state=installed tags: rpm - name: Install GIT yum: name=git state=installed + when: "'admin' in group_names" # Only on admin server tags: rpm - name: Install net-tools @@ -58,10 +83,21 @@ when: "'admin' in group_names" # Only on admin server tags: ssh-key - - name: Add SSH Public key of the admin server to the authorized_keys of each other server + - name: Add SSH Public key of the admin server to the authorized_keys of each server (including the admin server) authorized_key: key: "{{ lookup('file', basedir + '/admin.pub' ) }}" user: "{{ ansible_ssh_user }}" state: present - when: "'admin' not in group_names" # Only on other servers tags: ssh-key + + - name: pre-authorize all ssh keys of the other machines + command: ssh -o StrictHostKeyChecking=no {{ item }} /bin/true + become: no # need to run the ssh command as user "redhat" + with_items: "{{ groups['lab'] }}" + when: "'admin' in group_names" # Only on admin server + tags: ssh-key + + - name: Install atomic-openshift-utils (only on the admin node) + yum: name=atomic-openshift-utils state=installed + when: "'admin' in group_names" # Only on admin server + tags: rpm diff --git a/OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml b/OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml index df042b8..a430b16 100644 --- a/OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml +++ b/OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml @@ -13,6 +13,11 @@ autosubscribe: false tags: rhn +# +# To know which Pool ID you can use, run the following command on a registered host : +# +# sudo subscription-manager list --available --matches '*OpenShift*' +# - name: Attach the correct pool id to the new subscription - command: subscription-manager attach --pool=8a85f98159c85ca00159c9ad5a823661 # Red Hat Enterprise Linux Developer Suite + command: subscription-manager attach --pool={{ lookup('env','RHN_POOLID') }} tags: rhn diff --git a/OpenShift-Lab-Ansible-Playbook/site.yml b/OpenShift-Lab-Ansible-Playbook/site.yml index 15e470b..fdba5a7 100644 --- a/OpenShift-Lab-Ansible-Playbook/site.yml +++ b/OpenShift-Lab-Ansible-Playbook/site.yml @@ -13,20 +13,10 @@ vars: - basedir: "{{ lookup('env', 'BASEDIR') }}" roles: -# - base -# - openshift-prereq - - name-resolution - - - name: Install Docker - hosts: - - nodes - - masters - become: yes - roles: -# - docker - - - name: Install the admin node - hosts: admin - become: yes - roles: - # - nfs + - { role: base } + - { role: openshift-prereq } + - { role: name-resolution } + - { role: docker, when: "'admin' not in group_names" } + - { role: nfs, when: "'admin' in group_names" } + - { role: openshift-install, when: "'admin' in group_names" } + - { role: openshift-postinstall, when: "'masters' in group_names" }