Work-in-Progress: OpenShift installation is finished

OpenShift-Lab-Ansible-Playbook/ansible

@@ -38,6 +38,11 @@ case "$target" in
       read -s rhn_password
       export RHN_PASSWORD="$rhn_password"
     fi
+    if [ -z "$RHN_POOLID" ]; then
+      echo -n "Please enter your RHN Pool ID: "
+      read rhn_poolid
+      export RHN_POOLID="$rhn_poolid"
+    fi
     echo
     echo
     for host; do

OpenShift-Lab-Ansible-Playbook/group_vars/lab

@@ -1,3 +1,5 @@
 ---
   openshift_cluster_dns: app.openshift.test
   dns_suffix: openshift.test
+  openshift_version: 3.4
+  router_stats_password: redhat

OpenShift-Lab-Ansible-Playbook/hosts-lab

@@ -9,9 +9,9 @@ admin.openshift.test
 admin.openshift.test
 
 [nodes]
-nodeinfra1.openshift.test onlyforinfra=1
+nodeinfra1.openshift.test onlyforinfra=1 region=infra zone=infranodes
-node1.openshift.test
+node1.openshift.test region=primary zone=east
-node2.openshift.test
+node2.openshift.test region=primary zone=west
 
 [masters]
-master1.openshift.test
+master1.openshift.test region=infra

OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts

@@ -4,6 +4,10 @@
 #
 
 #<ip-address>	<hostname.domain.org>	<hostname>
-127.0.0.1	{{ inventory_hostname }} {{ inventory_hostname_short }} localhost.localdomain	localhost
+127.0.0.1	localhost.localdomain	localhost
+
+# This is a requirement from the OpenShift installer
+{{ ansible_default_ipv4['address'] }} {{ inventory_hostname }} {{ inventory_hostname_short }}
+
 
 # End of file

OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml

@@ -1,13 +1,5 @@
 ---
 
-  - name: Make sure optional repo is enabled
-    command: subscription-manager repos --enable rhel-7-server-optional-rpms
-    tags: rpm
-
-  - name: Make sure extras repo is enabled
-    command: subscription-manager repos --enable rhel-7-server-extras-rpms
-    tags: rpm
-
   - name: Install Docker
     yum: name=docker state=installed
     tags: rpm
@@ -15,6 +7,10 @@
   - name: Enable insecure registries
     lineinfile: state=present dest=/etc/sysconfig/docker regexp="^INSECURE_REGISTRY=" line="INSECURE_REGISTRY='--insecure-registry 172.30.0.0/16'" insertafter="^# INSECURE_REGISTRY="
 
+#
+# TODO : On the master nodes only ?
+#
+
   - name: Check if sdb is empty
     command: sfdisk -d /dev/sdb
     register: sfdisk
@@ -31,4 +27,3 @@
 
   - name: Start Docker
     service: name=docker state=started enabled=yes
-    tags: wip

OpenShift-Lab-Ansible-Playbook/roles/nfs/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+
+  - name: Make sure nfs-utils is installed
+    yum: name=nfs-utils state=installed
+
+  - name: set virt_use_nfs to 1
+    command: setsebool -P virt_use_nfs 1
+
+  - name: Create a directory for NFS storage
+    file: path=/openshift-storage state=directory owner=nfsnobody group=nfsnobody mode=0777
+
+  - name: Fill /etc/exports
+    template: dest=/etc/exports src=exports
+
+  - name: Start nfs-server
+    service: name=nfs-server state=started enabled=yes
+
+  - name: Add an iptable rule to allow port 2049 (tcp) from other hosts
+    lineinfile: dest=/etc/sysconfig/iptables regexp="^-A INPUT -p tcp .*--dport 2049" line="-A INPUT -p tcp -m state --state NEW -m tcp --dport 2049 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT"
+    tags: iptables
+
+  - name: Add an iptable rule to allow port 2049 (udp) from other hosts
+    lineinfile: dest=/etc/sysconfig/iptables line="-A INPUT -p udp --dport 2049 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT"
+    tags: iptables
+
+  - name: Add an iptable rule to allow port 111 (tcp) from other hosts
+    lineinfile: dest=/etc/sysconfig/iptables regexp="^-A INPUT -p tcp .*--dport 111" line="-A INPUT -p tcp -m state --state NEW -m tcp --dport 111 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT"
+    tags: iptables
+
+  - name: Add an iptable rule to allow port 111 (udp) from other hosts
+    lineinfile: dest=/etc/sysconfig/iptables line="-A INPUT -p udp --dport 111 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT"
+    tags: iptables
+
+  - name: Restart iptables
+    service: name=iptables enabled=yes state=restarted
+    tags: iptables

OpenShift-Lab-Ansible-Playbook/roles/nfs/templates/exports

@@ -0,0 +1 @@
+/openshift-storage *(rw,all_squash)

OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml

@@ -1,59 +1,7 @@
 ---
 
-  # install atomic-openshift-utils
+  - name: Fill-in the ansible inventory file on the admin server
-  # run atomic-openshift-installer install
+    template: src=ansible-hosts dest=/etc/ansible/hosts
-  # check answer file in ~/.config/openshift/installer.cfg.yml
 
-# oc label node master1.example.com region="infra" zone="na"
+  - name: Run the OpenShift installation playbook
-# oc label node infranode1.example.com region="infra" zone="infranodes"
+    fail: msg="run 'ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml' on the admin node"
-# oc label node node1.example.com region="primary" zone="east"
-# oc label node node2.example.com region="primary" zone="west"
-
-#oadm registry --config=/etc/origin/master/admin.kubeconfig \
-#    --service-account=registry \
-#    --selector='region=infra'
-# --mount-host=<path>
-
-# deploy openshift3/ose-haproxy-router
-
-#oc create|delete -f \
-#    examples/image-streams/image-streams-rhel7.json \
-#    -n openshift
-
-#oc create|delete -f \
-#    examples/xpaas-streams/jboss-image-streams.json
-#    -n openshift
-
-#oc create -f \
-#    examples/db-templates -n openshift
-
-#oc create|delete -f \
-#    examples/quickstart-templates -n openshift
-
-{
-  "apiVersion": "v1",
-  "kind": "PersistentVolume",
-  "metadata": {
-    "name": "pv0001"
-  },
-  "spec": {
-    "capacity": {
-        "storage": "5Gi"
-    },
-    "accessModes": [ "ReadWriteOnce" ],
-    "nfs": {
-        "path": "/tmp",
-        "server": "172.17.0.2"
-    },
-    "persistentVolumeReclaimPolicy": "Recycle"
-  }
-}
-
-# setsebool -P virt_use_nfs 1
-
-# /example_fs *(rw,all_squash) (in /etc/exports)
-
-# chown -R nfsnobody:nfsnobody /example_fs
-# chmod 777
-
-# see https://github.com/openshift/openshift-ansible/tree/master/roles/kube_nfs_volumes

OpenShift-Lab-Ansible-Playbook/roles/openshift-install/templates/ansible-hosts

@@ -0,0 +1,55 @@
+[OSEv3:children]
+masters
+nodes
+nfs
+
+[OSEv3:vars]
+ansible_ssh_user=redhat
+ansible_become=yes # Use SUDO
+deployment_type=openshift-enterprise
+openshift_release={{ openshift_version }}
+
+openshift_master_cluster_method=native
+openshift_master_cluster_hostname={{ hostvars[groups['masters'][0]]['inventory_hostname'] }}
+openshift_master_cluster_public_hostname={{ hostvars[groups['masters'][0]]['inventory_hostname'] }}
+
+os_sdn_network_plugin_name='redhat/openshift-ovs-multitenant'
+
+openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
+#openshift_master_htpasswd_users={'andrew': '$apr1$cHkRDw5u$eU/ENgeCdo/ADmHF7SZhP/', 'marina': '$apr1$cHkRDw5u$eU/ENgeCdo/ADmHF7SZhP/'
+
+# default project node selector
+osm_default_node_selector='region=primary'
+openshift_hosted_router_selector='region=infra'
+openshift_hosted_router_replicas=1
+#openshift_hosted_router_certificate={"certfile": "/path/to/router.crt", "keyfile": "/path/to/router.key", "cafile": "/path/to/router-ca.crt"}
+openshift_hosted_registry_selector='region=infra'
+openshift_hosted_registry_replicas=1
+
+openshift_master_default_subdomain={{ openshift_cluster_dns }}
+
+#openshift_use_dnsmasq=False
+#openshift_node_dnsmasq_additional_config_file=/home/bob/ose-dnsmasq.conf
+
+openshift_hosted_registry_storage_kind=nfs
+openshift_hosted_registry_storage_access_modes=['ReadWriteMany']
+openshift_hosted_registry_storage_host={{ hostvars[groups['admin'][0]]['inventory_hostname'] }}
+openshift_hosted_registry_storage_nfs_directory=/openshift-storage
+openshift_hosted_registry_storage_volume_name=registry
+openshift_hosted_registry_storage_volume_size=5Gi
+
+[nfs]
+{{ hostvars[groups['admin'][0]]['inventory_hostname'] }}
+
+[masters]
+{% for item in groups['masters'] %}
+{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }}
+{% endfor %}
+
+[nodes]
+{% for item in groups['masters'] %}
+{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_node_labels="{'region': '{{ hostvars[item]['region'] }}'}"
+{% endfor %}
+{% for item in groups['nodes'] %}
+{{ hostvars[item]['inventory_hostname'] }} openshift_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_public_hostname={{ hostvars[item]['inventory_hostname'] }} openshift_node_labels="{'region': '{{ hostvars[item]['region'] }}', 'zone': '{{ hostvars[item]['zone'] }}'}"
+{% endfor %}

OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+
+  - name: Make sure infra pods runs in the infra region
+    become: no
+    command: oc annotate namespace default openshift.io/node-selector='region=infra' --overwrite
+
+  - name: Make sure the httpd-tools package is installed (we need htpasswd)
+    yum: name=httpd-tools state=installed
+
+  - name: Create a few test users
+    command: htpasswd -b /etc/origin/master/htpasswd {{ item.login }} {{ item.password }}
+    with_items:
+      - { login: andrew, password: andrew }
+      - { login: marina, password: marina }
+
+  - name: Recreate the default router certificate
+    command: oadm ca create-server-cert --signer-cert=/etc/origin/master/ca.crt --signer-key=/etc/origin/master/ca.key --signer-serial=/etc/origin/master/ca.serial.txt --hostnames='*.{{ openshift_cluster_dns }}' --cert=/home/{{ ansible_ssh_user }}/cloudapps.crt --key=/home/{{ ansible_ssh_user }}/cloudapps.key
+
+  - name: Build the certificate + key bundle
+    shell: cat /home/{{ ansible_ssh_user }}/cloudapps.crt /home/{{ ansible_ssh_user }}/cloudapps.key /etc/origin/master/ca.crt > /home/{{ ansible_ssh_user }}/cloudapps.pem
+
+  - name: Destroy the existing router
+    command: oc delete dc/router svc/router
+    become: no
+
+  - name: Deploy the new router
+    command: oadm router customrouter --replicas=1 --default-cert=/home/{{ ansible_ssh_user }}/cloudapps.pem --service-account=router --stats-password='{{ router_stats_password }}'
+    become: no
+
+  - name: Ship the PV creation script
+    template: src=create-pv.sh dest=/home/{{ ansible_ssh_user }}/create-pv.sh mode=0755
+    become: no
+
+  - name: Run the PV creation script
+    command: /home/{{ ansible_ssh_user }}/create-pv.sh
+    become: no

OpenShift-Lab-Ansible-Playbook/roles/openshift-postinstall/templates/create-pv.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+mkdir -p "$HOME/pvs/"
+
+for size in 1Gi 5Gi 10Gi; do
+  for volume in pv{1..25} ; do
+    cat << EOF > $HOME/pvs/pv-${size}-${volume}.json
+{
+  "apiVersion": "v1",
+  "kind": "PersistentVolume",
+  "metadata": {
+    "name": "${volume}"
+  },
+  "spec": {
+    "capacity": {
+        "storage": "${size}"
+    },
+    "accessModes": [ "ReadWriteOnce" ],
+    "nfs": {
+        "path": "/openshift-storage/pv-${size}-${volume}",
+        "server": "{{ hostvars[groups['admin'][0]]['ansible_default_ipv4']['address'] }}"
+    },
+    "persistentVolumeReclaimPolicy": "Recycle"
+  }
+}
+EOF
+  done
+done

OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml

@@ -7,20 +7,45 @@
         - "ansible_os_family == 'RedHat'"
         - "ansible_distribution_version == '7.3'"
 
+  - name: First, disable any repos (using subscription-manager)
+    command: subscription-manager repos --disable="*"
+    tags: rpm
+
+  - name: Make sure mandatory repos are enabled
+    command: subscription-manager repos --enable {{ item }}
+    with_items:
+      - rhel-7-server-rpms
+      - rhel-7-server-optional-rpms
+      - rhel-7-server-extras-rpms
+      - rhel-7-server-ose-{{ openshift_version }}-rpms
+    tags: rpm
+
   - name: Install wget
     yum: name=wget state=installed
+    when: "'admin' in group_names" # Only on admin server
     tags: rpm
 
   - name: Install bridge-utils
     yum: name=bridge-utils state=installed
+    when: "'admin' in group_names" # Only on admin server
+    tags: rpm
+
+  - name: Install nfs-utils
+    yum: name=nfs-utils state=installed
     tags: rpm
 
   - name: Install bash-completion
     yum: name=bash-completion state=installed
+    when: "'admin' in group_names or 'masters' in group_names" # Only on admin or master server
+    tags: rpm
+
+  - name: Install NetworkManager
+    yum: name=NetworkManager state=installed
     tags: rpm
 
   - name: Install GIT
     yum: name=git state=installed
+    when: "'admin' in group_names" # Only on admin server
     tags: rpm
 
   - name: Install net-tools
@@ -58,10 +83,21 @@
     when: "'admin' in group_names" # Only on admin server
     tags: ssh-key
 
-  - name: Add SSH Public key of the admin server to the authorized_keys of each other server
+  - name: Add SSH Public key of the admin server to the authorized_keys of each server (including the admin server)
     authorized_key:
       key: "{{ lookup('file', basedir + '/admin.pub' ) }}"
       user: "{{ ansible_ssh_user }}"
       state: present
-    when: "'admin' not in group_names" # Only on other servers
     tags: ssh-key
+
+  - name: pre-authorize all ssh keys of the other machines
+    command: ssh -o StrictHostKeyChecking=no {{ item }} /bin/true
+    become: no # need to run the ssh command as user "redhat"
+    with_items: "{{ groups['lab'] }}"
+    when: "'admin' in group_names" # Only on admin server
+    tags: ssh-key
+
+  - name: Install atomic-openshift-utils (only on the admin node)
+    yum: name=atomic-openshift-utils state=installed
+    when: "'admin' in group_names" # Only on admin server
+    tags: rpm

OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml

@@ -13,6 +13,11 @@
       autosubscribe: false
     tags: rhn
 
+#
+# To know which Pool ID you can use, run the following command on a registered host :
+#
+#   sudo subscription-manager list --available --matches '*OpenShift*'
+#
   - name: Attach the correct pool id to the new subscription
-    command: subscription-manager attach --pool=8a85f98159c85ca00159c9ad5a823661 # Red Hat Enterprise Linux Developer Suite
+    command: subscription-manager attach --pool={{ lookup('env','RHN_POOLID') }}
     tags: rhn

OpenShift-Lab-Ansible-Playbook/site.yml

@@ -13,20 +13,10 @@
    vars:
     - basedir: "{{ lookup('env', 'BASEDIR') }}"
    roles:
-#    - base
+    - { role: base }
-#    - openshift-prereq
+    - { role: openshift-prereq }
-     - name-resolution
+    - { role: name-resolution }
-
+    - { role: docker, when: "'admin' not in group_names" }
- - name: Install Docker
+    - { role: nfs, when: "'admin' in group_names" }
-   hosts:
+    - { role: openshift-install, when: "'admin' in group_names" }
-     - nodes
+    - { role: openshift-postinstall, when: "'masters' in group_names" }
-     - masters
-   become: yes
-   roles:
-#    - docker
-
- - name: Install the admin node
-   hosts: admin
-   become: yes
-   roles:
-  #   - nfs