WiP : OpenShift Lab installation playbook

OpenShift-Lab-Ansible-Playbook/.gitignore

@@ -0,0 +1,2 @@
+admin.pub
+site.retry

OpenShift-Lab-Ansible-Playbook/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Nicolas MASSE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

OpenShift-Lab-Ansible-Playbook/README.md

@@ -0,0 +1,15 @@
+# OpenShift-Lab-Ansible-Playbook
+This project is the Ansible Playbook to install OpenShift in a Lab Environment.
+
+## Preparation work
+
+1. Edit group_vars/lab and change all passwords and DNS names (search for "changeme")
+2. Edit hosts-lab to target your Virtual Machines
+
+## Installation
+
+‘‘‘
+./ansible bootstrap master1.openshift.test node1.openshift.test node2.openshift.test nodeinfra1.openshift.test admin.openshift.test
+./ansible play
+./ansible run nodes "uptime -p"
+’’’

OpenShift-Lab-Ansible-Playbook/ansible

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+options=""
+ssh_key="$HOME/.ssh/id_rsa"
+user="root"
+inventory="lab"
+
+# Export our base directory so that any script launched localy can refer to it
+BASEDIR="$(dirname $0)"
+BASEDIR="$(python -c 'import os.path; import sys; print os.path.abspath(sys.argv[1])' "$BASEDIR")"
+export BASEDIR
+
+target="$1"
+shift
+case "$target" in
+  "")
+    echo "No target specified. Please specify an inventory or 'bootstrap' !"
+    exit 1
+  ;;
+
+  "bootstrap")
+    if [ -z "$1" ]; then
+      echo "Please specify the target host !"
+      exit 1
+    fi
+    echo "Bootstraping $@..."
+    echo
+    echo -n "Please enter the initial $user password: "
+    read -s password
+    echo
+    if [ -z "$RHN_LOGIN" ]; then
+      echo -n "Please enter your RHN login: "
+      read rhn_login
+      export RHN_LOGIN="$rhn_login"
+    fi
+    if [ -z "$RHN_PASSWORD" ]; then
+      echo -n "Please enter your RHN password: "
+      read -s rhn_password
+      export RHN_PASSWORD="$rhn_password"
+    fi
+    echo
+    echo
+    for host; do
+      echo "Connecting to $host to register the SSH Host Key !"
+      LC_ALL=C sshpass -p "$password" ssh -i $ssh_key -o StrictHostKeyChecking=no "$user@$host" /bin/true
+    done
+    auth=""
+    if [ -n "$password" ]; then
+      auth="ansible_ssh_pass=$password"
+    else
+      auth="ansible_ssh_private_key_file=$ssh_key"
+    fi
+    echo "[$target]" > "./hosts-$target"
+    for host; do
+      echo -e "$host ansible_ssh_user=$user $auth"
+    done >> "./hosts-$target"
+
+    ansible-playbook -i "./hosts-$target" $options site.yml
+
+    rm -f "./hosts-$target"
+  ;;
+  "play")
+    ansible-playbook -i "./hosts-$inventory" $options "$@" site.yml
+  ;;
+  "run")
+    group="$1"
+    cmd="$2"
+
+    ansible "$group" -i "./hosts-$inventory" -a "$cmd"
+  ;;
+  *)
+    echo "Usage: $0 {bootstrap|run} [options]"
+    exit 1
+  ;;
+esac

OpenShift-Lab-Ansible-Playbook/group_vars/all

@@ -0,0 +1,7 @@
+---
+  timezone: Europe/Paris
+  ansible_python_interpreter: /usr/bin/python2
+  ansible_ssh_user: redhat
+  ansible_ssh_private_key_file: "{{ lookup('env','HOME') }}/.ssh/id_rsa"
+  ansible_ssh_public_key: "{{ lookup('file', ansible_ssh_private_key_file + '.pub' ) }}"
+  ansible_connection: ssh

OpenShift-Lab-Ansible-Playbook/group_vars/lab

@@ -0,0 +1,3 @@
+---
+  openshift_cluster_dns: app.openshift.test
+  dns_suffix: openshift.test

OpenShift-Lab-Ansible-Playbook/hosts-lab

@@ -0,0 +1,17 @@
+[lab]
+master1.openshift.test
+nodeinfra1.openshift.test
+node1.openshift.test
+node2.openshift.test
+admin.openshift.test
+
+[admin]
+admin.openshift.test
+
+[nodes]
+nodeinfra1.openshift.test onlyforinfra=1
+node1.openshift.test
+node2.openshift.test
+
+[masters]
+master1.openshift.test

OpenShift-Lab-Ansible-Playbook/roles/base/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+
+  - name: restart sshd
+    service: name=sshd state=reloaded
+
+  - name: update hostname
+    command: hostname {{ inventory_hostname_short }}

OpenShift-Lab-Ansible-Playbook/roles/base/tasks/main.yml

@@ -0,0 +1,94 @@
+---
+  - name: This module has only been tested on RHEL 7.3 x64
+    assert:
+      that:
+        - "ansible_userspace_bits == '64'"
+        - "ansible_os_family == 'RedHat'"
+        - "ansible_distribution_version == '7.3'"
+
+  - name: Tell SSHD not to use DNS
+    lineinfile: dest=/etc/ssh/sshd_config regexp="^#* *UseDNS +" line="UseDNS no"
+    notify: restart sshd
+    tags: config
+
+  - name: Tell SSHD to forbid root accesses
+    lineinfile: dest=/etc/ssh/sshd_config regexp="^#* *PermitRootLogin +" line="PermitRootLogin no"
+    notify: restart sshd
+    tags: config
+
+  - name: Tell SSHD to forbid password accesses
+    lineinfile: dest=/etc/ssh/sshd_config regexp="^#* *PasswordAuthentication +" line="PasswordAuthentication no"
+    notify: restart sshd
+    tags: config
+
+  - name: Install VIM
+    yum: name=vim-enhanced state=installed
+
+  - name: Install Open-VM tools
+    yum: name=open-vm-tools state=installed
+
+  - name: Install Screen
+    yum: name=screen state=installed
+
+  - name: Install unzip
+    yum: name=unzip state=installed
+
+  - name: Install tcpdump
+    yum: name=tcpdump state=installed
+
+  - name: Install telnet
+    yum: name=telnet state=installed
+
+  - name: Install strace
+    yum: name=strace state=installed
+
+  - name: Install man-pages
+    yum: name=man-pages state=installed
+
+  - name: Install man
+    yum: name=man state=installed
+
+  - name: Install iptraf
+    yum: name=iptraf state=installed
+
+  - name: Install wget
+    yum: name=wget state=installed
+
+  - name: Fix /etc/environment to include PATH
+    lineinfile: dest=/etc/environment regexp="^PATH=" line="PATH=/bin:/usr/bin:/sbin:/usr/sbin"
+    tags: config
+
+  - name: Persist the hostname
+    lineinfile: dest=/etc/sysconfig/network regexp="^HOSTNAME=" line="HOSTNAME={{ inventory_hostname_short }}"
+    notify: update hostname
+    tags:
+      - config
+      - dns
+
+  - name: Set the hostname
+    command: hostnamectl set-hostname {{ inventory_hostname_short }} --static
+    tags:
+      - config
+      - dns
+
+  - name: Edit /etc/hosts
+    template: src=etc_hosts dest=/etc/hosts owner=root group=root mode=0644
+    tags:
+      - config
+      - dns
+
+  - name: Ensure consistent locale across systems (1/2)
+    lineinfile: dest=/etc/locale.conf regexp="^LANG=" line="LANG=en_US.utf8"
+
+  - name: Ensure consistent locale across systems (2/2)
+    lineinfile: dest=/etc/locale.conf line="LC_CTYPE=en_US.utf8"
+
+  - name: Install the OpenSSH clients
+    yum: name=openssh-clients state=installed
+
+  - name: Install the custom banner script
+    template: src=rc.local dest=/usr/local/etc/rc.local mode=0755
+    tags: config
+
+  - name: Run the custom banner script at startup
+    lineinfile: dest=/etc/rc.d/rc.local line="/usr/local/etc/rc.local" state=present insertafter=EOF

OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hostname

@@ -0,0 +1 @@
+{{ inventory_hostname_short }}

OpenShift-Lab-Ansible-Playbook/roles/base/templates/etc_hosts

@@ -0,0 +1,9 @@
+# {{ ansible_managed }}
+#
+# /etc/hosts: static lookup table for host names
+#
+
+#<ip-address>	<hostname.domain.org>	<hostname>
+127.0.0.1	{{ inventory_hostname }} {{ inventory_hostname_short }} localhost.localdomain	localhost
+
+# End of file

OpenShift-Lab-Ansible-Playbook/roles/base/templates/ifcfg-ethX

@@ -0,0 +1,15 @@
+DEVICE={{ item }}
+TYPE=Ethernet
+ONBOOT=yes
+BOOTPROTO=dhcp
+USERCTL=no
+PEERDNS=yes
+IPV6INIT=no
+DHCP_HOSTNAME={{ shortname }}
+
+## Static configuration sample.
+## Gateway to be configured in /etc/sysconfig/network.
+##
+# BOOTPROTO=static
+# IPADDR=192.168.38.179
+# NETMASK=255.255.255.0

OpenShift-Lab-Ansible-Playbook/roles/base/templates/local.repo

@@ -0,0 +1,5 @@
+[localrepo]
+name=Demo Local Repository
+baseurl=file://{{ sources_dir }}/rpms
+enabled=1
+gpgcheck=0

OpenShift-Lab-Ansible-Playbook/roles/base/templates/rc.local

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+sed -ri 's/^(eth[0-9]: .*|)$//g; T; d' /etc/issue
+echo >> /etc/issue
+ip addr show scope global |sed -r 's/^.*inet ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+ brd .* (eth[0-9])$/\2: \1/g; t; d' >> /etc/issue
+echo >> /etc/issue

OpenShift-Lab-Ansible-Playbook/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+ - name: This module has only been tested on RHEL and CentOS
+   assert:
+     that:
+       - "ansible_os_family == 'RedHat' or ansible_os_family == 'CentOS'"
+
+ - name: Create user RedHat
+   user: name=redhat group=users groups=users,wheel state=present comment="RedHat privileged user" password="*"
+   tags:
+     - bootstrap
+     - user
+
+ - name: Set SSH key for root
+   authorized_key: user=root key="{{ ansible_ssh_public_key }}" manage_dir=yes
+   tags:
+     - bootstrap
+     - user
+
+ - name: Set SSH key for user RedHat
+   authorized_key: user=redhat key="{{ ansible_ssh_public_key }}" manage_dir=yes
+   tags:
+     - bootstrap
+     - user
+
+ - name: Configure SUDO
+   template: src=sudoers dest=/etc/sudoers owner=root group=root mode=0440 validate="/usr/sbin/visudo -cf %s"
+   tags:
+     - bootstrap
+     - config

OpenShift-Lab-Ansible-Playbook/roles/bootstrap/templates/sudoers

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+%wheel ALL=(ALL) NOPASSWD: ALL
+root ALL=(ALL) NOPASSWD: ALL

OpenShift-Lab-Ansible-Playbook/roles/docker/tasks/main.yml

@@ -0,0 +1,34 @@
+---
+
+  - name: Make sure optional repo is enabled
+    command: subscription-manager repos --enable rhel-7-server-optional-rpms
+    tags: rpm
+
+  - name: Make sure extras repo is enabled
+    command: subscription-manager repos --enable rhel-7-server-extras-rpms
+    tags: rpm
+
+  - name: Install Docker
+    yum: name=docker state=installed
+    tags: rpm
+
+  - name: Enable insecure registries
+    lineinfile: state=present dest=/etc/sysconfig/docker regexp="^INSECURE_REGISTRY=" line="INSECURE_REGISTRY='--insecure-registry 172.30.0.0/16'" insertafter="^# INSECURE_REGISTRY="
+
+  - name: Check if sdb is empty
+    command: sfdisk -d /dev/sdb
+    register: sfdisk
+    failed_when: sfdisk.stdout != "" or sfdisk.stderr != "" # sdb is empty
+    tags: storage
+
+  - name: Configure docker-storage-setup
+    template: dest=/etc/sysconfig/docker-storage-setup src=docker-storage-setup
+    tags: storage
+
+  - name: Run docker-storage-setup
+    command: docker-storage-setup
+    tags: storage
+
+  - name: Start Docker
+    service: name=docker state=started enabled=yes
+    tags: wip

OpenShift-Lab-Ansible-Playbook/roles/docker/templates/docker-storage-setup

@@ -0,0 +1,4 @@
+STORAGE_DRIVER=devicemapper
+CONTAINER_THINPOOL=docker-lv1
+DEVS=/dev/sdb
+VG=docker

OpenShift-Lab-Ansible-Playbook/roles/docker/templates/sdb-partitions

@@ -0,0 +1,7 @@
+label: dos
+unit: sectors
+
+start=     2048, size= 41940992, Id=8e
+start=        0, size=        0, Id= 0
+start=        0, size=        0, Id= 0
+start=        0, size=        0, Id= 0

OpenShift-Lab-Ansible-Playbook/roles/name-resolution/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+
+  - name: Install dnsmasq
+    yum: name=dnsmasq state=installed
+    when: "'admin' in group_names" # Only on admin server
+    tags: rpm
+
+  - name: Set dnsmasq config
+    template: src=dnsmasq.conf dest=/etc/dnsmasq.conf
+    when: "'admin' in group_names" # Only on admin server
+    tags: config
+
+  - name: Generate an /etc/hosts with all hosts
+    template: dest=/etc/hosts.dnsmasq src=hosts
+    when: "'admin' in group_names" # Only on admin server
+    tags: config
+
+  - name: Make sure dnsmasq daemon is enabled and started
+    service: name=dnsmasq state=started enabled=yes
+    when: "'admin' in group_names" # Only on admin server
+    tags: config
+
+  - name: Add an iptable rule to allow DNS queries from other hosts
+    lineinfile: dest=/etc/sysconfig/iptables line="-A INPUT -p udp --dport 53 -j ACCEPT" insertafter="-A INPUT -i lo -j ACCEPT"
+    when: "'admin' in group_names" # Only on admin server
+    tags: iptables
+
+  - name: Restart iptables
+    service: name=iptables enabled=yes state=restarted
+    when: "'admin' in group_names" # Only on admin server
+    tags: iptables
+
+  - name: Fix the /etc/resolv.conf of other hosts
+    template: dest=/etc/resolv.conf src=resolv.conf
+    when: "'admin' not in group_names" # On all other nodes
+    tags: config

OpenShift-Lab-Ansible-Playbook/roles/name-resolution/templates/dnsmasq.conf

@@ -0,0 +1,28 @@
+# {{ ansible_managed }}
+
+domain-needed
+bogus-priv
+expand-hosts
+log-queries
+local-ttl=60
+
+# Do not read the default /etc/hosts
+no-hosts
+
+# But read this one...
+addn-hosts=/etc/hosts.dnsmasq
+
+# Default suffix for all machines
+domain={{ dns_suffix }}
+
+#
+# Wildcard DNS entries (see openshift_cluster_dns variable)
+#
+# note: will generate something like this :
+#       address=/app.openshift.test/192.168.23.20
+#
+{% for item in groups['nodes'] %}
+{% if 'onlyforinfra' in hostvars[item] %}
+address=/{{openshift_cluster_dns}}/{{ hostvars[item]['ansible_default_ipv4']['address'] }}
+{% endif %}
+{% endfor %}

OpenShift-Lab-Ansible-Playbook/roles/name-resolution/templates/hosts

@@ -0,0 +1,5 @@
+# {{ ansible_managed }}
+
+{% for item in groups['lab'] %}
+{{ hostvars[item]['ansible_default_ipv4']['address'] }} {{ hostvars[item]['inventory_hostname']}} {{ hostvars[item]['inventory_hostname_short']}}
+{% endfor %}

OpenShift-Lab-Ansible-Playbook/roles/name-resolution/templates/resolv.conf

@@ -0,0 +1,4 @@
+search {{ dns_suffix }}
+{% for item in groups['admin'] %}
+nameserver {{ hostvars[item]['ansible_default_ipv4']['address'] }}
+{% endfor %}

OpenShift-Lab-Ansible-Playbook/roles/openshift-install/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+
+  # install atomic-openshift-utils
+  # run atomic-openshift-installer install
+  # check answer file in ~/.config/openshift/installer.cfg.yml
+
+# oc label node master1.example.com region="infra" zone="na"
+# oc label node infranode1.example.com region="infra" zone="infranodes"
+# oc label node node1.example.com region="primary" zone="east"
+# oc label node node2.example.com region="primary" zone="west"
+
+#oadm registry --config=/etc/origin/master/admin.kubeconfig \
+#    --service-account=registry \
+#    --selector='region=infra'
+# --mount-host=<path>
+
+# deploy openshift3/ose-haproxy-router
+
+#oc create|delete -f \
+#    examples/image-streams/image-streams-rhel7.json \
+#    -n openshift
+
+#oc create|delete -f \
+#    examples/xpaas-streams/jboss-image-streams.json
+#    -n openshift
+
+#oc create -f \
+#    examples/db-templates -n openshift
+
+#oc create|delete -f \
+#    examples/quickstart-templates -n openshift
+
+{
+  "apiVersion": "v1",
+  "kind": "PersistentVolume",
+  "metadata": {
+    "name": "pv0001"
+  },
+  "spec": {
+    "capacity": {
+        "storage": "5Gi"
+    },
+    "accessModes": [ "ReadWriteOnce" ],
+    "nfs": {
+        "path": "/tmp",
+        "server": "172.17.0.2"
+    },
+    "persistentVolumeReclaimPolicy": "Recycle"
+  }
+}
+
+# setsebool -P virt_use_nfs 1
+
+# /example_fs *(rw,all_squash) (in /etc/exports)
+
+# chown -R nfsnobody:nfsnobody /example_fs
+# chmod 777
+
+# see https://github.com/openshift/openshift-ansible/tree/master/roles/kube_nfs_volumes

OpenShift-Lab-Ansible-Playbook/roles/openshift-prereq/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+
+  - name: This module has only been tested on RHEL 7.3 x64
+    assert:
+      that:
+        - "ansible_userspace_bits == '64'"
+        - "ansible_os_family == 'RedHat'"
+        - "ansible_distribution_version == '7.3'"
+
+  - name: Install wget
+    yum: name=wget state=installed
+    tags: rpm
+
+  - name: Install bridge-utils
+    yum: name=bridge-utils state=installed
+    tags: rpm
+
+  - name: Install bash-completion
+    yum: name=bash-completion state=installed
+    tags: rpm
+
+  - name: Install GIT
+    yum: name=git state=installed
+    tags: rpm
+
+  - name: Install net-tools
+    yum: name=net-tools state=installed
+    tags: rpm
+
+  - name: Install bind-utils
+    yum: name=bind-utils state=installed
+    tags: rpm
+
+  - name: Install iptables-services
+    yum: name=iptables-services state=installed
+    tags: rpm
+
+  - name: Disable firewalld
+    service: name=firewalld state=stopped enabled=no
+
+  - name: Enable iptables
+    service: name=iptables state=started enabled=yes
+
+  - name: Check for existing SSH Private Key on the admin server
+    stat: path=/home/{{ ansible_ssh_user }}/.ssh/id_rsa
+    register: key
+    when: "'admin' in group_names" # Only on admin server
+    tags: ssh-key
+
+  - name: Generate an SSH Private Key on the admin server
+    command: ssh-keygen -t rsa -b 2048 -f /home/{{ ansible_ssh_user }}/.ssh/id_rsa -q -N ''
+    become_user: "{{ ansible_ssh_user }}"
+    when: "'admin' in group_names and key.stat.exists == False" # Only on admin server and if key does not exists
+    tags: ssh-key
+
+  - name: Fetch the SSH Public Key of the admin server
+    fetch: src=/home/{{ ansible_ssh_user }}/.ssh/id_rsa.pub dest="{{ basedir }}/admin.pub" flat=yes
+    when: "'admin' in group_names" # Only on admin server
+    tags: ssh-key
+
+  - name: Add SSH Public key of the admin server to the authorized_keys of each other server
+    authorized_key:
+      key: "{{ lookup('file', basedir + '/admin.pub' ) }}"
+      user: "{{ ansible_ssh_user }}"
+      state: present
+    when: "'admin' not in group_names" # Only on other servers
+    tags: ssh-key

OpenShift-Lab-Ansible-Playbook/roles/register-rhn/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+  - name: This module should only work on RHEL
+    assert:
+      that:
+        - "ansible_os_family == 'RedHat'"
+
+  - name: Register this system on RHN
+    redhat_subscription:
+      state: present
+      username: "{{ lookup('env','RHN_LOGIN') }}"
+      password: "{{ lookup('env','RHN_PASSWORD') }}"
+      consumer_name: "{{ inventory_hostname }}"
+      autosubscribe: false
+    tags: rhn
+
+  - name: Attach the correct pool id to the new subscription
+    command: subscription-manager attach --pool=8a85f98159c85ca00159c9ad5a823661 # Red Hat Enterprise Linux Developer Suite
+    tags: rhn

OpenShift-Lab-Ansible-Playbook/site.yml

@@ -0,0 +1,32 @@
+---
+
+ - name: Bootstrap one or more RHEL7 nodes
+   hosts: bootstrap
+   become: no
+   roles:
+    - bootstrap
+    - register-rhn
+
+ - name: Install the required package for an OpenShift Lab
+   hosts: lab
+   become: yes
+   vars:
+    - basedir: "{{ lookup('env', 'BASEDIR') }}"
+   roles:
+#    - base
+#    - openshift-prereq
+     - name-resolution
+
+ - name: Install Docker
+   hosts:
+     - nodes
+     - masters
+   become: yes
+   roles:
+#    - docker
+
+ - name: Install the admin node
+   hosts: admin
+   become: yes
+   roles:
+  #   - nfs