see #31: split the CI tests in multiple jobs so that Travis builds do not timeout

.travis.yml

@@ -2,9 +2,17 @@ language: python
 matrix:
   include:
   - python: '2.7'
-    env: ANSIBLE_VERSION=2.4.6 THREESCALE_POOL=0
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas-apicast-self-managed
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=onpremise-2.3
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas-apicast-self-managed
   - python: '3.6'
-    env: ANSIBLE_VERSION=2.7.5 THREESCALE_POOL=1
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=onpremise-2.3
 install:
 - pip install ansible==$ANSIBLE_VERSION
 - pip install jmespath

tests/inventory.j2

@@ -1,7 +1,3 @@
-{# Select the target CI platform using the THREESCALE_POOL environment variable #}
-{% set pool = lookup('env', 'THREESCALE_POOL') %}
-{% set test_environment = test_environment[pool|int] %}
-
 [all:vars]
 ansible_connection=local
 
@@ -22,3 +18,9 @@ threescale_cicd_apicast_sandbox_endpoint='{% raw %}{{ threescale_cicd_default_ap
 {% if 'apicast_production_domain' in test_environment.threescale %}
 threescale_cicd_apicast_production_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ ((threescale_cicd_api_environment_name ~ "-" if threescale_cicd_api_environment_name is defined else "") ~ threescale_cicd_api_system_name)|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_production_suffix }}{% endraw %}.{{ test_environment.threescale.apicast_production_domain }}'
 {% endif %}
+{% if test_environment.threescale.admin_portal is match(".*[.]3scale[.]net") %}
+# The 3scale SaaS environment is sometimes slow. Make sure to give enough time
+# to the SaaS environment to synchronise the OIDC clients in RH-SSO.
+threescale_cicd_delay=15
+threescale_cicd_retries=200
+{% endif %}

tests/results/.gitignore

@@ -0,0 +1 @@
+*

tests/run-tests.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
+# Make sure all Ansible failed tasks go to the stderr. Failed tasks usually
+# output sensitive informations, by routing them to stderr we can filter
+# them out.
+export ANSIBLE_DISPLAY_FAILED_STDERR=yes
 
 cd "${0%/*}" || exit 1
 
 echo "--> Generating the Ansible inventory files..."
-ansible-playbook -i /dev/null write-inventory-files.yml
+ansible-playbook -i /dev/null write-inventory-files.yml &>results/write-inventory-files
 ret=$?
 if [ "$ret" -gt 0 ]; then
     echo "--> Ansible inventory files generation FAILED !"
@@ -23,10 +27,16 @@ fi
 #
 cd ".." || exit 1
 
-for environment in tests/environments/3scale-*; do
+for environment in tests/environments/3scale-${THREESCALE_ENV:-*}; do
     for testcase in tests/test-cases/*.y*ml; do
         echo "--> Running $testcase against $environment..."
+        if [ -z "$THREESCALE_VERBOSE" ] || [ "$THREESCALE_VERBOSE" == "no" ]; then
+            # reduce output verbosity and make sure not to output sensitive information
+            logfile="tests/results/$(basename "$environment")-$(basename "$testcase")"
+            DISPLAY_SKIPPED_HOSTS=no ANSIBLE_DISPLAY_OK_HOSTS=no ansible-playbook -i "$environment" "$testcase" 2>"$logfile"
+        else
             ansible-playbook -i "$environment" -v "$testcase"
+        fi
         ret=$?
         if [ "$ret" -gt 0 ]; then
             echo "--> $testcase against $environment FAILED !"