From 51422387a92323a678b1f761d25ef19b9c55913c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= <nicolas.masse@itix.fr>
Date: Thu, 31 Jan 2019 11:30:34 +0100
Subject: [PATCH] see #31: split the CI tests in multiple jobs so that Travis
 builds do not timeout

---
 .travis.yml                     |  12 ++++++++++--
 tests/3scale-inventory.yaml.enc | Bin 2256 -> 1168 bytes
 tests/inventory.j2              |  10 ++++++----
 tests/results/.gitignore        |   1 +
 tests/run-tests.sh              |  16 +++++++++++++---
 5 files changed, 30 insertions(+), 9 deletions(-)
 create mode 100644 tests/results/.gitignore

diff --git a/.travis.yml b/.travis.yml
index 3b447be..c85a4af 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,9 +2,17 @@ language: python
 matrix:
   include:
   - python: '2.7'
-    env: ANSIBLE_VERSION=2.4.6 THREESCALE_POOL=0
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas-apicast-self-managed
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=onpremise-2.3
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas-apicast-self-managed
   - python: '3.6'
-    env: ANSIBLE_VERSION=2.7.5 THREESCALE_POOL=1
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=onpremise-2.3
 install:
 - pip install ansible==$ANSIBLE_VERSION
 - pip install jmespath
diff --git a/tests/3scale-inventory.yaml.enc b/tests/3scale-inventory.yaml.enc
index 032a4d713ae4ce3ac7b82bb9676bd6474d1aa7d9..390d2e5f0f8e346a7c4cf17543253f134db8358f 100644
GIT binary patch
literal 1168
zcmV;B1aJE>%X7lmH({<wXL#M5?I5csRuqs}3kZCwzgt`2E19t@`({yfHgH9}0fcE+
zu%4kw9<~X@>D$+R9c=6+zQT|W>RPlhkFY}L(pECfofS|rB_14KLNs-XI6I#k^^^ID
zB-@eZNj~f;Apipyp8{px?&O$#DEV-3)3pEedG>oOCB|c*{~h+{zr7SRCp6<Yeb!9<
zb4GvU>wQT__g7{it9(WbsdH@I<BQ|WbfH6goI_M=ACa+kFDk(5q_xU^tXayAS}ys(
zFf--To=g_pO@AqlDz9>Ppq>tQ9NObA^}EAbye<qXB<s}Je=3dbQ5VESI)fM0XfZ7=
z+!7&D<_=a2(9!hvKmkq#cx|oJ9jP<${CheSpUsChr1_HYlYIMqw(2ObWN<B#azzSU
zgi5~ob;D1kqt-sSZ1GMba)S?r?pJ!N?Pk9=pK8RAJ8S+NeS|Yp@>+{w5+t93&~PFz
zy=j#dpfbV6z=h`<LYfHAT6{(`wK=A6F`>LYbI*;Xnn0b&7~KC@<zjsHQq#UAr-c?@
z<);=5r5x(zfRt^3Y-dixz7KBB|IzRSiF%d6`qK~iUCO-q7H5BGNb?yPhmW>f@$zA6
zcCC-6c<C7|rj~fGo61ZVo>2@lQHLym9Ts_x)LzA8_ya}cOL>gGPDTx3y52x5F0|Vy
zvcH~mpHkqJZ(K1c3?$~i9jhPzkR}wFZfothI|<Vfy!Zk^+x1R?g|OWBt<Mv9^_9<a
zu+-ysb8%eOe>~uTn98VH6puUk8&$~bXGMVLeB=0X{)~bDsw9RU>7u8DvLhKQzGInk
zuezc*GJujJJHTcsKK;cyE_KNI;o8rT*h=eNk?AXD*0pb~(o4g3AUIzw7`0D>7S80p
z$k()oKR-HfOiDsSxjQ{Mx6iSwQ42qm&jeHkIzTx*KHE{X=aoBBc>OWf65)is&oRTb
zTomf|rgU26XKrN5^EuKu6Nf{2Y=}PS1=dlMpt|2%aF!LZB;Bx>Baf<2iu<TQRBt*v
zNDc%YxGZwNx=k;Nv6P<!zKml<LALwkG(XtX#<q1SwUuo##u>Z=W`Z#bWad(D_<Gz+
zIbjqFWAX4B%j?P+l)LWva%ng?`=>n0B&HR`Z(e-?dE7%@w*M-BGr;PIPo%uUuMzk=
z9@mGOdcDYN?w#dnAHh)!8G<|}`iVpFou<`i^BW7(ow`5jkbA$$f(-!EY6FA{rsNxk
za}^B+LBGm}=pAcikc-xPJNEU!JQYzV+*Q`1bvwdrQUZV*J#)6$w`OKZ*v#vCL^a8@
zq@C$$uO{+LTg7n*gcJ^IkmzTV$cIb10>@^=DBcF&?t3_(kyE#H+MaT%%{#zfc5(uY
ziNPo0cqTeUv_gDX=jB2t)Rl(3rxDpLS6ci%W}_JV-dC};^Z?tf*3*HC8estfj}#>&
zVcO&-Llm&3ZgcGv9pV@dZr%5YzxdoNZ%3%_$OryKlBPq;y&&#ve2tfg-iP6O3~c=Y
ic-$?IlxB&MR)qRT)Mj}KuM=$@aS-<RPzeL<o*3jkVM#au

literal 2256
zcmV;>2ru{0_{!I78DNf-k51TbM5Kh+kzR(O0L<C$n<+c|;B_AL;cBQ2-fw0W&y^j#
z<nf>keKhrTT4iHfn@YaV;#zXM*3IIOOo%db!xy_C;)C+9<{jBNi!J-RVio`xl2V>&
z1Rl$ICjdSU(zN?Wg9lTM?*9EkY?o1zcPNSoWy8$Bn1FfYo)nq%MGD?>JFoHSjdP2!
zS>QQRh6$FKxXqS4t>pusHm2UJIN<iS2~(}BpjfD%-8WpDpVk@|Swv(+Z2<eYw2Yoo
zNe<4D=m`Q?_-#T^V0DlZ$&AWb{Tl8S_<@x#xJie)O=nH`GRu+K@yFG|8UDVKSK?tO
z69oeS3%Cr%m5e*5kRFkCv-~7k1@0++sOWie^9USh<3Jfo?VUiq1p)%l*>ek4zG0#!
z@ef8nMItQV-_#Gxeq4GX;vy~c32cD0Tvr&gO0*!h{9$gPI&wK0Ir7&~>4OY6+ocN;
zJb8ZXi6g94U$}A}Fvr7M)98p8bOF5XE>tgq;FFhEbe@;6@xMdIjYOyIU9dYByw3(I
zrKuZi)D8ay-3<7OgM@o=!9C2hR?Z5FA7^weybTvB(kjbzHayY*(QAh|bxjij>iS&r
z0OWul7pd0*Y4bIRkpe5`rkI!v=i1+cy5eXE0lg*lXQPM)t<%YmfNP0><%o41Wl2*c
zbjW^a^VYHporXp`q(&=)_P#SP|Kx`NoHpy5V-xckey*A&wPl}MmrB~bl!ADUtc&&m
z-a6dL8<Sa(|2pjpkzQi{(}PJH!L{FE8kVIIZ^V4}CSshE66XgJUC)*)2Yq_`(L<@l
zX>{!{NmMuu7#!-WNOK2w<H!+5XJ{`<5n@}H=$Ytxnef3~l|2Xwr=ev?{4>_WsGi3T
zZm0U_*GxeI^FbgY#+|qJ<|=JYrWC@+`s9L?<&TCRsiRs_>^mP4&r4n#F+r9V^?iWi
zSe)yG!8a&ry|z8QaR$HHg$Q6UE?@L=Nb<0erwdqMgG0@y)M{`IhNtljP|QWiur{Hz
zT3>lK^b{CwDqr`pisf&fbDofm=uSrZ`@S&$)3d2O%kjCY9VPIpX?S47_mlzY2kBN-
zw|Hvdxe4e*D>t7WE8*oC585<&l8LtCXn<9DaM3q(xsw&U!SOSDy$WB1v&%X$<@v;3
z0J%7+vAI9$DQ2N_y*(jQ+$_@MhqCI(lcN{f;W_s0Qzn$-x>Mn2I)LLS;l9X)`VHN{
zcqGNQ{lM1FB&<vNp?CGQ?fLI)h$kxb53)d<)zug<|DC(q+9wBP*u0-Ct`|)iZT9)L
z2*x@W7RIRADdC6G!Vv8tTelS#qk#5w5R0Kl|0khuIa8!6W}6B73N8gZur;K3oWt-{
z9?)5$<3+bW#qCYVt^o2B;jM^h3(BlmwMg0c!*>bV;rK^4%Fyy1opX^*#4%y?!ECzC
z8`Y0lz_k7`lG()wuKXHu{At1YQxHP)OyW+a9hYkI2`k}hPwzW^MTV$&R{Ox(a=WGY
z?@=M}k@(y^{;nrOz-EZ};9WjNtn|Q#<L-4_Zkiv(US2lNGKGj$NwcTQ;@qeA1dYsa
zTHCe=1{Y)32h0wc-XI&@U3)DDI{oSmxiGoHvy=nQa4krZD0@MRL;Kr4e;oGgu(fwS
ze&Ikb`SWg@c+<h6@FMTf?5a%5jyX{ElR(<12GN6N@ia&FOO#3@Um9{S@^Jl!U0#Z~
zbP?;p9#%Fp<}<#=)%-T7;qwSuO<8X;<6L-vM~HjfE2(V`ta*+C{=Mdnnf>YGyY5&$
z^mO!@lLV+(NqslNiGcH-pOK(H?upluUwl@!Zw2v$$E!pCj8HJR$G>XUU=sKLR1r9y
z&9`*jNdi2XMZd@W%Q~#kFmv;yd$tyjSh<8&%F^ZJb2!$Q$W$O1K+3D$fo=if9s6}$
z%IMV*i`vDSfaDzF)S7u*xN^y4f$$=L>!J))n=TTppWoa&89hg&txxUZ(ECzPs%CQa
z-YkK^1O4=)m2nEy{6T!XHTohCM*?J*x(^#+(1~F)^-|tzHrbC+6o;N7%52Q*HZ}Wt
zZDJaHlI^%sT2aanhWkxfP3b~{cd_NG%Y}5fxZ92{dfdase^Qq{h=ionjG%*v=)#xy
zNLMDVt@{O(5ok?zaY@f1(if^i$H@}Q;TZxfS9nU;%{Q;ehe#Pe^zkrTuYMUUTdX2%
z!Mr*4_L&vu5IC3AyqhIG1pOr^0+<q54wsS+oKvLzqk*~IQ08KnIzU=lO9qDwjja_K
z0F6kH1IjyfvR;(m(zt~>>6_2l%n-h@G(eVgdwsQFi6Y&4K%CgdGvvr$LN7`pDq5*u
zq!<Zq%r|tS$0~Gq*6iB7cjh15iDF1CsVU;CX3iKk2s_FXq8VBHtMyk<C`2i9`I(#$
z=bq7*@#-XaNo2Etmgsx6pwNJwG@W7$o{x3k4`Fh7@~5c9QRnKCT}i)^^57E_ALc{&
z9_PA_H`y*RjFcbd$b63E?NTB93@)6`dfc%+fnw*45t9lH!_l*osk{ei^gj=#Q9eu9
zcI{>7UEWgXvWdrV=TIi@j(o8@p^o;jv%PrZ{sgVk(kAJZTp>sX$|@n81mQO|>STg{
zl=Cipq=fnd)uE(^OEiEqm;K@*t3nfD!))jao*3{9^iYC0?i_~;<kK1ynyqx>_Krpt
zWiJ(*7*P@hPV|v>3rPs4Qdhpc$?JeSv4yVzJUo=0s$nT;I)hjQA=8>3;C69<Q_~+(
zw_j@P5+s|yY5V?b;Ic~OTDIK}{|xJ`laBSHK*r4B=T$u@4fHWbE`cIk2>Li4X_5UK
zZpW1S5v5{RL<Rw3#<O(e;FPm+b-;1U9|C~Ak&OAE-*(b;?dN#7D78Li-{zC3$p|GS
z{xB8t&2zs?BH_?D{%W*7szL*~U>Y`v&~RgZ2lkmzQY==@Qegh@1*INr7EE8iyJe}o
eS0ET!_s#e%YfX_-8eBX^&TPj^stVJ@DOaYmb%C(}

diff --git a/tests/inventory.j2 b/tests/inventory.j2
index 5176629..037c3a7 100644
--- a/tests/inventory.j2
+++ b/tests/inventory.j2
@@ -1,7 +1,3 @@
-{# Select the target CI platform using the THREESCALE_POOL environment variable #}
-{% set pool = lookup('env', 'THREESCALE_POOL') %}
-{% set test_environment = test_environment[pool|int] %}
-
 [all:vars]
 ansible_connection=local
 
@@ -22,3 +18,9 @@ threescale_cicd_apicast_sandbox_endpoint='{% raw %}{{ threescale_cicd_default_ap
 {% if 'apicast_production_domain' in test_environment.threescale %}
 threescale_cicd_apicast_production_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ ((threescale_cicd_api_environment_name ~ "-" if threescale_cicd_api_environment_name is defined else "") ~ threescale_cicd_api_system_name)|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_production_suffix }}{% endraw %}.{{ test_environment.threescale.apicast_production_domain }}'
 {% endif %}
+{% if test_environment.threescale.admin_portal is match(".*[.]3scale[.]net") %}
+# The 3scale SaaS environment is sometimes slow. Make sure to give enough time
+# to the SaaS environment to synchronise the OIDC clients in RH-SSO.
+threescale_cicd_delay=15
+threescale_cicd_retries=200
+{% endif %}
diff --git a/tests/results/.gitignore b/tests/results/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/tests/results/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/tests/run-tests.sh b/tests/run-tests.sh
index 09aa388..483b525 100755
--- a/tests/run-tests.sh
+++ b/tests/run-tests.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
+# Make sure all Ansible failed tasks go to the stderr. Failed tasks usually
+# output sensitive informations, by routing them to stderr we can filter
+# them out.
+export ANSIBLE_DISPLAY_FAILED_STDERR=yes
 
 cd "${0%/*}" || exit 1
 
 echo "--> Generating the Ansible inventory files..."
-ansible-playbook -i /dev/null write-inventory-files.yml
+ansible-playbook -i /dev/null write-inventory-files.yml &>results/write-inventory-files
 ret=$?
 if [ "$ret" -gt 0 ]; then
     echo "--> Ansible inventory files generation FAILED !"
@@ -23,10 +27,16 @@ fi
 #
 cd ".." || exit 1
 
-for environment in tests/environments/3scale-*; do
+for environment in tests/environments/3scale-${THREESCALE_ENV:-*}; do
     for testcase in tests/test-cases/*.y*ml; do
         echo "--> Running $testcase against $environment..."
-        ansible-playbook -i "$environment" -v "$testcase"
+        if [ -z "$THREESCALE_VERBOSE" ] || [ "$THREESCALE_VERBOSE" == "no" ]; then
+            # reduce output verbosity and make sure not to output sensitive information
+            logfile="tests/results/$(basename "$environment")-$(basename "$testcase")"
+            DISPLAY_SKIPPED_HOSTS=no ANSIBLE_DISPLAY_OK_HOSTS=no ansible-playbook -i "$environment" "$testcase" 2>"$logfile"
+        else
+            ansible-playbook -i "$environment" -v "$testcase"
+        fi
         ret=$?
         if [ "$ret" -gt 0 ]; then
             echo "--> $testcase against $environment FAILED !"