Improve the CI tests (#42)

#31 : CI tests are run against SaaS, with hosted and self-managed APIcast + on-premise AMP 2.3
#29 : Fix the CORS smoketests that were failing
#41 : Make sure the playbook did not expose the 3scale access token in the output

.travis.yml

@@ -2,28 +2,30 @@ language: python
 matrix:
   include:
   - python: '2.7'
-    env: ANSIBLE_VERSION=2.4.6 THREESCALE_POOL=0
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=saas-apicast-selfmanaged
+  - python: '2.7'
+    env: ANSIBLE_VERSION=2.4.6 THREESCALE_ENV=onpremise-2.3
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas
+  - python: '3.6'
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=saas-apicast-selfmanaged
   - python: '3.6'
-    env: ANSIBLE_VERSION=2.7.5 THREESCALE_POOL=1
+    env: ANSIBLE_VERSION=2.7.5 THREESCALE_ENV=onpremise-2.3
 install:
 - pip install ansible==$ANSIBLE_VERSION
 - pip install jmespath
 # Pre-install go-swagger locally since it cannot be fetched from the Travis-CI
 # infrastructures because of rate limits imposed by GitHub on its API.
-- mkdir tests/bin/ && curl -L -o tests/bin/swagger https://github.com/go-swagger/go-swagger/releases/download/0.16.0/swagger_linux_amd64 && chmod 755 tests/bin/swagger
+- mkdir -p tests/test-cases/bin/ && curl -L -o tests/test-cases/bin/swagger https://github.com/go-swagger/go-swagger/releases/download/0.16.0/swagger_linux_amd64 && chmod 755 tests/test-cases/bin/swagger
 script:
-- ansible-playbook tests/write-inventory-files.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-apikey.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-oidc.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-with-basePath.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-multi-environment.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-with-smoketest-in-extra-vars.yml
-- ansible-playbook -v -i tests/inventory tests/3scale-saas-with-hosted-apicast-without-smoketest.yml
+- tests/run-tests.sh
 notifications:
   webhooks: https://galaxy.ansible.com/api/v1/notifications/
 branches:
   except:
   - /^dev.*/
 before_install:
-- openssl aes-256-cbc -K $encrypted_7bf6043000c3_key -iv $encrypted_7bf6043000c3_iv
-  -in tests/3scale-inventory.yaml.enc -out tests/3scale-inventory.yaml -d
+# travis encrypt-file tests/3scale-inventory.yaml tests/3scale-inventory.yaml.enc
+- openssl aes-256-cbc -K $encrypted_5ba3c614c7e1_key -iv $encrypted_5ba3c614c7e1_iv -in tests/3scale-inventory.yaml.enc -out tests/3scale-inventory.yaml -d

defaults/main.yml

@@ -17,6 +17,13 @@ threescale_cicd_application_plans:
   state: hidden
   name: Ansible Test Plan
 
+# Controls when to log sensitive information. Can be set to false for
+# production environments.
+#
+# By default, log sensitive information only when Ansible is called with
+# A verbosity level of at least one "-v".
+threescale_cicd_nolog: '{{ ansible_verbosity|default(0) == 0 }}'
+
 # A folder where to download dependencies, when required
 threescale_cicd_local_bin_path: '{{ playbook_dir }}/bin'
 

tasks/api-calls/create_activedoc.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_activedoc_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the ActiveDocs
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_existing_activedocs: '{{ threescale_cicd_existing_activedocs|union([ threescale_cicd_tmpresponse.json.api_doc.system_name ]) }}'

tasks/api-calls/create_application.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_application_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the application
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_default_application_details: '{{ threescale_cicd_tmpresponse.json.application }}'

tasks/api-calls/create_application_plan.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_application_plan_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the application plan
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_existing_application_plans: '{{ threescale_cicd_existing_application_plans|union([ threescale_cicd_application_plan.system_name ]) }}'

tasks/api-calls/create_mapping_rule.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_mapping_rule_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the mapping rule
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/create_method.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_method_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the method
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_existing_metrics: '{{ threescale_cicd_existing_metrics|union([ threescale_cicd_api_operation.key ]) }}'

tasks/api-calls/create_service.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_create_service_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Create the service
   uri:
@@ -13,6 +14,7 @@
     status_code: 201
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_existing_services: '{{ threescale_cicd_existing_services|union([ threescale_cicd_tmpresponse.json.service.system_name ]) }}'

tasks/api-calls/delete_mapping_rule.yml

@@ -8,6 +8,7 @@
     status_code: 200,404
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/delete_metric.yml

@@ -11,6 +11,7 @@
     status_code: 200,404
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/find_application.yml

@@ -7,6 +7,7 @@
     method: GET
     status_code: 200,404
   register: threescale_cicd_tmpresponse
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_default_application_id: '{{ threescale_cicd_tmpresponse.json.application.id }}'

tasks/api-calls/find_first_account.yml

@@ -5,6 +5,7 @@
     url: https://{{ inventory_hostname }}/admin/api/accounts.json?access_token={{ threescale_cicd_access_token|urlencode }}&state=approved&page=1&per_page=1
     validate_certs: no
   register: threescale_cicd_tmpresponse
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_default_account_id: '{{ threescale_cicd_tmpresponse.json.accounts[0].account.id }}'

tasks/api-calls/get_proxy_version.yml

@@ -5,6 +5,7 @@
     url: 'https://{{ inventory_hostname }}/admin/api/services/{{ threescale_cicd_api_service_id }}/proxy/configs/{{ threescale_cicd_staging_environment_name }}/latest.json?access_token={{ threescale_cicd_access_token|urlencode }}'
     validate_certs: no
   register: threescale_cicd_tmpresponse
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_staging_proxy_version: '{{ threescale_cicd_tmpresponse.json.proxy_config.version }}'
@@ -15,6 +16,7 @@
     validate_certs: no
     status_code: 200,404
   register: threescale_cicd_tmpresponse
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_production_proxy_version: '{{ threescale_cicd_tmpresponse.json.proxy_config.version if threescale_cicd_tmpresponse.status == 200 else ''NONE'' }}'

tasks/api-calls/keycloak/authenticate.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_authenticate_to_keycloak_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Authenticate to RH-SSO
   uri:
@@ -16,7 +17,9 @@
   delay: '{{ threescale_cicd_delay }}'
   # temporary fix for https://github.com/ansible/ansible/issues/28078
   until: 'threescale_cicd_tmpresponse is success'
+  no_log: '{{ threescale_cicd_nolog }}'
   
 - name: Extract the access_token
   set_fact:
     threescale_cicd_keycloak_access_token: '{{ threescale_cicd_tmpresponse.json |json_query("access_token") }}'
+  no_log: '{{ threescale_cicd_nolog }}'

tasks/api-calls/keycloak/patch_client.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_patch_keycloak_client_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Patch the client in RH-SSO to support the "client_credentials" and "password" grant_type.
   uri:
@@ -17,6 +18,7 @@
       Content-Type: 'application/json'
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/keycloak/wait_for_client.yml

@@ -12,6 +12,7 @@
   retries: '{{ threescale_cicd_retries }}'
   delay: '{{ threescale_cicd_delay }}'
   until: 'threescale_cicd_tmpresponse is success and threescale_cicd_tmpresponse.json|length > 0'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_default_application_sso_id: '{{ threescale_cicd_tmpresponse.json[0].id }}'

tasks/api-calls/promote_proxy.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_promote_proxy_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Promote to production
   uri:
@@ -13,6 +14,7 @@
     method: POST
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 201'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/smoke_test.yml

@@ -22,9 +22,10 @@
 - name: Running smoke tests (CORS) !
   uri:
     url: '{{ threescale_cicd_smoke_test_url }}'
-    headers: '{{ threescale_cicd_smoke_test_headers|combine({ ''Origin'': threescale_cicd_smoke_test_url}) }}'
+    headers: '{{ threescale_cicd_smoke_test_headers|combine({ ''Origin'': threescale_cicd_smoke_test_url, ''Access-Control-Request-Method'': ''GET'' }) }}'
     validate_certs: no
     method: OPTIONS
+    status_code: 200,204
   register: threescale_cicd_tmpresponse
   retries: '{{ threescale_cicd_retries }}'
   delay: '{{ threescale_cicd_delay }}'

tasks/api-calls/update_activedoc.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_activedoc_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the ActiveDocs
   uri:
@@ -13,6 +14,7 @@
     status_code: 200
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/update_application.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_application_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the application
   uri:
@@ -12,6 +13,7 @@
     body: '{{ threescale_cicd_update_application_payload }}'
     status_code: 200
   register: threescale_cicd_tmpresponse
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - set_fact:
     threescale_cicd_default_application_details: '{{ threescale_cicd_tmpresponse.json.application }}'

tasks/api-calls/update_application_plan.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_application_plan_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the application plan
   uri:
@@ -13,6 +14,7 @@
     status_code: 200
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/update_mapping_rule.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_mapping_rule_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the mapping rule
   uri:
@@ -15,6 +16,7 @@
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
   vars:
     threescale_cicd_mapping_rule_id: '{{ threescale_cicd_existing_mapping_rules[threescale_cicd_mapping_rule] }}'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/update_method.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_method_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the method
   uri:
@@ -12,6 +13,7 @@
     body: '{{ threescale_cicd_update_method_payload }}'
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/update_policies.yml

@@ -7,6 +7,7 @@
 - debug:
     var: threescale_cicd_update_policies_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the policies chain
   uri:
@@ -17,6 +18,7 @@
     status_code: 200
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tasks/api-calls/update_proxy.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_proxy_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the proxy definition
   uri:
@@ -12,6 +13,7 @@
     body: '{{ threescale_cicd_update_proxy_payload }}'
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Extract the staging and production gateway endpoint from the proxy definition
   set_fact:

tasks/api-calls/update_service.yml

@@ -3,6 +3,7 @@
 - debug:
     var: threescale_cicd_update_service_payload
     verbosity: 1
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Update the service
   uri:
@@ -13,6 +14,7 @@
     status_code: 200
   register: threescale_cicd_tmpresponse
   changed_when: 'threescale_cicd_tmpresponse.status == 200'
+  no_log: '{{ threescale_cicd_nolog }}'
 
 - name: Wait for a couple seconds
   pause:

tests/ansible.cfg

@@ -0,0 +1 @@
+../ansible.cfg

tests/environments/.gitignore

@@ -0,0 +1 @@
+*

tests/inventory.j2

@@ -2,8 +2,25 @@
 ansible_connection=local
 
 [threescale]
-{{ threescale_inventory.threescale_hosted.admin_portal }}
+{{ test_environment.threescale.admin_portal }}
 
 [threescale:vars]
-threescale_cicd_access_token={{ threescale_inventory.threescale_hosted.access_token }}
-threescale_cicd_sso_issuer_endpoint=https://{{ threescale_inventory.sso.client_id }}:{{ threescale_inventory.sso.client_secret }}@{{ threescale_inventory.sso.host }}/auth/realms/{{ threescale_inventory.sso.realm }}
+threescale_cicd_access_token={{ test_environment.threescale.access_token }}
+threescale_cicd_sso_issuer_endpoint=https://{{ test_environment.sso.client_id }}:{{ test_environment.sso.client_secret }}@{{ test_environment.sso.host }}/auth/realms/{{ test_environment.sso.realm }}
+{% if 'wildcard_domain' in test_environment.threescale %}
+threescale_cicd_wildcard_domain={{ test_environment.threescale.wildcard_domain }}
+threescale_cicd_apicast_sandbox_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ threescale_cicd_api_system_name|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_staging_suffix }}.{{ threescale_cicd_wildcard_domain }}{% endraw %}'
+threescale_cicd_apicast_production_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ threescale_cicd_api_system_name|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_production_suffix }}.{{ threescale_cicd_wildcard_domain }}{% endraw %}'
+{% endif %}
+{% if 'apicast_staging_domain' in test_environment.threescale %}
+threescale_cicd_apicast_sandbox_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ ((threescale_cicd_api_environment_name ~ "-" if threescale_cicd_api_environment_name is defined else "") ~ threescale_cicd_api_system_name)|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_staging_suffix }}{% endraw %}.{{ test_environment.threescale.apicast_staging_domain }}'
+{% endif %}
+{% if 'apicast_production_domain' in test_environment.threescale %}
+threescale_cicd_apicast_production_endpoint='{% raw %}{{ threescale_cicd_default_apicast_scheme }}://{{ ((threescale_cicd_api_environment_name ~ "-" if threescale_cicd_api_environment_name is defined else "") ~ threescale_cicd_api_system_name)|regex_replace("[^a-zA-Z0-9-]+", "-")|lower }}{{ threescale_cicd_default_production_suffix }}{% endraw %}.{{ test_environment.threescale.apicast_production_domain }}'
+{% endif %}
+{% if test_environment.threescale.admin_portal is match(".*[.]3scale[.]net") %}
+# The 3scale SaaS environment is sometimes slow. Make sure to give enough time
+# to the SaaS environment to synchronise the OIDC clients in RH-SSO.
+threescale_cicd_delay=15
+threescale_cicd_retries=200
+{% endif %}

tests/results/.gitignore

@@ -0,0 +1 @@
+*

tests/roles/nmasse-itix.threescale-cicd

@@ -1 +0,0 @@
-../..

tests/run-tests.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Make sure all Ansible failed tasks go to the stderr. Failed tasks usually
+# output sensitive informations, by routing them to stderr we can filter
+# them out.
+export ANSIBLE_DISPLAY_FAILED_STDERR=yes
+
+cd "${0%/*}" || exit 1
+
+echo "--> Generating the Ansible inventory files..."
+ansible-playbook -i /dev/null write-inventory-files.yml &>results/write-inventory-files
+ret=$?
+if [ "$ret" -gt 0 ]; then
+    echo "--> Ansible inventory files generation FAILED !"
+    exit 1
+else
+    echo "--> Ansible inventory files generation SUCCEEDED !"
+fi
+
+# Because of a bug in Ansible, we need to move one directory upper before running
+# the playbooks. 
+# 
+# The bug makes the playbooks fail after the Application Plans creation/update
+# with this error message:
+# 
+# ERROR! Unexpected Exception, this is probably a bug: expected str, bytes or os.PathLike object, not NoneType
+#
+cd ".." || exit 1
+
+for environment in tests/environments/3scale-${THREESCALE_ENV:-*}; do
+    for testcase in tests/test-cases/*.y*ml; do
+        echo "--> Running $testcase against $environment..."
+        if [ -z "$THREESCALE_VERBOSE" ] || [ "$THREESCALE_VERBOSE" == "no" ]; then
+            # reduce output verbosity and make sure not to output sensitive information
+            logfile="tests/results/$(basename "$environment")-$(basename "$testcase")"
+            DISPLAY_SKIPPED_HOSTS=no ANSIBLE_DISPLAY_OK_HOSTS=no ansible-playbook -i "$environment" "$testcase" 2>"$logfile"
+        else
+            ansible-playbook -i "$environment" -v "$testcase"
+        fi
+        ret=$?
+        if [ "$ret" -gt 0 ]; then
+            echo "--> $testcase against $environment FAILED !"
+            exit 1
+        else
+            echo "--> $testcase against $environment SUCCEEDED !"
+        fi
+    done
+done

tests/setup/README.md

@@ -0,0 +1,67 @@
+# Test Environment setup
+
+## 3scale SaaS with APIcast Self-Managed
+
+Create a project in an OpenShift cluster:
+
+```sh
+oc new-project apicast-3scale-ci
+```
+
+Deploy two 3scale gateways (staging and production):
+
+```sh
+oc create secret generic 3scale-tenant-<NAME> --from-literal=password=https://<TOKEN>@<NAME>-admin.3scale.net
+oc create -f https://raw.githubusercontent.com/3scale/apicast/v3.4.0/openshift/apicast-template.yml
+oc new-app --template=3scale-gateway --name=apicast-<NAME>-staging -p CONFIGURATION_URL_SECRET=3scale-tenant-<NAME> -p CONFIGURATION_CACHE=0 -p RESPONSE_CODES=true -p LOG_LEVEL=info -p CONFIGURATION_LOADER=lazy -p APICAST_NAME=apicast-<NAME>-staging -p DEPLOYMENT_ENVIRONMENT=sandbox -p IMAGE_NAME=quay.io/3scale/apicast:v3.4.0
+oc new-app --template=3scale-gateway --name=apicast-<NAME>-production -p CONFIGURATION_URL_SECRET=3scale-tenant-<NAME> -p CONFIGURATION_CACHE=60 -p RESPONSE_CODES=true -p LOG_LEVEL=info -p CONFIGURATION_LOADER=boot -p APICAST_NAME=apicast-<NAME>-production -p DEPLOYMENT_ENVIRONMENT=production -p IMAGE_NAME=quay.io/3scale/apicast:v3.4.0
+oc scale dc/apicast-<NAME>-staging --replicas=1
+oc scale dc/apicast-<NAME>-production --replicas=1
+oc expose svc/apicast-<NAME>-staging --wildcard-policy=Subdomain --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }' --hostname=wildcard.<NAME>-staging.app...
+oc expose svc/apicast-<NAME>-production --wildcard-policy=Subdomain --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }' --hostname=wildcard.<NAME>-production.app...
+```
+
+Provision the Red Hat SSO tenants with the included playbooks:
+
+```sh
+ansible-playbook tests/setup/setup-sso.yml -e sso_admin_password=secret -e sso_hostname=sso.app.example.test
+```
+
+## 3scale on-premise
+
+Create a project in an OpenShift cluster:
+
+```sh
+oc new-project 3scale-ci-23 --display-name="3scale CI 2.3"
+```
+
+Deploy 3scale AMP 2.3:
+
+```sh
+oc create -f https://raw.githubusercontent.com/3scale/3scale-amp-openshift-templates/2.3.0.GA/amp/amp.yml
+oc new-app --template=3scale-api-management -p WILDCARD_DOMAIN=3scale-ci-23.app.example.test -p WILDCARD_POLICY=Subdomain
+```
+
+Create two tenants: `pool1` and `pool2` and expose them:
+
+```sh
+oc expose svc/system-provider --hostname pool1-admin.3scale-ci-23.app.example.test --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }' --name=pool1-admin
+oc expose svc/system-provider --hostname pool2-admin.3scale-ci-23.app.example.test --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }' --name=pool2-admin
+```
+
+Provision the Red Hat SSO tenants with the included playbooks:
+
+```sh
+ansible-playbook tests/setup/setup-sso.yml -e sso_admin_password=secret -e sso_hostname=sso.app.example.test
+```
+
+Delete the wildcard route and recreate it with two more routes:
+
+```sh
+oc delete route apicast-wildcard-router
+oc expose svc/apicast-wildcard-router --wildcard-policy=Subdomain --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }'  --hostname=apicast-wildcard.pool1.3scale-ci-23.app.example.test --name=pool1-apicast-wildcard-router
+oc expose svc/apicast-wildcard-router --wildcard-policy=Subdomain --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }'  --hostname=apicast-wildcard.pool2.3scale-ci-23.app.example.test --name=pool2-apicast-wildcard-router
+oc expose svc/apicast-wildcard-router --wildcard-policy=Subdomain --overrides='{ "apiVersion": "route.openshift.io/v1", "kind": "Route", "spec": { "tls": { "insecureEdgeTerminationPolicy": "Allow", "termination": "edge" } } }'  --hostname=apicast-wildcard.3scale-ci-23.app.example.test --name=apicast-wildcard-router
+```
+
+Do the same with other versions of 3scale.

tests/setup/common/create-sso-client.yml

@@ -0,0 +1,77 @@
+---
+
+- name: Create the SSO client
+  keycloak_client: 
+    auth_keycloak_url: 'https://{{ sso_hostname }}/auth'
+    auth_password: '{{ sso_admin_password }}'
+    auth_realm: '{{ sso_admin_realm }}'
+    auth_username: '{{ sso_admin_username }}'
+    name: '{{ item.client_id }}'
+    description: 'Zync account for 3scale instance {{ item.admin_portal }}'
+    realm: '{{ item.realm }}'
+    enabled: true
+    state: present
+    protocol: openid-connect
+    client_id: '{{ item.client_id }}'
+    secret: '{{ item.client_secret }}'
+    direct_access_grants_enabled: false
+    full_scope_allowed: true
+    implicit_flow_enabled: false
+    public_client: false
+    service_accounts_enabled: true
+    standard_flow_enabled: false
+    validate_certs: no
+  register: create_client_response
+
+- name: Get the service account user tied to the client
+  uri:
+    url: 'https://{{ sso_hostname }}/auth/admin/realms/{{ item.realm }}/clients/{{ client_uuid }}/service-account-user'
+    validate_certs: no
+    headers:
+      Authorization: 'Bearer {{ access_token }}'
+  register: service_account_response
+  changed_when: false
+  vars:
+    client_uuid: '{{ create_client_response.end_state.id }}'
+
+- name: Get the "realm-management" client details
+  keycloak_client: 
+    auth_keycloak_url: 'https://{{ sso_hostname }}/auth'
+    auth_password: '{{ sso_admin_password }}'
+    auth_realm: '{{ sso_admin_realm }}'
+    auth_username: '{{ sso_admin_username }}'
+    realm: '{{ item.realm }}'
+    state: present
+    client_id: realm-management
+    validate_certs: no
+  check_mode: yes
+  register: realm_management_client_response
+
+- name: Get the role description of the "realm-management" client
+  uri:
+    url: 'https://{{ sso_hostname }}/auth/admin/realms/{{ item.realm }}/clients/{{ realm_management_client_uuid }}/roles/manage-clients'
+    validate_certs: no
+    headers:
+      Authorization: 'Bearer {{ access_token }}'
+  register: get_role_response
+  changed_when: false
+  vars:
+    realm_management_client_uuid: '{{ realm_management_client_response.existing.id }}'
+
+- name: Add the 'manage-clients' role mapping to the service account user
+  uri:
+    url: 'https://{{ sso_hostname }}/auth/admin/realms/{{ item.realm }}/users/{{ service_account_uuid }}/role-mappings/clients/{{ realm_management_client_uuid }}'
+    body: 
+    - '{{ manage_clients_role }}'
+    body_format: json
+    method: POST
+    validate_certs: no
+    headers:
+      Authorization: 'Bearer {{ access_token }}'
+    status_code: "204"
+  register: set_role_mapping_response
+  changed_when: set_role_mapping_response.status == 204
+  vars:
+    service_account_uuid: '{{ service_account_response.json.id }}'
+    realm_management_client_uuid: '{{ realm_management_client_response.existing.id }}'
+    manage_clients_role: '{{ get_role_response.json }}'

tests/setup/delete-travis-logs.yml

@@ -0,0 +1,56 @@
+---
+
+- name: Delete the Travis logs of a build
+  hosts: localhost
+  gather_facts: no
+  vars:
+    ansible_connection: local
+    travis_repo: nmasse-itix/threescale-cicd
+    travis_api: https://api.travis-ci.org
+  tasks:
+  - assert:
+      that:
+      - travis_token is defined
+      msg: >
+        Please pass your Travis Token in the 'travis_token' extra var
+
+  - assert:
+      that:
+      - travis_build is defined
+      msg: >
+        Please pass Travis build number in the 'travis_build' extra var
+
+  - name: Find Build
+    uri:
+      url: '{{ travis_api }}/repos/{{ travis_repo }}/builds?number={{ travis_build }}'
+      headers:
+        Authorization: "token {{ travis_token }}"
+    register: find_build_response
+    changed_when: false
+
+  - name: Get Build
+    uri:
+      url: '{{ travis_api }}/repos/{{ travis_repo }}/builds/{{ travis_build_id }}'
+      headers:
+        Authorization: "token {{ travis_token }}"
+    register: get_build_response
+    changed_when: false
+    vars:
+      travis_build_id: '{{ find_build_response.json|json_query(''[0].id'') }}'
+
+  - name: Delete logs
+    uri:
+      url: '{{ travis_api }}/jobs/{{ item }}/log'
+      headers:
+        Authorization: "token {{ travis_token }}"
+      body_format: form-urlencoded
+      body:
+        reason: "Logs removed because it contains sensitive data"
+      method: PATCH
+      status_code: "200,409"
+    register: delete_logs_response
+    changed_when: delete_logs_response.status == 200
+    with_items: '{{ travis_jobs }}'
+    vars:
+      travis_jobs: '{{ get_build_response.json|json_query(''@.matrix[].id'') }}'
+

tests/setup/setup-sso.yml

@@ -0,0 +1,106 @@
+---
+
+- name: Configure Red Hat SSO according to the 3scale inventory file
+  hosts: localhost
+  gather_facts: no
+  vars:
+    ansible_connection: local
+    sso_admin_username: admin
+    sso_admin_realm: master
+    sso_admin_client_id: admin-cli
+  tasks:
+  - assert:
+      that:
+      - sso_admin_password is defined
+      - sso_hostname is defined
+      msg: >
+        Please pass the SSO admin credentials as extra vars
+
+  - set_fact:
+      threescale_inventory: '{{ lookup(''env'', ''THREESCALE_INVENTORY'')|b64decode|from_json }}'
+    when: 'threescale_inventory is not defined and lookup(''env'', ''THREESCALE_INVENTORY'')|length > 0'
+
+  - set_fact:
+      threescale_inventory: '{{ lookup(''file'', ''{{ playbook_dir }}/../3scale-inventory.yaml'')|from_yaml }}'
+    when: 'threescale_inventory is not defined'
+
+  - name: Authenticate to RH-SSO
+    uri:
+      url: 'https://{{ sso_hostname }}/auth/realms/{{ sso_admin_realm }}/protocol/openid-connect/token'
+      body: 'grant_type=password&client_id={{ sso_admin_client_id|urlencode }}&username={{ sso_admin_username|urlencode }}&password={{ sso_admin_password|urlencode }}'
+      method: POST
+      validate_certs: no
+    register: auth_response
+    changed_when: false
+
+  - name: Delete the RH-SSO realm
+    uri:
+      url: 'https://{{ sso_hostname }}/auth/admin/realms/{{ item }}'
+      method: DELETE
+      validate_certs: no
+      headers:
+        Authorization: 'Bearer {{ access_token }}'
+      status_code: "204,404"
+    register: delete_realm_response
+    changed_when: delete_realm_response.status == 204
+    with_items: '{{ realms }}'
+    vars:
+      realms: '{{ threescale_inventory|json_query(''@.*[].sso.realm'')|unique }}'
+      access_token: '{{ auth_response.json.access_token }}'
+
+  - name: Create the RH-SSO realm
+    uri:
+      url: 'https://{{ sso_hostname }}/auth/admin/realms'
+      body: 
+        id: '{{ item }}'
+        enabled: true
+        realm: '{{ item }}'
+        displayName: '{{ item }}'
+        notBefore: 0
+        revokeRefreshToken: false
+        refreshTokenMaxReuse: 0
+        registrationAllowed: false
+        registrationEmailAsUsername: false
+        rememberMe: false
+        verifyEmail: false
+        loginWithEmailAllowed: false
+        duplicateEmailsAllowed: false
+        resetPasswordAllowed: false
+        bruteForceProtected: false
+        permanentLockout: false
+        roles:
+          realm: []
+        defaultRoles: []
+        requiredCredentials:
+        - password
+        scopeMappings: []
+        editUsernameAllowed: false
+        accessTokenLifespanForImplicitFlow: 86400 # 1 day
+        accessTokenLifespan: 86400 # 1 day
+        accessCodeLifespanUserAction: 86400 # 1 day
+        accessCodeLifespanLogin: 86400 # 1 day
+        accessCodeLifespan: 86400 # 1 day
+        ssoSessionIdleTimeout: 86400 # 1 day
+        ssoSessionMaxLifespan: 86400 # 1 day
+        offlineSessionIdleTimeout: 2592000 # 30 days
+        actionTokenGeneratedByAdminLifespan: 86400 # 1 day
+        actionTokenGeneratedByUserLifespan: 86400 # 1 day
+        sslRequired: none
+      body_format: json
+      method: POST
+      validate_certs: no
+      headers:
+        Authorization: 'Bearer {{ access_token }}'
+      status_code: "201,409"
+    register: create_realm_response
+    changed_when: create_realm_response.status == 201
+    with_items: '{{ realms }}'
+    vars:
+      realms: '{{ threescale_inventory|json_query(''@.*[].sso.realm'')|unique }}'
+      access_token: '{{ auth_response.json.access_token }}'
+
+  - include_tasks: "common/create-sso-client.yml"
+    with_items: '{{ clients }}'
+    vars:
+      clients: '{{ threescale_inventory|json_query(''@.*[].{client_id: sso.client_id, client_secret: sso.client_secret, realm: sso.realm, admin_portal: threescale.admin_portal }'')|unique }}'
+      access_token: '{{ auth_response.json.access_token }}'

tests/3scale-saas-with-hosted-apicast-apikey.yml → tests/test-cases/01-beer-catalog-apikey.yml

@@ -1,15 +1,17 @@
 ---
 
-- name: Deploy the Beer Catalog API to a 3scale SaaS instance, with hosted APIcasts
+- name: Deploy the Beer Catalog API with API Key
   hosts: threescale
   gather_facts: no
   vars:
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/beer-catalog-api.json'
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/beer-catalog-api.json'
     threescale_cicd_openapi_file_format: 'JSON'
     threescale_cicd_api_backend_hostname: echo-api.3scale.net
     threescale_cicd_openapi_smoketest_operation: GET_beer
-    threescale_cicd_api_base_system_name: beer_catalog_apikey
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Test a first deployment
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/3scale-saas-with-hosted-apicast-oidc.yml → tests/test-cases/02-echo-api-oidc.yml

@@ -1,12 +1,14 @@
 ---
 
-- name: Deploy the Echo API to a 3scale SaaS instance, with hosted APIcasts and OpenID Connect
+- name: Deploy the Echo API with OpenID Connect
   hosts: threescale
   gather_facts: no
   vars:
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/echo-api-oidc.yaml'
-    threescale_cicd_api_base_system_name: echo_api_oidc
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/echo-api-oidc.yaml'
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Test a first deployment
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/3scale-saas-with-hosted-apicast-multi-environment.yml → tests/test-cases/03-multi-environment.yml

@@ -1,15 +1,17 @@
 ---
 
-- name: Deploy the Beer Catalog API to a 3scale SaaS instance in multi environment
+- name: Deploy the Beer Catalog API in multi environment
   hosts: threescale
   gather_facts: no
   vars:
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/beer-catalog-api.json'
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/beer-catalog-api.json'
     threescale_cicd_openapi_file_format: 'JSON'
     threescale_cicd_api_backend_hostname: echo-api.3scale.net
     threescale_cicd_openapi_smoketest_operation: GET_beer
-    threescale_cicd_api_base_system_name: beer_catalog_envs
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+
   # Deploy in DEV
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/test-cases/04-one-gateway.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Deploy the Beer Catalog API with only one gateway
+  hosts: threescale
+  gather_facts: no
+  vars:
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/beer-catalog-api.json'
+    threescale_cicd_openapi_file_format: 'JSON'
+    threescale_cicd_api_backend_hostname: echo-api.3scale.net
+    threescale_cicd_openapi_smoketest_operation: GET_beer
+    # Both Public Base URL are the same
+    threescale_cicd_apicast_sandbox_endpoint: '{{ threescale_cicd_apicast_production_endpoint }}'
+  tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
+  # Deploy the service
+  - import_role:
+      name: 'nmasse-itix.threescale-cicd'
+  # Delete the service
+  - import_role:
+      name: 'nmasse-itix.threescale-cicd'
+      tasks_from: 'cleanup'

tests/3scale-saas-with-hosted-apicast-with-basePath.yml → tests/test-cases/05-echo-api-with-basePath.yml

@@ -1,12 +1,14 @@
 ---
 
-- name: Deploy the Echo API to a 3scale SaaS instance, with hosted APIcasts
+- name: Deploy the Echo API with a basepath
   hosts: threescale
   gather_facts: no
   vars:
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/echo-api-with-basePath.yaml'
-    threescale_cicd_api_base_system_name: echo_api_with_basepath
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/echo-api-with-basePath.yaml'
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Test a first deployment
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/3scale-saas-with-hosted-apicast-with-cors-policy.yml → tests/test-cases/06-echo-api-with-cors-policy.yml

@@ -1,14 +1,16 @@
 ---
 
-- name: Deploy the Echo API to a 3scale SaaS instance, with hosted APIcasts and CORS
+- name: Deploy the Echo API with CORS
   hosts: threescale
   gather_facts: no
   vars:
     threescale_cicd_openapi_smoketest_operation: Echo
     threescale_cicd_apicast_policies_cors: yes
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/echo-api-bare.yaml'
-    threescale_cicd_api_base_system_name: echo_api_cors
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/echo-api-bare.yaml'
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Deploy the service
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/3scale-saas-with-hosted-apicast-with-smoketest-in-extra-vars.yml → tests/test-cases/07-echo-api-with-smoketest-in-extra-vars.yml

@@ -1,13 +1,15 @@
 ---
 
-- name: Deploy the Echo API to a 3scale SaaS instance, with hosted APIcasts and smoketests in extra vars
+- name: Deploy the Echo API with smoketests in extra vars
   hosts: threescale
   gather_facts: no
   vars:
     threescale_cicd_openapi_smoketest_operation: Echo
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/echo-api-bare.yaml'
-    threescale_cicd_api_base_system_name: echo_api_bare_extra
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/echo-api-bare.yaml'
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Deploy the service
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/3scale-saas-with-hosted-apicast-without-smoketest.yml → tests/test-cases/08-echo-api-without-smoketest.yml

@@ -1,13 +1,15 @@
 ---
 
-- name: Deploy the Echo API to a 3scale SaaS instance, with hosted APIcasts and no smoketests
+- name: Deploy the Echo API with no smoketests
   hosts: threescale
   gather_facts: no
   vars:
     # There is no "threescale_cicd_openapi_smoketest_operation" variable
-    threescale_cicd_openapi_file: '{{ playbook_dir }}/api/echo-api-bare.yaml'
-    threescale_cicd_api_base_system_name: echo_api_no_smoketests
+    threescale_cicd_openapi_file: '{{ playbook_dir }}/api-contracts/echo-api-bare.yaml'
   tasks:
+  - name: Generate a random system_name for this test run
+    import_tasks: "common/random-system-name.yml"
+    
   # Deploy the service
   - import_role:
       name: 'nmasse-itix.threescale-cicd'

tests/test-cases/common/random-system-name.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Generate a random system_name for this test run
+  set_fact:
+    threescale_cicd_api_base_system_name: 'testcase_{{ lookup(''password'', ''/dev/null length=12 chars=hexdigits'')|lower }}'

tests/test-cases/roles/nmasse-itix.threescale-cicd

@@ -0,0 +1 @@
+../../..

tests/write-inventory-files.yml

@@ -11,24 +11,13 @@
     when: 'threescale_inventory is not defined and lookup(''env'', ''THREESCALE_INVENTORY'')|length > 0'
 
   - set_fact:
-      threescale_inventory: '{{ lookup(''file'', ''3scale-inventory.yaml'')|from_yaml }}'
+      threescale_inventory: '{{ lookup(''file'', ''{{ playbook_dir }}/3scale-inventory.yaml'')|from_yaml }}'
     when: 'threescale_inventory is not defined'
 
-  - name: Find the Jinja2 templates to process
-    find:
-      paths: '{{ playbook_dir }}'
-      patterns: '*.j2'
-      recurse: no
-    register: templates
-
-  - name: Select the target CI platform using the THREESCALE_POOL environment variable
-    set_fact:
-      threescale_inventory: '{{ threescale_inventory[env|int] }}'
-    vars:
-      env: '{{ lookup(''env'', ''THREESCALE_POOL'') }}'
-
   - name: Process the Jinja2 templates
     template:
-      src: '{{ item }}'
-      dest: '{{ item|replace(".j2", "") }}'
-    with_items: '{{ templates.files|json_query(''[*].path'') }}'
+      src: '{{ playbook_dir }}/inventory.j2'
+      dest: '{{ playbook_dir }}/environments/{{ item.key }}'
+    with_dict: '{{ threescale_inventory }}'
+    vars:
+      test_environment: '{{ item.value }}'