From 74c4147cbc0808a29b6651b5ecbd14de4e61e504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= Date: Mon, 18 Nov 2019 14:43:02 +0100 Subject: [PATCH 1/3] 2019-11-18 update --- ...adm-release-mirror-between-environments.md | 164 ++++++++++++++++++ .../blog/ansible-add-prefix-suffix-to-list.md | 42 +++++ .../check-ansible-version-number-playbook.md | 20 +++ .../pull.yaml | 63 +++++++ 4 files changed, 289 insertions(+) create mode 100644 content/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments.md create mode 100644 content/blog/ansible-add-prefix-suffix-to-list.md create mode 100644 content/blog/check-ansible-version-number-playbook.md create mode 100644 static/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments/pull.yaml diff --git a/content/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments.md b/content/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments.md new file mode 100644 index 0000000..1539790 --- /dev/null +++ b/content/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments.md @@ -0,0 +1,164 @@ +--- +title: "Airgap OpenShift Installation: move the registry created using oc adm release mirror between environments" +date: 2019-11-18T00:00:00+02:00 +opensource: +- Ansible +- OpenShift +- Skopeo +--- + +Some customers, especially large banks, have very tight security requirements. +Most of them enforce a complete disconnection of their internal networks from the Internet. + +When installing OpenShift in such environments (this is named "disconnected" or ["airgap" installation](http://www.cloud-computing-koeln.de/openshift-4-2-disconnected-install/)), all the OpenShift images have to be fetched (thanks to [oc adm release mirror](https://docs.openshift.com/container-platform/4.2/installing/installing_restricted_networks/installing-restricted-networks-preparations.html)) in a dedicated registry from a bastion host that is both on the internal network and on the Internet. + +However, for some customers this is not secure enough. Most of them would rather download all the images locally (using *oc adm release mirror* ?), transport them on a removable media to the internal network and provision the target registry. + +As described in this article, [skopeo](https://github.com/nmasse-itix/OpenShift-Examples/blob/master/Using-Skopeo/README.md) and Ansible can be a nice complement of *oc adm release mirror* to achieve this setup. Let's discover how! + +The rest of this guide assumes that you followed [the official documentation](https://docs.openshift.com/container-platform/4.2/installing/installing_restricted_networks/installing-restricted-networks-preparations.html) and fetched on the bastion node all the required images in a dedicated registry using *oc adm release mirror*. + +First, you will need a token that has administrative privileges on this registry. + +```raw +$ oc whoami -t +AZERTYUIOPQSDFGHJKLMWXCVBN1234567890azertyu +``` + +Store it somewhere for later use. + +```sh +export TOKEN=$(oc whoami -t) +``` + +Confirm your token can fetch the registry catalog. + +```raw +$ curl -s https://docker-registry.default.svc:5000/v2/_catalog -H "Authorization: Bearer $TOKEN" |jq . + +{ + "repositories": [ + "openshift/httpd", + "openshift/java", + "openshift/jenkins", + [...] + "openshift/php", + "openshift/python" + ] +} +``` + +As you may have guessed, this is the list of all the images provisioned by the *oc adm release mirror* command that we will need to export using skopeo. +But before doing so, we need to get the list of all the tags of each image. + +Hopefully, there is also an API for this. + +```raw +$ curl -s https://docker-registry.default.svc:5000/v2/openshift/php/tags/list -H "Authorization: Bearer $TOKEN" |jq . + +{ + "name": "openshift/php", + "tags": [ + "latest", + "5.5", + "5.6", + "7.0", + "7.1" + ] +} +``` + +As an example, to export *openshift/php:5.5* from the *docker-registry.default.svc:5000* registry to the local filesystem (in */tmp/oci_registry*), you could use: + +```sh +skopeo --insecure-policy copy --src-tls-verify=false --src-creds=admin:$TOKEN docker://docker-registry-default.app.itix.fr/openshift/php:5.5 oci:/tmp/oci_registry:openshift/php:5.5 +``` + +We now have the basis to build the Ansible playbook that will dump the registry created by *oc adm release mirror* to the filesystem. + +The first step of this playbook would be to fetch the registry catalog. +There is nothing fancy here, just a plain application of the [uri module](https://docs.ansible.com/ansible/latest/modules/uri_module.html). + +```yaml +- hosts: localhost + gather_facts: no + vars: + registry: docker-registry.default.svc:5000 + validate_certs: false + tasks: + - name: Fetch the catalog of the docker registry + uri: + url: 'https://{{ registry }}/v2/_catalog' + headers: + Authorization: Bearer {{ token }} + status_code: 200 + return_content: yes + validate_certs: '{{ validate_certs }}' + register: catalog +``` + +The next step is to iterate on this catalog to fetch the tags of each image. +The [url lookup](https://docs.ansible.com/ansible/latest/plugins/lookup/url.html) plugin is used to query the registry API. +The image name is added in front of each tag (to construct the full image name: `image:tag`) using the [cartesian product filter](https://docs.ansible.com/ansible/latest/user_guide/playbooks_filters.html#product-filters). + +```yaml + - name: Construct a list of all available images + set_fact: + images: > + {{ images|default([]) + new_images }} + vars: + image_tags: > + {{ (lookup("url", "https://"~registry~"/v2/"~item~"/tags/list", + headers={"Authorization": "Bearer "~token}, + validate_certs=validate_certs)|from_json).tags }} + new_images: > + {{ [item] | product(image_tags) | map('join', ':') | list }} + loop: '{{ catalog.json.repositories }}' + + - debug: + var: images +``` + +When using the url lookup plugin on MacOS, you might need to set the *OBJC_DISABLE_INITIALIZE_FORK_SAFETY* environment variable as explained in [#32499](https://github.com/ansible/ansible/issues/32499). + +```sh +export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES +``` + +Finally, skopeo is called to download each image to */tmp/oci_registry*. + +```yaml +- hosts: localhost + gather_facts: no + vars: + target: /tmp/oci_registry + [...] + + tasks: + + [...] + + - name: Downloading OpenShift images... + command: skopeo --insecure-policy copy --src-tls-verify={{ validate_certs|bool|ternary('true','false') }} --src-creds=admin:{{ token }} 'docker://{{ registry }}/{{ item }}' 'oci:{{ target }}:{{ item }}' + with_items: '{{ images }}' +``` + +The complete playbook [is available here](pull.yaml) and can be run as follow. + +```sh +ansible-playbook pull.yaml -e token=$TOKEN +``` + +This will dump the registry at *docker-registry.default.svc:5000* to */tmp/oci_registry*. +If you want to target another registry or store the images somewhere else, you can pass the *registry* or *target* extra variables. + +```sh +ansible-playbook pull.yaml -e token=$TOKEN -e registry=docker-registry-default.app.openshift.test -e target=/tmp/oci_registry +``` + +The images are stored as an OCI registry whose format is standardized. +It can be moved somewhere else, using a removable media for instance. + +How this OCI registry can be imported in the target registry is a subject for another article. + +Stay tuned. diff --git a/content/blog/ansible-add-prefix-suffix-to-list.md b/content/blog/ansible-add-prefix-suffix-to-list.md new file mode 100644 index 0000000..52c007c --- /dev/null +++ b/content/blog/ansible-add-prefix-suffix-to-list.md @@ -0,0 +1,42 @@ +--- +title: "Ansible: Add a prefix or suffix to all items of a list" +date: 2019-11-18T00:00:00+02:00 +opensource: +- Ansible +--- + +Recently, in [one of my Ansible playbooks](../airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments) I had to prefix all items of a list with a chosen string. + +Namely, from the following list: + +```python +[ "bar", "bat", "baz" ] +``` + +I want to have: + +```python +[ "foobar", "foobat", "foobaz" ] +``` + +The recipe I used to add a prefix to all items of a list is: + +```yaml +- debug: + var: result + vars: + prefix: foo + a_list: [ "bar", "bat", "baz" ] + result: "{{ [prefix] | product(a_list) | map('join') | list }}" +``` + +If you need to add a suffix to all items of a list instead, you can use: + +```yaml +- debug: + var: result + vars: + suffix: foo + a_list: [ "bar", "bat", "baz" ] + result: "{{ a_list | product([suffix]) | map('join') | list }}" +``` diff --git a/content/blog/check-ansible-version-number-playbook.md b/content/blog/check-ansible-version-number-playbook.md new file mode 100644 index 0000000..d8f25f5 --- /dev/null +++ b/content/blog/check-ansible-version-number-playbook.md @@ -0,0 +1,20 @@ +--- +title: "Check the Ansible version number in a playbook" +date: 2019-11-18T00:00:00+02:00 +opensource: +- Ansible +--- + +My Ansible playbooks sometimes use features that are available only in a very recent versions of Ansible. + +To prevent unecessary troubles to the team mates that will execute them, I like to add a task at the very beginning of my playbooks to check the Ansible version number and abort if the requirements are not met. + +```yaml +- name: Verify that Ansible version is >= 2.4.6 + assert: + that: "ansible_version.full is version_compare('2.4.6', '>=')" + msg: >- + This module requires at least Ansible 2.4.6. The version that comes + with RHEL and CentOS by default (2.4.2) has a known bug that prevent + this role from running properly. +``` diff --git a/static/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments/pull.yaml b/static/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments/pull.yaml new file mode 100644 index 0000000..81514a7 --- /dev/null +++ b/static/blog/airgap-openshift-installation-move-registry-created-using-oc-adm-release-mirror-between-environments/pull.yaml @@ -0,0 +1,63 @@ +--- +- name: Pull a complete Docker registry + hosts: localhost + become: no + gather_facts: no + vars: + registry: docker-registry.default.svc:5000 + validate_certs: false + target: /tmp/oci_registry + tasks: + + - name: Verify that Ansible version is >= 2.9 + assert: + that: "ansible_version.full is version_compare('2.9.0', '>=')" + msg: >- + This playbook uses the 'headers' property of the 'url' filter and thus + requires Ansible 2.9 + + - assert: + that: + - token is defined + msg: > + Please pass an administrative token to connect to '{{ registry }}' + as an extra var (-e token=bla.bla.bla). + + - name: Fetch the catalog of the docker registry + uri: + url: 'https://{{ registry }}/v2/_catalog' + headers: + Authorization: Bearer {{ token }} + status_code: 200 + return_content: yes + validate_certs: '{{ validate_certs }}' + register: catalog + + - name: Construct a list of all available images + set_fact: + images: > + {{ images|default([]) + new_images }} + vars: + image_tags: > + {{ (lookup("url", "https://" ~ registry ~ "/v2/" ~ item ~ "/tags/list", + headers={"Authorization": "Bearer " ~ token}, + validate_certs=validate_certs)|from_json).tags }} + new_images: > + {{ [item] | product(image_tags) | map('join', ':') | list }} + loop: '{{ catalog.json.repositories }}' + + - debug: + var: images + + - pause: + prompt: "Would pull {{ images|length }} images from {{ registry }}. Continue?" + + - name: Create a directory to hold the images + file: + path: '{{ target }}' + state: directory + register: mkdir + + - name: Downloading OpenShift images... + command: skopeo --insecure-policy copy --src-tls-verify={{ validate_certs|bool|ternary('true','false') }} --src-creds=admin:{{ token }} 'docker://{{ registry }}/{{ item }}' 'oci:{{ target }}:{{ item }}' + with_items: '{{ images }}' From da02aee8f01fcdb395a553802f2c7aa38fbd490b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= Date: Mon, 18 Nov 2019 15:38:17 +0100 Subject: [PATCH 2/3] 2019-11-18 update --- .../feed-url-drupal-wordpress-wix-youtube.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 content/blog/feed-url-drupal-wordpress-wix-youtube.md diff --git a/content/blog/feed-url-drupal-wordpress-wix-youtube.md b/content/blog/feed-url-drupal-wordpress-wix-youtube.md new file mode 100644 index 0000000..88c9c24 --- /dev/null +++ b/content/blog/feed-url-drupal-wordpress-wix-youtube.md @@ -0,0 +1,16 @@ +--- +title: "Feed URLs for the most common CMS: Drupal, Wordpress, WiX and YouTube" +date: 2019-11-18T00:00:00+02:00 +--- + +If like me you are using [an RSS reader](../deploying-miniflux-openshift/) to stay informed, there is nothing more frustrating than reading a website that does not advertise an RSS feed. + +But since most website are based on commonly found CMS, it is highly probable the RSS feeds are there, just not advertised. + +Here are the URL patterns for the most common CMS on the market: + +- **Wordpress**: `/feed/` or `/?feed=rss2` +- **Drupal**: `/rss.xml` +- **Wix**: `/blog-feed.xml` +- **YouTube Channel**: `https://www.youtube.com/feeds/videos.xml?channel_id=` + From 3a1d31f017b9af032322bf8a91b6dc5ea0721a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= Date: Fri, 22 Nov 2019 09:38:30 +0100 Subject: [PATCH 3/3] add robot.txt --- config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.toml b/config.toml index bffb34a..38d9e9d 100644 --- a/config.toml +++ b/config.toml @@ -5,6 +5,7 @@ author = "Nicolas Massé" theme = "cocoa" pygmentsUseClasses = true pygmentsCodefences = true +enableRobotsTXT = true [taxonomies] opensource = "opensource"