diff --git a/cookbooks/lego/Makefile b/cookbooks/lego/Makefile index 54052a7..1c85a71 100644 --- a/cookbooks/lego/Makefile +++ b/cookbooks/lego/Makefile @@ -6,6 +6,10 @@ PROJECT_UID = 10023 PROJECT_GID = 10000 +# .path units are not picked up by common.mk's wildcard (only .service/.target/ +# .timer/.mount), so install it explicitly. It is started by lego.target. +TARGET_FILES += $(TARGET_CHROOT)/etc/systemd/system/lego-hooks.path + # Include common Makefile include ../../scripts/common.mk diff --git a/cookbooks/lego/README.md b/cookbooks/lego/README.md index a922376..e4e3182 100644 --- a/cookbooks/lego/README.md +++ b/cookbooks/lego/README.md @@ -6,16 +6,54 @@ Lego is a Let's Encrypt/ACME client started as a Podman Quadlet. It handles auto This cookbook: -- Runs an initial certificate fetch via **lego-run.service** when no certificates exist. -- Schedules automatic certificate renewal via **lego-renew.timer**. -- Stores certificates with secure permissions (umask 0077). -- Supports renewal hooks to reload dependent services when certificates are renewed. +- Runs Lego v5 with a configuration file (`/etc/quadlets/lego/lego.yaml`) describing the accounts, challenges and certificates. +- Obtains the missing certificates and renews the expiring ones via **lego.service**, started at boot and daily by **lego.timer**. +- Stores certificates with secure permissions (0600, owned by the `lego` user). +- Runs renewal hooks (`/etc/quadlets/lego/renew-hooks.d/*.sh`) to reload dependent services when certificates are obtained or renewed. ## Prerequisites -- Configuration file `/etc/quadlets/lego/config.env` must exist with ACME configuration. +- Configuration file `/etc/quadlets/lego/lego.yaml` must exist. See [config/examples/lego.yaml](config/examples/lego.yaml) and the [Lego configuration file reference](https://go-acme.github.io/lego/references/ref-file/). - DNS or HTTP challenge must be properly configured. +## Certificate naming convention + +Lego v5 names the certificate files after the certificate ID in the configuration file: `/var/lib/quadlets/lego/certificates/.{crt,key,issuer.crt,pem,json}`. + +The other cookbooks (vsftpd, smtprelay, etc.) expect a certificate named **default**: + +```yaml +certificates: + default: + domains: + - ftp.example.tld +``` + +## How hooks work + +1. Lego obtains or renews a certificate and calls its deploy hook (`/etc/quadlets/lego/deploy-hook.sh`, inside the container). +2. The deploy hook creates a `.renewed` file next to the certificate. +3. **lego-hooks.path** notices the `.renewed` file and starts **lego-hooks.service**. +4. **lego-hooks.service** runs every script of `/etc/quadlets/lego/renew-hooks.d/` (on the host, as root) and removes the `.renewed` files. + +Other cookbooks drop their hook in `other/lego/.sh`: it is installed in `/etc/quadlets/lego/renew-hooks.d/` automatically. + +## Using a custom ACME server + +To use an ACME server whose TLS certificate is signed by a private CA (Pebble, Step CA, etc.), set the `server` of the account in `lego.yaml` and add a drop-in to trust the CA: + +```ini +# /etc/containers/systemd/lego.container.d/custom-ca.conf +[Container] +Environment=LEGO_CA_CERTIFICATES=/etc/lego/my-ca.pem +``` + +The CA certificate goes in `/etc/quadlets/lego/my-ca.pem` (`/etc/quadlets/lego` is mounted on `/etc/lego` in the container). + +## Development mode + +In development mode (`make fcos-vm` or `make install`, which install the examples), the **lego-dev.conf** drop-in replaces Lego with a script that generates a self-signed certificate for `localhost`, named `default`. + ## Usage In a separate terminal, follow the logs. @@ -30,7 +68,7 @@ Install the Podman Quadlets and fetch the initial certificate. sudo make clean install ``` -You should see the **lego-run.service** fetching a certificate from Let's Encrypt. +You should see the **lego.service** fetching a certificate from Let's Encrypt. The certificate will be stored in `/var/lib/quadlets/lego/certificates/`. Check the certificate: @@ -39,12 +77,12 @@ Check the certificate: sudo ls -la /var/lib/quadlets/lego/certificates/ ``` -The **lego-renew.timer** will periodically check and renew the certificate before expiration. +The **lego.timer** will periodically check and renew the certificate before expiration. To manually trigger a renewal check: ```sh -sudo systemctl start lego-renew.service +sudo systemctl start lego.service ``` Restart the **lego.target** unit. @@ -59,8 +97,29 @@ Finally, remove the quadlets, their configuration and their data. sudo make uninstall clean ``` +## Upgrading from Lego v4 + +Lego v5 uses a configuration file and a new storage layout. +The simplest upgrade path is to obtain new certificates. + +```sh +sudo systemctl stop lego.target lego-renew.timer +sudo rm -rf /var/lib/quadlets/lego/* +sudo rm -rf /etc/quadlets/lego/config.env /etc/quadlets/lego/renew-hook +sudo rm -rf /etc/containers/systemd/lego-run.container /etc/containers/systemd/lego-renew.container \ + /etc/containers/systemd/lego-run.container.d /etc/containers/systemd/lego-renew.container.d \ + /etc/systemd/system/lego-renew.timer /etc/systemd/system/lego-renew-hooks.service +sudo make install +``` + +Then, write `/etc/quadlets/lego/lego.yaml` (with a certificate named `default`) and start Lego. + +```sh +sudo systemctl start lego.target +``` + ## Integration tests ```sh -sudo make test +sudo make pytest ``` diff --git a/cookbooks/lego/config/deploy-hook.sh b/cookbooks/lego/config/deploy-hook.sh new file mode 100755 index 0000000..504065a --- /dev/null +++ b/cookbooks/lego/config/deploy-hook.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# This hook runs inside the Lego container (busybox sh) after a certificate +# has been obtained or renewed. It flags the certificate as renewed so that +# lego-hooks.path triggers the renewal hooks on the host. +set -eu +touch "${LEGO_HOOK_CERT_PATH%.*}.renewed" diff --git a/cookbooks/lego/config/examples/config.env b/cookbooks/lego/config/examples/config.env deleted file mode 100644 index 59d0915..0000000 --- a/cookbooks/lego/config/examples/config.env +++ /dev/null @@ -1,3 +0,0 @@ -LEGO_GLOBAL_ARGS=-a -m nicolas.masse@itix.fr -d changeme.example.tld --http -LEGO_RUN_ARGS= -LEGO_RENEW_ARGS=--days 30 --renew-hook=/etc/lego/renew-hook/flag-as-renewed.sh diff --git a/cookbooks/lego/config/examples/lego-dev.sh b/cookbooks/lego/config/examples/lego-dev.sh index 8a5620f..63a7e74 100755 --- a/cookbooks/lego/config/examples/lego-dev.sh +++ b/cookbooks/lego/config/examples/lego-dev.sh @@ -2,18 +2,24 @@ set -Eeuo pipefail -# In development mode, it is not possible to get a certificate from Let's Encrypt, so we just create a self-signed certificate for localhost, so that other services can still use it. -mkdir -p /var/lib/quadlets/lego/certificates -if [ -f /var/lib/quadlets/lego/certificates/localhost.crt ] && [ -f /var/lib/quadlets/lego/certificates/localhost.key ]; then - renewal="yes" -else - renewal="no" +# In development mode, it is not possible to get a certificate from Let's Encrypt, +# so we just create a self-signed certificate for localhost, so that other services can still use it. +# The certificate is named "default", like the certificate declared in the example lego.yaml. +CERT_DIR=/var/lib/quadlets/lego/certificates +CERT_NAME=default + +if [ -f "$CERT_DIR/$CERT_NAME.crt" ] && [ -f "$CERT_DIR/$CERT_NAME.key" ] && openssl x509 -in "$CERT_DIR/$CERT_NAME.crt" -noout -checkend $((30 * 86400)) &>/dev/null; then + echo "Self-signed certificate is still valid for more than 30 days." + exit 0 fi echo "Generating self-signed certificate for localhost..." -openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /var/lib/quadlets/lego/certificates/localhost.key -out /var/lib/quadlets/lego/certificates/localhost.crt -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost" +install -d -m 0700 -o 10023 -g 10000 "$CERT_DIR" +( + umask 0077 + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout "$CERT_DIR/$CERT_NAME.key" -out "$CERT_DIR/$CERT_NAME.crt" -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost" +) +chown 10023:10000 "$CERT_DIR/$CERT_NAME.key" "$CERT_DIR/$CERT_NAME.crt" -if [[ "$renewal" == "yes" ]]; then - echo "Flagging certificate as renewed..." - touch /var/lib/quadlets/lego/certificates/localhost.renewed -fi +echo "Flagging certificate as renewed..." +install -m 0600 -o 10023 -g 10000 /dev/null "$CERT_DIR/$CERT_NAME.renewed" diff --git a/cookbooks/lego/config/examples/lego.yaml b/cookbooks/lego/config/examples/lego.yaml new file mode 100644 index 0000000..1a52ac9 --- /dev/null +++ b/cookbooks/lego/config/examples/lego.yaml @@ -0,0 +1,51 @@ +# Lego v5 configuration file. +# Reference: https://go-acme.github.io/lego/references/ref-file/ +# +# Heads up! This file is readable by the lego user (uid 10023). +# If you use a DNS-01 challenge, put the DNS provider credentials in a +# separate file (see "envFile" below) with restrictive permissions. + +# Where accounts and certificates are stored (inside the container). +# Mapped to /var/lib/quadlets/lego on the host. +storage: /var/lib/lego + +# Logs go to the systemd journal: no colors. +log: + format: text + +accounts: + default: + # Let's Encrypt staging: https://acme-staging-v02.api.letsencrypt.org/directory + server: https://acme-v02.api.letsencrypt.org/directory + email: changeme@example.tld + acceptsTermsOfService: true + +challenges: + http: + http: + address: ":80" + # Example of a DNS-01 challenge: + # + # dns: + # dns: + # provider: gandiv5 + # envFile: /etc/lego/dns.env + +certificates: + # The certificate ID ("default") is the name of the files in /var/lib/quadlets/lego/certificates. + # Other cookbooks (vsftpd, smtprelay, etc.) expect a certificate named "default". + default: + account: default + challenge: http + domains: + - changeme.example.tld + renew: + # The lego.timer unit already adds a random delay. + disableRandomSleep: true + +hooks: + # Flag certificates as renewed so that the renewal hooks of other cookbooks are triggered. + # The script is interpreted by /bin/sh because SELinux forbids the container + # to execute files from /etc/quadlets/lego (mounted read-only, not relabeled). + deploy: + command: /bin/sh /etc/lego/deploy-hook.sh diff --git a/cookbooks/lego/config/renew-hook/flag-as-renewed.sh b/cookbooks/lego/config/renew-hook/flag-as-renewed.sh deleted file mode 100755 index 2def757..0000000 --- a/cookbooks/lego/config/renew-hook/flag-as-renewed.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -set -Eeuo pipefail -touch ${LEGO_CERT_PATH%.*}.renewed diff --git a/cookbooks/lego/dropins/lego-renew.container.d/lego-dev.conf b/cookbooks/lego/dropins/lego-renew.container.d/lego-dev.conf deleted file mode 100644 index b29053f..0000000 --- a/cookbooks/lego/dropins/lego-renew.container.d/lego-dev.conf +++ /dev/null @@ -1,7 +0,0 @@ -[Container] -# When in development mode, it is not possible to renew a certificate from Let's Encrypt, so we just skip the entrypoint. -Entrypoint=/bin/true - -[Service] -# And we update the self-signed certificate and flag it as renewed. -ExecStartPost=/etc/quadlets/lego/lego-dev.sh diff --git a/cookbooks/lego/dropins/lego-run.container.d/lego-dev.conf b/cookbooks/lego/dropins/lego.container.d/lego-dev.conf similarity index 100% rename from cookbooks/lego/dropins/lego-run.container.d/lego-dev.conf rename to cookbooks/lego/dropins/lego.container.d/lego-dev.conf diff --git a/cookbooks/lego/lego-hooks.path b/cookbooks/lego/lego-hooks.path new file mode 100644 index 0000000..ad7f94d --- /dev/null +++ b/cookbooks/lego/lego-hooks.path @@ -0,0 +1,13 @@ +[Unit] +Description=Watch for certificates obtained or renewed by Lego +Documentation=https://go-acme.github.io/lego/ +PartOf=lego.target + +[Path] +# The Lego deploy hook touches a .renewed file next to each certificate +# it obtains or renews. +PathExistsGlob=/var/lib/quadlets/lego/certificates/*.renewed +Unit=lego-hooks.service + +[Install] +WantedBy=lego.target diff --git a/cookbooks/lego/lego-renew-hooks.service b/cookbooks/lego/lego-hooks.service similarity index 59% rename from cookbooks/lego/lego-renew-hooks.service rename to cookbooks/lego/lego-hooks.service index cd9048a..a2bba5d 100644 --- a/cookbooks/lego/lego-renew-hooks.service +++ b/cookbooks/lego/lego-hooks.service @@ -1,12 +1,10 @@ [Unit] Description=Process Lego certificate renewal hooks -# Lego touches .renewed files when renewed certificates are available +Documentation=https://go-acme.github.io/lego/ +# Lego touches .renewed files when new certificates are available ConditionPathExistsGlob=/var/lib/quadlets/lego/certificates/*.renewed -After=lego-renew.service +After=lego.service [Service] Type=oneshot ExecStart=/etc/quadlets/lego/process-hooks.sh - -[Install] -WantedBy=lego-renew.service diff --git a/cookbooks/lego/lego-renew.container b/cookbooks/lego/lego-renew.container deleted file mode 100644 index 4cae08a..0000000 --- a/cookbooks/lego/lego-renew.container +++ /dev/null @@ -1,53 +0,0 @@ -[Unit] -Description=Lego Let's Encrypt client - Renew a certificate -Documentation=https://go-acme.github.io/lego/ - -# Only start if there are certificates to renew -ConditionPathExistsGlob=/var/lib/quadlets/lego/certificates/*.crt -# and if Lego has been configured ! -ConditionPathExists=/etc/quadlets/lego/config.env - -[Container] -ContainerName=lego-renew - -# No need for root privileges -User=10023 -Group=10000 -AddCapability=CAP_NET_BIND_SERVICE - -# Image -Image=lego.image -AutoUpdate=registry - -# Network configuration -Network=host - -# Environment variables from config -EnvironmentFile=/etc/quadlets/lego/config.env - -# Volume mounts -Volume=/var/lib/quadlets/lego:/.lego:z -Volume=/etc/quadlets/lego/renew-hook:/etc/lego/renew-hook:ro - -# Be safe, set the umask to 0077 so that private keys are not world-readable -PodmanArgs=--umask=0077 - -# Command to run -Exec=$LEGO_GLOBAL_ARGS renew $LEGO_RENEW_ARGS - -[Service] -# If the command fails, don't try to restart it. -# Otherwise, we would hammer the Let's Encrypt servers and possibly get banned. -Restart=no - -# Getting a certificate may take some time -TimeoutStartSec=600 - -# This container is a job - run once to completion -Type=oneshot - -# These environment variables are sourced to be used by systemd in the Exec* commands -EnvironmentFile=/etc/quadlets/lego/config.env - -[Install] -WantedBy=lego.target diff --git a/cookbooks/lego/lego-run.container b/cookbooks/lego/lego.container similarity index 52% rename from cookbooks/lego/lego-run.container rename to cookbooks/lego/lego.container index fddf339..c7b904c 100644 --- a/cookbooks/lego/lego-run.container +++ b/cookbooks/lego/lego.container @@ -1,15 +1,15 @@ [Unit] -Description=Lego Let's Encrypt client - Fetch a certificate +Description=Lego Let's Encrypt client - Obtain or renew the certificates Documentation=https://go-acme.github.io/lego/ +After=network-online.target +Wants=network-online.target PartOf=lego.target -# Only start if no certificates exist yet -ConditionPathExistsGlob=!/var/lib/quadlets/lego/certificates/*.crt -# and if Lego has been configured ! -ConditionPathExists=/etc/quadlets/lego/config.env +# Only start if Lego has been configured ! +ConditionPathExists=/etc/quadlets/lego/lego.yaml [Container] -ContainerName=lego-run +ContainerName=lego # No need for root privileges User=10023 @@ -23,17 +23,16 @@ AutoUpdate=registry # Network configuration Network=host -# Environment variables from config -EnvironmentFile=/etc/quadlets/lego/config.env - # Volume mounts -Volume=/var/lib/quadlets/lego:/.lego:z +Volume=/var/lib/quadlets/lego:/var/lib/lego:z +Volume=/etc/quadlets/lego:/etc/lego:ro # Be safe, set the umask to 0077 so that private keys are not world-readable PodmanArgs=--umask=0077 -# Command to run -Exec=$LEGO_GLOBAL_ARGS run $LEGO_RUN_ARGS +# Lego v5 obtains the missing certificates and renews the expiring ones +# described in its configuration file. It does nothing otherwise. +Exec=--config /etc/lego/lego.yaml [Service] # If the command fails, don't try to restart it. @@ -41,13 +40,10 @@ Exec=$LEGO_GLOBAL_ARGS run $LEGO_RUN_ARGS Restart=no # Getting a certificate may take some time -TimeoutStartSec=600 +TimeoutStartSec=900 # This container is a job - run once to completion Type=oneshot -# These environment variables are sourced to be used by systemd in the Exec* commands -EnvironmentFile=/etc/quadlets/lego/config.env - [Install] WantedBy=lego.target diff --git a/cookbooks/lego/lego.image b/cookbooks/lego/lego.image index c84e1c1..1111b03 100644 --- a/cookbooks/lego/lego.image +++ b/cookbooks/lego/lego.image @@ -3,5 +3,4 @@ Description=podman pull docker.io/goacme/lego Documentation=https://go-acme.github.io/lego/ [Image] -# TODO: Migrate this Quadlet to Lego v5 (https://go-acme.github.io/lego/migration/index.html) -Image=docker.io/goacme/lego:v4 +Image=docker.io/goacme/lego:v5 diff --git a/cookbooks/lego/lego.target b/cookbooks/lego/lego.target index a8ac1d0..90d479a 100644 --- a/cookbooks/lego/lego.target +++ b/cookbooks/lego/lego.target @@ -1,8 +1,11 @@ [Unit] Description=Lego Let's Encrypt client Documentation=https://go-acme.github.io/lego/ -Requires=lego-run.service lego-renew.timer -After=lego-run.service lego-renew.timer +Requires=lego.timer lego-hooks.path +# A failure to obtain or renew a certificate (network issue, ACME server down, etc.) +# must not stop the target. lego.timer will try again later. +Wants=lego.service +After=lego.service lego.timer lego-hooks.path # Allow isolation - can stop/start this target independently AllowIsolate=yes diff --git a/cookbooks/lego/lego-renew.timer b/cookbooks/lego/lego.timer similarity index 67% rename from cookbooks/lego/lego-renew.timer rename to cookbooks/lego/lego.timer index 20efcea..b234efb 100644 --- a/cookbooks/lego/lego-renew.timer +++ b/cookbooks/lego/lego.timer @@ -1,5 +1,5 @@ [Unit] -Description=Lego Let's Encrypt client - Renew a certificate +Description=Lego Let's Encrypt client - Renew the certificates Documentation=https://go-acme.github.io/lego/ PartOf=lego.target @@ -7,6 +7,7 @@ PartOf=lego.target OnCalendar=daily RandomizedDelaySec=1h Persistent=true +Unit=lego.service [Install] WantedBy=lego.target diff --git a/cookbooks/lego/tests/test_01_install.py b/cookbooks/lego/tests/test_01_install.py new file mode 100644 index 0000000..93354e0 --- /dev/null +++ b/cookbooks/lego/tests/test_01_install.py @@ -0,0 +1,281 @@ +import subprocess +import textwrap +import time +import test_quadlet # noqa: F401 +import pytest +import testinfra + +CERT_DIR = "/var/lib/quadlets/lego/certificates" +CERT_PATH = f"{CERT_DIR}/default.crt" +KEY_PATH = f"{CERT_DIR}/default.key" +LEGO_CONFIG = "/etc/quadlets/lego/lego.yaml" +HOOK_MARKER = "/tmp/lego-hook-executed" + + +def prime_dns_cache(dns_server_ip: str, names: list[str]) -> None: + """Workaround for flaky ACME validations in the test environment. + + Pebble resolves the domains over TCP. When the libvirt dnsmasq has to forward the AAAA + query of a test domain upstream over TCP, the query may time out. A UDP query puts the + (negative) answer in the dnsmasq cache, so that the subsequent TCP query of Pebble succeeds. + """ + for name in names: + for qtype in ("A", "AAAA"): + subprocess.run(["dig", "+time=2", "+tries=2", qtype, f"@{dns_server_ip}", name], capture_output=True) + + +@pytest.fixture(scope="module") +def dns_names() -> list[str]: + """lego domain must be DNS-resolvable so Pebble can reach the VM on port 80 for HTTP-01 validation.""" + return ["lego"] + + +@pytest.fixture(scope="module") +def fcos_extra_files(pebble_acme_server, top_level_domain) -> dict: + """Extra files to inject into the FCOS VM for lego tests.""" + return { + # Lego configuration pointing at the Pebble ACME test server. + LEGO_CONFIG: ( + textwrap.dedent(f"""\ + storage: /var/lib/lego + log: + format: text + accounts: + default: + server: {pebble_acme_server['directory_url']} + email: test@{top_level_domain} + acceptsTermsOfService: true + challenges: + http: + http: + address: ":80" + certificates: + default: + account: default + challenge: http + domains: + - lego.{top_level_domain} + renew: + disableRandomSleep: true + hooks: + deploy: + command: /bin/sh /etc/lego/deploy-hook.sh + """), + 10023, # owner: lego + 10000, # group: itix-svc + 0o644, + ), + # Pebble root CA certificate, needed by lego to authenticate the Pebble ACME server. + "/etc/quadlets/lego/pebble.pem": ( + pebble_acme_server["ca_cert"], + 0, # owner: root + 0, # group: root + 0o644, + ), + # Drop-in to trust the Pebble CA. + "/etc/containers/systemd/lego.container.d/pebble.conf": ( + textwrap.dedent("""\ + [Container] + Environment=LEGO_CA_CERTIFICATES=/etc/lego/pebble.pem + """), + 0, # owner: root + 0, # group: root + 0o644, + ), + # Test hook: creates a marker file to verify that process-hooks.sh calls it. + "/etc/quadlets/lego/renew-hooks.d/test-hook.sh": ( + textwrap.dedent(f"""\ + #!/bin/sh + set -eu + touch {HOOK_MARKER} + """), + 0, # owner: root + 0, # group: root + 0o700, + ), + } + + +""" +Verify that the Lego Quadlet is correctly installed and configured on a fresh VM boot. +""" +class TestLegoQuadlet(test_quadlet.TestQuadlet): + expected_services = [ + {"name": "lego.target", "state": "active", "exists": True}, + {"name": "lego.timer", "state": "active", "exists": True}, + {"name": "lego-hooks.path", "state": "active", "exists": True}, + ] + + expected_files = [ + {"path": "/var/lib/quadlets/lego", "type": "directory", "owner": "lego", "group": "itix-svc", "mode": 0o755}, + {"path": "/etc/quadlets/lego", "type": "directory"}, + {"path": "/etc/quadlets/lego/deploy-hook.sh", "type": "file", "owner": "root", "mode": 0o755}, + {"path": "/etc/quadlets/lego/process-hooks.sh", "type": "file", "owner": "root", "mode": 0o755}, + {"path": "/etc/quadlets/lego/renew-hooks.d", "type": "directory"}, + ] + + expected_podman_images = [ + {"name": "docker.io/goacme/lego", "tag": "v5", "state": "present"}, + ] + + expected_main_service = "lego.target" + expected_main_service_timeout = 600 # Allow time for image pull + ACME negotiation + + def get_serial(self, host) -> str: + result = host.run(f"openssl x509 -in {CERT_PATH} -noout -serial") + assert result.rc == 0, f"Failed to read certificate serial: {result.stderr}" + return result.stdout.strip() + + def wait_for_hooks(self, host, timeout: int = 120) -> None: + """Block until the hook marker exists and all .renewed files have been processed.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + marker = host.file(HOOK_MARKER).exists + renewed = host.run(f"ls {CERT_DIR}/*.renewed 2>/dev/null | wc -l").stdout.strip() + if marker and renewed == "0": + return + time.sleep(2) + raise TimeoutError( + f"Renewal hooks were not processed within {timeout}s:\n" + + host.run("journalctl -u lego-hooks.service -u lego-hooks.path --no-pager -n 50").stdout + ) + + def run_lego(self, host) -> None: + result = host.run("systemctl start lego.service") + assert result.rc == 0, ( + f"lego.service failed: {result.stderr}\n" + + host.run("journalctl -u lego.service --no-pager -n 50").stdout + ) + + # ------------------------------------------------------------------ + # Test 1 — TLS certificate retrieval + # ------------------------------------------------------------------ + + @pytest.mark.flaky(reruns=6, reruns_delay=10) + def test_lego_certificate_obtained(self, fcos_host, top_level_domain, dns_server_ip): + """lego.service must successfully fetch a TLS certificate from the Pebble ACME server.""" + state = self.wait_for_unit_done(fcos_host, "lego.service", timeout=600) + if state == "failed": + # The first attempt may fail if the DNS record of the VM was not + # resolvable by Pebble when lego started: try again. + prime_dns_cache(dns_server_ip, [f"lego.{top_level_domain}"]) + self.run_lego(fcos_host) + + for path in (CERT_PATH, KEY_PATH): + f = fcos_host.file(path) + assert f.is_file, f"{path} not found" + assert f.user == "lego", f"{path} must be owned by lego, got {f.user}" + assert f.mode == 0o600, f"{path} must not be world-readable: got {oct(f.mode)}" + + domain = f"lego.{top_level_domain}" + result = fcos_host.run(f"openssl x509 -in {CERT_PATH} -noout -ext subjectAltName") + assert result.rc == 0, f"openssl failed to parse certificate: {result.stderr}" + assert f"DNS:{domain}" in result.stdout, ( + f"Expected {domain!r} in the certificate SAN:\n{result.stdout}" + ) + + # ------------------------------------------------------------------ + # Test 2 — Hooks are executed after the first issuance + # ------------------------------------------------------------------ + + def test_lego_hooks_executed_on_issuance(self, fcos_host): + """lego-hooks.path must trigger process-hooks.sh, which runs renew-hooks.d/ and cleans up .renewed files.""" + self.wait_for_hooks(fcos_host) + + # ------------------------------------------------------------------ + # Test 3 — Lego is idempotent + # ------------------------------------------------------------------ + + def test_lego_skips_renewal(self, fcos_host): + """A second run of lego.service must not renew a fresh certificate.""" + serial_before = self.get_serial(fcos_host) + fcos_host.run(f"rm -f {HOOK_MARKER}") + + self.run_lego(fcos_host) + + assert self.get_serial(fcos_host) == serial_before, "The certificate has been renewed but should not" + result = fcos_host.run("journalctl -u lego.service --no-pager -n 20") + assert "Skip renewal" in result.stdout, f"Expected 'Skip renewal' in lego logs:\n{result.stdout}" + time.sleep(5) + assert not fcos_host.file(HOOK_MARKER).exists, "Hooks have been run but no certificate was renewed" + + # ------------------------------------------------------------------ + # Test 4 — Certificate renewal + # ------------------------------------------------------------------ + + @pytest.mark.flaky(reruns=3, reruns_delay=5) + def test_lego_certificate_renewal(self, fcos_host, top_level_domain, dns_server_ip): + """lego.service must obtain a new certificate when forced to (renew.days: -1) and run the hooks.""" + prime_dns_cache(dns_server_ip, [f"lego.{top_level_domain}"]) + serial_before = self.get_serial(fcos_host) + fcos_host.run(f"rm -f {HOOK_MARKER}") + + # Force the renewal: a negative number of days always renews the certificate. + result = fcos_host.run( + f"cp -a {LEGO_CONFIG} {LEGO_CONFIG}.orig && " + f"sed -i 's/disableRandomSleep: true/disableRandomSleep: true\\n days: -1/' {LEGO_CONFIG}" + ) + assert result.rc == 0, f"Failed to edit lego.yaml: {result.stderr}" + try: + self.run_lego(fcos_host) + finally: + fcos_host.run(f"mv {LEGO_CONFIG}.orig {LEGO_CONFIG}") + + assert self.get_serial(fcos_host) != serial_before, ( + f"Certificate serial did not change after renewal (still {serial_before})" + ) + self.wait_for_hooks(fcos_host) + + # ------------------------------------------------------------------ + # Test 5 — Timer configuration + # ------------------------------------------------------------------ + + def test_lego_timer_configured(self, fcos_host): + """lego.timer must be active and scheduled for daily execution.""" + result = fcos_host.run("systemctl show lego.timer --property=ActiveState,TimersCalendar,Unit") + assert result.rc == 0 + assert "ActiveState=active" in result.stdout, f"lego.timer is not active:\n{result.stdout}" + # OnCalendar=daily is normalized as "*-*-* 00:00:00" + assert "OnCalendar=*-*-* 00:00:00" in result.stdout, ( + f"Expected a daily schedule in timer properties:\n{result.stdout}" + ) + assert "Unit=lego.service" in result.stdout, f"lego.timer must start lego.service:\n{result.stdout}" + + # ------------------------------------------------------------------ + # Test 6 — Reboot survival (must run last) + # ------------------------------------------------------------------ + + def test_lego_survives_reboot(self, fcos_vm, fcos_host, test_ssh_key): + """After a reboot, lego.target must be active and the certificate must be intact.""" + serial_before = self.get_serial(fcos_host) + + # Schedule the reboot so that the SSH command returns before the connection drops. + result = fcos_host.run("systemd-run --on-active=3 systemctl reboot") + assert result.rc == 0, f"Failed to schedule the reboot: {result.stderr}" + time.sleep(15) # let the reboot begin before we start polling SSH + + fcos_vm.wait_ssh(ssh_key=test_ssh_key, timeout=300) + + # The module-scoped fcos_host fixture is stale after reboot; create a fresh connection. + new_host = testinfra.get_host( + f"ssh://root@{fcos_vm.ip}", + ssh_extra_args=( + f"-i {test_ssh_key}" + " -o StrictHostKeyChecking=no" + " -o UserKnownHostsFile=/dev/null" + ), + ) + + self.wait_for_service(new_host, "lego.target", timeout=300) + self.check_expected_services(new_host, self.expected_services) + + # lego.service runs at each boot but must not renew a fresh certificate. + state = self.wait_for_unit_done(new_host, "lego.service", timeout=300) + assert state == "inactive", ( + f"lego.service ended in unexpected state: {state!r}\n" + + new_host.run("journalctl -b -u lego.service --no-pager -n 50").stdout + ) + result = new_host.run("systemctl show lego.service --property=Result") + assert "Result=success" in result.stdout, f"lego.service did not succeed after reboot:\n{result.stdout}" + + assert self.get_serial(new_host) == serial_before, "Certificate changed after reboot" diff --git a/cookbooks/quay/other/lego/quay.sh b/cookbooks/quay/other/lego/quay.sh index 3294d9b..e25c955 100644 --- a/cookbooks/quay/other/lego/quay.sh +++ b/cookbooks/quay/other/lego/quay.sh @@ -2,5 +2,13 @@ set -Eeuo pipefail +# On the first issuance, quay-app has not loaded any certificate yet: it is not +# running (its ExecStartPre loads the certificates) and will pick up the new +# certificate by itself when it starts. Restarting it would be useless. +if [ ! -s /run/quadlets/quay/tls/ssl.cert ]; then + echo "No TLS certificate loaded in Quay yet, skipping the restart of quay-app." + exit 0 +fi + /etc/quadlets/quay/quay_load_tls_certs.sh systemctl --no-block restart quay-app.service diff --git a/cookbooks/quay/quay-app.container b/cookbooks/quay/quay-app.container index bb8b4cc..da6776c 100644 --- a/cookbooks/quay/quay-app.container +++ b/cookbooks/quay/quay-app.container @@ -1,8 +1,9 @@ [Unit] Description=Quay Container Registry Application Documentation=https://docs.projectquay.io/ -After=network.target redis-server.service var-lib-virtiofs-data.mount +After=network.target redis-server.service var-lib-virtiofs-data.mount lego.target Requires=redis-server.service var-lib-virtiofs-data.mount +Wants=lego.target # Only start if Quay has been configured ConditionPathExists=/etc/quadlets/quay/app/config.yaml diff --git a/cookbooks/smtprelay/SPECS.md b/cookbooks/smtprelay/SPECS.md index 508bfe8..3263166 100644 --- a/cookbooks/smtprelay/SPECS.md +++ b/cookbooks/smtprelay/SPECS.md @@ -44,8 +44,8 @@ allowed_nets = 0.0.0.0/0 ; Enable TLS for incoming connections on port 587 listen = starttls://0.0.0.0:587 -local_cert = /etc/smtprelay/tls/localhost.crt -local_key = /etc/smtprelay/tls/localhost.key +local_cert = /etc/smtprelay/tls/default.crt +local_key = /etc/smtprelay/tls/default.key ; Enforce encrypted connection on STARTTLS ports before ; accepting mails from client. diff --git a/cookbooks/smtprelay/config/examples/smtprelay.ini b/cookbooks/smtprelay/config/examples/smtprelay.ini index d250511..242d1a6 100644 --- a/cookbooks/smtprelay/config/examples/smtprelay.ini +++ b/cookbooks/smtprelay/config/examples/smtprelay.ini @@ -20,14 +20,14 @@ allowed_nets = 0.0.0.0/0 ; ----------------------------------------------------------------------- ; Inbound TLS (certificates provided by the lego cookbook) ; -; Replace "localhost" with the actual certificate filename from -; /var/lib/quadlets/lego/certificates/. +; The lego cookbook stores the certificate named "default" in +; /var/lib/quadlets/lego/certificates/default.{crt,key}. ; ----------------------------------------------------------------------- ; Enable TLS for incoming connections on port 587 listen = starttls://0.0.0.0:587 -local_cert = /etc/smtprelay/tls/localhost.crt -local_key = /etc/smtprelay/tls/localhost.key +local_cert = /etc/smtprelay/tls/default.crt +local_key = /etc/smtprelay/tls/default.key ; Enforce encrypted connection on STARTTLS ports before ; accepting mails from client. diff --git a/cookbooks/smtprelay/other/lego/smtprelay.sh b/cookbooks/smtprelay/other/lego/smtprelay.sh index 0b4a060..a1eeb2f 100755 --- a/cookbooks/smtprelay/other/lego/smtprelay.sh +++ b/cookbooks/smtprelay/other/lego/smtprelay.sh @@ -3,4 +3,5 @@ set -Eeuo pipefail install -o 10030 -g 10000 -m 0600 -t /run/quadlets/smtprelay/tls /var/lib/quadlets/lego/certificates/*.crt /var/lib/quadlets/lego/certificates/*.key -systemctl --no-block restart smtprelay.service +# "try-restart" does nothing if smtprelay is not running yet (first issuance at boot). +systemctl --no-block try-restart smtprelay.service diff --git a/cookbooks/traefik/README.md b/cookbooks/traefik/README.md index d7d1570..67a3fd6 100644 --- a/cookbooks/traefik/README.md +++ b/cookbooks/traefik/README.md @@ -122,18 +122,18 @@ HealthCmd=wget -q -O /dev/null --no-check-certificate --header 'Host: ping' http ExecStartPre=/bin/sh -c 'install -o 10001 -g 10000 -m 0600 -t /run/quadlets/traefik/tls /var/lib/quadlets/lego/certificates/*.crt /var/lib/quadlets/lego/certificates/*.key' ``` -**/etc/quadlets/traefik/conf.d/tls.yaml**: +**/etc/quadlets/traefik/conf.d/tls.yaml** (the lego cookbook stores the certificate under the `default` ID, see `certificates.default` in `/etc/quadlets/lego/lego.yaml`): ```yaml tls: certificates: - - certFile: /etc/traefik/tls/f.q.d.n.crt - keyFile: /etc/traefik/tls/f.q.d.n.key + - certFile: /etc/traefik/tls/default.crt + keyFile: /etc/traefik/tls/default.key stores: default: defaultCertificate: - certFile: /etc/traefik/tls/f.q.d.n.crt - keyFile: /etc/traefik/tls/f.q.d.n.key + certFile: /etc/traefik/tls/default.crt + keyFile: /etc/traefik/tls/default.key http: routers: @@ -185,3 +185,18 @@ http: servers: - url: "http://127.0.0.1:8080" ``` + +**/etc/quadlets/lego/renew-hooks.d/traefik.sh** (mode 0755, owned by root): + +```sh +#!/bin/bash + +set -Eeuo pipefail + +# The TLS certificates are copied by the ExecStartPre of traefik.service. +# "try-restart" does nothing if Traefik is not running yet (first issuance at boot). +systemctl --no-block try-restart traefik.service +``` + +This hook is run by the lego cookbook each time a certificate is obtained or renewed. +Without it, Traefik would keep serving the old certificate until its next restart. diff --git a/cookbooks/vsftpd/Makefile b/cookbooks/vsftpd/Makefile index 04d5f39..c0b083c 100644 --- a/cookbooks/vsftpd/Makefile +++ b/cookbooks/vsftpd/Makefile @@ -10,3 +10,7 @@ PROJECT_GID = 10000 # Include common Makefile include ../../scripts/common.mk + +# All config file are owned by root +$(filter %.conf, $(TARGET_CONFIG_FILES) $(TARGET_EXAMPLES_CONFIG_FILES)): + install -o root -g root -m 0644 $< $@ diff --git a/cookbooks/vsftpd/README.md b/cookbooks/vsftpd/README.md index e76d0ab..8789181 100644 --- a/cookbooks/vsftpd/README.md +++ b/cookbooks/vsftpd/README.md @@ -7,7 +7,7 @@ vsftpd (Very Secure FTP Daemon) is started as a Podman Quadlet. It provides a se This cookbook: - Builds a custom vsftpd container image locally. -- Supports TLS encryption with automatic certificate loading from Let's Encrypt (integrates with the `lego` cookbook). +- Supports TLS encryption with a certificate provided by the `lego` cookbook (Let's Encrypt in production, self-signed in development). - Maps system users into the container for authentication. - Includes a timer to periodically rebuild the container image. - Reloads certificates automatically when renewed. @@ -15,7 +15,21 @@ This cookbook: ## Prerequisites - Configuration file `/etc/quadlets/vsftpd/vsftpd.conf.d/local.conf` must exist. -- For TLS support, the `lego` cookbook should be configured to provide certificates. +- For TLS support, the `lego` cookbook must be configured to provide a certificate named `default` (see the lego cookbook's README). +- The volumes to share must be mounted on `/data` with a drop-in (see [dropins/vsftpd.container.d/storage.conf](dropins/vsftpd.container.d/storage.conf)). + +## TLS certificates + +The TLS settings are in `local.conf` (see [config/examples/vsftpd.conf.d/local.conf](config/examples/vsftpd.conf.d/local.conf)): + +```ini +ssl_enable=YES +rsa_cert_file=/etc/vsftpd/tls/default.crt +rsa_private_key_file=/etc/vsftpd/tls/default.key +``` + +- Before starting vsftpd, **vsftpd.service** copies the certificates from `/var/lib/quadlets/lego/certificates` to `/run/quadlets/vsftpd/tls` (mounted on `/etc/vsftpd/tls` in the container). +- When lego obtains or renews a certificate, its renewal hook ([other/lego/vsftpd.sh](other/lego/vsftpd.sh)) restarts **vsftpd.service** (if it is running) to load the new certificate. ## Usage @@ -52,7 +66,7 @@ Or with TLS: lftp -u username localhost ``` -When Let's Encrypt certificates are renewed, the **vsftpd-load-renewed-certificate.service** automatically reloads them. +When the certificates are renewed by lego, **vsftpd.service** is restarted automatically to load them. Restart the **vsftpd.target** unit. @@ -69,5 +83,5 @@ sudo make uninstall clean ## Integration tests ```sh -sudo make test +sudo make pytest ``` diff --git a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/local-users.conf b/cookbooks/vsftpd/config/examples/vsftpd.conf.d/local.conf similarity index 66% rename from cookbooks/vsftpd/config/examples/vsftpd.conf.d/local-users.conf rename to cookbooks/vsftpd/config/examples/vsftpd.conf.d/local.conf index 6eede24..1ba5ba2 100644 --- a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/local-users.conf +++ b/cookbooks/vsftpd/config/examples/vsftpd.conf.d/local.conf @@ -22,3 +22,11 @@ delete_failed_uploads=NO ftpd_banner=Storage file_open_mode=0660 ls_recurse_enable=YES + +# TLS (certificate provided by the lego cookbook, see /var/lib/quadlets/lego/certificates) +ssl_enable=YES +ssl_request_cert=NO +ssl_tlsv1_2=NO +ssl_tlsv1_3=YES +rsa_cert_file=/etc/vsftpd/tls/default.crt +rsa_private_key_file=/etc/vsftpd/tls/default.key diff --git a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/tls.conf b/cookbooks/vsftpd/config/examples/vsftpd.conf.d/tls.conf deleted file mode 100644 index 7b2e418..0000000 --- a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/tls.conf +++ /dev/null @@ -1,5 +0,0 @@ -ssl_enable=YES -ssl_request_cert=NO -ssl_tlsv1_2=NO -rsa_cert_file=/etc/vsftpd/tls/f.q.d.n.crt -rsa_private_key_file=/etc/vsftpd/tls/f.q.d.n.key diff --git a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/virtual-users.conf b/cookbooks/vsftpd/config/examples/vsftpd.conf.d/virtual-users.conf index ee24d84..39ac357 100644 --- a/cookbooks/vsftpd/config/examples/vsftpd.conf.d/virtual-users.conf +++ b/cookbooks/vsftpd/config/examples/vsftpd.conf.d/virtual-users.conf @@ -16,3 +16,10 @@ local_root=/data #user_sub_token=$USER #local_root=/data/$USER +# TLS (certificate provided by the lego cookbook, see /var/lib/quadlets/lego/certificates) +ssl_enable=YES +ssl_request_cert=NO +ssl_tlsv1_2=NO +ssl_tlsv1_3=YES +rsa_cert_file=/etc/vsftpd/tls/default.crt +rsa_private_key_file=/etc/vsftpd/tls/default.key diff --git a/cookbooks/vsftpd/dropins/vsftpd.container.d/storage.conf b/cookbooks/vsftpd/dropins/vsftpd.container.d/storage.conf new file mode 100644 index 0000000..422df2b --- /dev/null +++ b/cookbooks/vsftpd/dropins/vsftpd.container.d/storage.conf @@ -0,0 +1,2 @@ +[Container] +Volume=/var/lib/virtiofs/data/vsftpd-local-user:/data diff --git a/cookbooks/vsftpd/other/lego/vsftpd.sh b/cookbooks/vsftpd/other/lego/vsftpd.sh index b5913b0..9222c3d 100644 --- a/cookbooks/vsftpd/other/lego/vsftpd.sh +++ b/cookbooks/vsftpd/other/lego/vsftpd.sh @@ -2,5 +2,6 @@ set -Eeuo pipefail -install -o 10015 -g 10000 -m 0600 -t /run/quadlets/vsftpd/tls /var/lib/quadlets/lego/certificates/*.crt /var/lib/quadlets/lego/certificates/*.key -systemctl --no-block restart vsftpd.service +# The vsftpd service copies the TLS certificates provided by lego when it starts (see ExecStartPre). +# "try-restart" does nothing if vsftpd is not running yet (first issuance at boot). +systemctl --no-block try-restart vsftpd.service diff --git a/cookbooks/vsftpd/tests/test_01_install.py b/cookbooks/vsftpd/tests/test_01_install.py new file mode 100644 index 0000000..548c05b --- /dev/null +++ b/cookbooks/vsftpd/tests/test_01_install.py @@ -0,0 +1,365 @@ +import ftplib +import io +import ssl +import subprocess +import textwrap +import time +import test_quadlet # noqa: F401 +import pytest +import testinfra + +LEGO_CONFIG = "/etc/quadlets/lego/lego.yaml" +LEGO_CERT_PATH = "/var/lib/quadlets/lego/certificates/default.crt" +FTP_USER = "ftptest" +FTP_UID = 1500 # vsftpd.container rewrites the home of uids 1000-1999 to /data +FTP_PASSWORD = "S3cret-ftp-passw0rd" + + +def prime_dns_cache(dns_server_ip: str, names: list[str]) -> None: + """Workaround for flaky ACME validations in the test environment. + + Pebble resolves the domains over TCP. When the libvirt dnsmasq has to forward the AAAA + query of a test domain upstream over TCP, the query may time out. A UDP query puts the + (negative) answer in the dnsmasq cache, so that the subsequent TCP query of Pebble succeeds. + """ + for name in names: + for qtype in ("A", "AAAA"): + subprocess.run(["dig", "+time=2", "+tries=2", qtype, f"@{dns_server_ip}", name], capture_output=True) + + +class ReusedSessionFTP_TLS(ftplib.FTP_TLS): + """FTP_TLS client that reuses the TLS session of the control connection on the data connections. + + vsftpd requires TLS session reuse by default (require_ssl_reuse=YES). + """ + + def ntransfercmd(self, cmd, rest=None): + conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest) + if self._prot_p: + conn = self.context.wrap_socket(conn, server_hostname=self.host, session=self.sock.session) + return conn, size + + +@pytest.fixture(scope="module") +def dns_names() -> list[str]: + """The FTP domain must be DNS-resolvable so Pebble can reach the VM on port 80 for HTTP-01 validation.""" + return ["ftp"] + + +@pytest.fixture(scope="module") +def ftp_domain(top_level_domain) -> str: + return f"ftp.{top_level_domain}" + + +@pytest.fixture(scope="module") +def fcos_extra_files(pebble_acme_server, ftp_domain) -> dict: + """Extra files to inject into the FCOS VM for vsftpd tests.""" + return { + # Lego configuration pointing at the Pebble ACME test server. + LEGO_CONFIG: ( + textwrap.dedent(f"""\ + storage: /var/lib/lego + log: + format: text + accounts: + default: + server: {pebble_acme_server['directory_url']} + email: test@{ftp_domain} + acceptsTermsOfService: true + challenges: + http: + http: + address: ":80" + certificates: + default: + account: default + challenge: http + domains: + - {ftp_domain} + renew: + disableRandomSleep: true + hooks: + deploy: + command: /bin/sh /etc/lego/deploy-hook.sh + """), + 10023, # owner: lego + 10000, # group: itix-svc + 0o644, + ), + # Pebble root CA certificate, needed by lego to authenticate the Pebble ACME server. + "/etc/quadlets/lego/pebble.pem": ( + pebble_acme_server["ca_cert"], + 0, + 0, + 0o644, + ), + # Drop-in to trust the Pebble CA. + "/etc/containers/systemd/lego.container.d/pebble.conf": ( + textwrap.dedent("""\ + [Container] + Environment=LEGO_CA_CERTIFICATES=/etc/lego/pebble.pem + """), + 0, + 0, + 0o644, + ), + # vsftpd configuration: local users + TLS with the certificate provided by lego. + "/etc/quadlets/vsftpd/vsftpd.conf.d/local.conf": ( + textwrap.dedent(f"""\ + pasv_address={ftp_domain} + pasv_addr_resolve=YES + local_enable=YES + local_root=/data + chroot_local_user=YES + pam_service_name=vsftpd + write_enable=YES + allow_writeable_chroot=YES + anonymous_enable=NO + file_open_mode=0660 + ssl_enable=YES + ssl_request_cert=NO + ssl_tlsv1_2=NO + ssl_tlsv1_3=YES + rsa_cert_file=/etc/vsftpd/tls/default.crt + rsa_private_key_file=/etc/vsftpd/tls/default.key + """), + 0, + 0, + 0o644, + ), + # Storage shared through FTP. + "/etc/tmpfiles.d/vsftpd-test.conf": ( + f"d /var/lib/vsftpd-test 0770 {FTP_UID} {FTP_UID} -\n", + 0, + 0, + 0o644, + ), + "/etc/containers/systemd/vsftpd.container.d/storage.conf": ( + textwrap.dedent("""\ + [Container] + Volume=/var/lib/vsftpd-test:/data:z + """), + 0, + 0, + 0o644, + ), + } + + +""" +Verify that vsftpd serves FTPS with a certificate obtained by lego. +""" +class TestVsftpdQuadlet(test_quadlet.TestQuadlet): + # vsftpd needs the certificate obtained by lego, so the vsftpd services are + # checked once lego has obtained it (see test_create_ftp_user). + expected_services = [ + {"name": "lego.target", "state": "active", "exists": True}, + {"name": "lego-hooks.path", "state": "active", "exists": True}, + ] + + expected_vsftpd_services = [ + {"name": "vsftpd.target", "state": "active", "exists": True}, + {"name": "vsftpd.service", "state": "active", "exists": True}, + ] + + expected_vsftpd_ports = [ + {"number": 21, "protocol": "tcp", "state": "open"}, + ] + + expected_files = [ + {"path": "/etc/quadlets/lego/renew-hooks.d/vsftpd.sh", "type": "file", "owner": "root", "mode": 0o755}, + ] + + # Certificates copied by vsftpd.service from the lego storage + expected_tls_files = [ + {"path": "/run/quadlets/vsftpd/tls/default.crt", "type": "file", "owner": "root", "mode": 0o600}, + {"path": "/run/quadlets/vsftpd/tls/default.key", "type": "file", "owner": "root", "mode": 0o600}, + ] + + # The vsftpd image is built on first boot (see test_create_ftp_user). + expected_vsftpd_images = [ + {"name": "localhost/vsftpd", "tag": "latest", "state": "present"}, + ] + + expected_main_service = "lego.target" + expected_main_service_timeout = 600 # Image pull + ACME negotiation + + def ftps_connect(self, host_ip: str, ca_bundle: str) -> ReusedSessionFTP_TLS: + """Open an authenticated FTPS session, trusting only the Pebble CA.""" + ctx = ssl.create_default_context(cadata=ca_bundle) + # The VM is reached by IP: the hostname is checked separately against the certificate SAN. + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_REQUIRED + ftp = ReusedSessionFTP_TLS(context=ctx, timeout=30) + ftp.connect(host_ip, 21) + ftp.login(FTP_USER, FTP_PASSWORD) + ftp.prot_p() + return ftp + + def served_certificate(self, host_ip: str, ca_bundle: str) -> dict: + ftp = self.ftps_connect(host_ip, ca_bundle) + try: + return ftp.sock.getpeercert() + finally: + ftp.quit() + + def lego_serial(self, host) -> str: + result = host.run(f"openssl x509 -in {LEGO_CERT_PATH} -noout -serial") + assert result.rc == 0, f"Failed to read the lego certificate: {result.stderr}" + return result.stdout.strip().removeprefix("serial=").upper() + + def wait_for_served_serial(self, host_ip: str, ca_bundle: str, serial: str, timeout: int = 180) -> None: + deadline = time.monotonic() + timeout + served = None + while time.monotonic() < deadline: + try: + served = self.served_certificate(host_ip, ca_bundle)["serialNumber"].upper() + if served == serial: + return + except (OSError, ftplib.Error): + pass # vsftpd may be restarting + time.sleep(5) + raise TimeoutError(f"vsftpd serves certificate {served}, expected {serial}") + + # ------------------------------------------------------------------ + # Test 1 — Lego obtains the certificate + # ------------------------------------------------------------------ + + @pytest.mark.flaky(reruns=3, reruns_delay=10) + def test_lego_certificate_obtained(self, fcos_host, dns_server_ip, ftp_domain): + """lego.service must obtain the certificate from Pebble.""" + state = self.wait_for_unit_done(fcos_host, "lego.service", timeout=600) + if state == "failed": + # The first attempt may fail if the DNS record of the VM was not + # resolvable by Pebble when lego started: try again. + prime_dns_cache(dns_server_ip, [ftp_domain]) + result = fcos_host.run("systemctl start lego.service") + assert result.rc == 0, ( + f"lego.service failed: {result.stderr}\n" + + fcos_host.run("journalctl -u lego.service --no-pager -n 50").stdout + ) + assert fcos_host.file(LEGO_CERT_PATH).is_file, f"{LEGO_CERT_PATH} not found" + + # ------------------------------------------------------------------ + # Test 2 — Create a local user + # ------------------------------------------------------------------ + + def test_create_ftp_user(self, fcos_host): + """Create a local user and restart vsftpd so that it is part of its passwd cache.""" + result = fcos_host.run( + f"id {FTP_USER} || useradd -u {FTP_UID} -U -M -d /var/lib/vsftpd-test {FTP_USER}" + ) + assert result.rc == 0, f"useradd failed: {result.stderr}" + result = fcos_host.run(f"echo '{FTP_USER}:{FTP_PASSWORD}' | chpasswd") + assert result.rc == 0, f"chpasswd failed: {result.stderr}" + # The vsftpd image build may take a while on first boot. + state = self.wait_for_unit_done(fcos_host, "vsftpd-build.service", timeout=900) + assert state == "inactive", ( + f"vsftpd-build.service ended in unexpected state: {state!r}\n" + + fcos_host.run("journalctl -u vsftpd-build.service --no-pager -n 50").stdout + ) + self.check_expected_podman_images(fcos_host, self.expected_vsftpd_images) + result = fcos_host.run("systemctl restart vsftpd.service && systemctl start vsftpd.target") + assert result.rc == 0, ( + f"Failed to restart vsftpd: {result.stderr}\n" + + fcos_host.run("journalctl -u vsftpd.service --no-pager -n 50").stdout + ) + self.check_expected_services(fcos_host, self.expected_vsftpd_services) + self.check_expected_files(fcos_host, self.expected_tls_files) + + def test_vsftpd_ports(self, fcos_vm): + """The FTP control port must be reachable.""" + self.check_expected_ports(fcos_vm, self.expected_vsftpd_ports) + + # ------------------------------------------------------------------ + # Test 3 — FTPS with the certificate obtained by lego + # ------------------------------------------------------------------ + + @pytest.mark.flaky(reruns=6, reruns_delay=10) + def test_ftps_certificate(self, fcos_vm, fcos_host, pebble_acme_server, ftp_domain): + """vsftpd must serve the certificate obtained by lego from Pebble.""" + cert = self.served_certificate(fcos_vm.ip, pebble_acme_server["ca_bundle"]) + san = [value for kind, value in cert.get("subjectAltName", ()) if kind == "DNS"] + assert ftp_domain in san, f"Expected {ftp_domain!r} in the certificate SAN, got {san}" + assert cert["serialNumber"].upper() == self.lego_serial(fcos_host), ( + "vsftpd does not serve the certificate obtained by lego" + ) + + def test_ftps_upload(self, fcos_vm, fcos_host, pebble_acme_server): + """An authenticated user must be able to upload and list files over FTPS.""" + ftp = self.ftps_connect(fcos_vm.ip, pebble_acme_server["ca_bundle"]) + try: + ftp.storbinary("STOR hello.txt", io.BytesIO(b"Hello, FTPS!\n")) + assert "hello.txt" in ftp.nlst(), "Uploaded file not listed" + finally: + ftp.quit() + assert fcos_host.file("/var/lib/vsftpd-test/hello.txt").content_string == "Hello, FTPS!\n" + + def test_plain_ftp_rejected(self, fcos_vm): + """Local users must not be able to log in without TLS.""" + ftp = ftplib.FTP(timeout=30) + ftp.connect(fcos_vm.ip, 21) + try: + with pytest.raises(ftplib.error_perm): + ftp.login(FTP_USER, FTP_PASSWORD) + finally: + ftp.close() + + # ------------------------------------------------------------------ + # Test 4 — Renewal: the lego hook restarts vsftpd with the new certificate + # ------------------------------------------------------------------ + + @pytest.mark.flaky(reruns=3, reruns_delay=5) + def test_certificate_renewal(self, fcos_vm, fcos_host, pebble_acme_server, dns_server_ip, ftp_domain): + """When lego renews the certificate, vsftpd must serve the new one.""" + prime_dns_cache(dns_server_ip, [ftp_domain]) + serial_before = self.lego_serial(fcos_host) + + # Force the renewal: a negative number of days always renews the certificate. + result = fcos_host.run( + f"cp -a {LEGO_CONFIG} {LEGO_CONFIG}.orig && " + f"sed -i 's/disableRandomSleep: true/disableRandomSleep: true\\n days: -1/' {LEGO_CONFIG}" + ) + assert result.rc == 0, f"Failed to edit lego.yaml: {result.stderr}" + try: + result = fcos_host.run("systemctl start lego.service") + assert result.rc == 0, ( + f"lego.service failed: {result.stderr}\n" + + fcos_host.run("journalctl -u lego.service --no-pager -n 50").stdout + ) + finally: + fcos_host.run(f"mv {LEGO_CONFIG}.orig {LEGO_CONFIG}") + + serial_after = self.lego_serial(fcos_host) + assert serial_after != serial_before, "lego did not renew the certificate" + self.wait_for_served_serial(fcos_vm.ip, pebble_acme_server["ca_bundle"], serial_after) + + # ------------------------------------------------------------------ + # Test 5 — Reboot survival (must run last) + # ------------------------------------------------------------------ + + def test_survives_reboot(self, fcos_vm, fcos_host, test_ssh_key, pebble_acme_server): + """After a reboot, vsftpd must serve FTPS again (the tmpfs holding the certificates is repopulated).""" + serial_before = self.lego_serial(fcos_host) + + # Schedule the reboot so that the SSH command returns before the connection drops. + result = fcos_host.run("systemd-run --on-active=3 systemctl reboot") + assert result.rc == 0, f"Failed to schedule the reboot: {result.stderr}" + time.sleep(15) # let the reboot begin before we start polling SSH + + fcos_vm.wait_ssh(ssh_key=test_ssh_key, timeout=300) + + # The module-scoped fcos_host fixture is stale after reboot; create a fresh connection. + new_host = testinfra.get_host( + f"ssh://root@{fcos_vm.ip}", + ssh_extra_args=( + f"-i {test_ssh_key}" + " -o StrictHostKeyChecking=no" + " -o UserKnownHostsFile=/dev/null" + ), + ) + + self.wait_for_service(new_host, "vsftpd.target", timeout=300) + self.check_expected_services(new_host, self.expected_services + self.expected_vsftpd_services) + self.check_expected_files(new_host, self.expected_files + self.expected_tls_files) + self.wait_for_served_serial(fcos_vm.ip, pebble_acme_server["ca_bundle"], serial_before) diff --git a/cookbooks/vsftpd/tmpfiles.d/examples/vsftpd-storage.conf b/cookbooks/vsftpd/tmpfiles.d/examples/vsftpd-storage.conf new file mode 100644 index 0000000..663e428 --- /dev/null +++ b/cookbooks/vsftpd/tmpfiles.d/examples/vsftpd-storage.conf @@ -0,0 +1,6 @@ +d$ /var/lib/virtiofs/data/vsftpd-local-user 0770 0 1000 - +d$ /var/lib/virtiofs/data/vsftpd-local-user/foo 0770 0 1000 - +d$ /var/lib/virtiofs/data/vsftpd-local-user/bar 0770 0 1000 - +d$ /var/lib/virtiofs/data/vsftpd-virtual-user 0770 0 0 - +d$ /var/lib/virtiofs/data/vsftpd-virtual-user/foo 0770 0 0 - +d$ /var/lib/virtiofs/data/vsftpd-virtual-user/bar 0770 0 0 - diff --git a/cookbooks/vsftpd/tmpfiles.d/vsftpd.conf b/cookbooks/vsftpd/tmpfiles.d/vsftpd.conf index 9befdab..5807c33 100644 --- a/cookbooks/vsftpd/tmpfiles.d/vsftpd.conf +++ b/cookbooks/vsftpd/tmpfiles.d/vsftpd.conf @@ -1,6 +1,5 @@ d$ /run/quadlets/vsftpd 0700 0 0 - d$ /run/quadlets/vsftpd/cache 0700 0 0 - d$ /run/quadlets/vsftpd/tls 0700 0 0 - -d$ /var/lib/virtiofs/data/storage 0755 0 0 - d$ /var/lib/quadlets/vsftpd/log 0755 0 0 - d$ /var/lib/quadlets/vsftpd/data 0755 0 0 - diff --git a/cookbooks/vsftpd/vsftpd.container b/cookbooks/vsftpd/vsftpd.container index be35829..c21a8a5 100644 --- a/cookbooks/vsftpd/vsftpd.container +++ b/cookbooks/vsftpd/vsftpd.container @@ -25,6 +25,15 @@ ExecStartPre=/bin/sh -Eeuo pipefail -c '\ fi) > /run/quadlets/vsftpd/cache/$file ; \ done' +# Get the TLS certificates provided by the lego cookbook in place before starting vsftpd. +# /run/quadlets/vsftpd/tls is a tmpfs: it has to be populated at each start. +# When the certificates are renewed, the lego hook (other/lego/vsftpd.sh) copies them again. +ExecStartPre=/bin/bash -Eeuo pipefail -c '\ + shopt -s nullglob ; \ + for file in /var/lib/quadlets/lego/certificates/*.crt /var/lib/quadlets/lego/certificates/*.key; do \ + install -o root -g root -m 0600 -t /run/quadlets/vsftpd/tls "$file" ; \ + done' + [Container] ContainerName=vsftpd @@ -38,12 +47,12 @@ User=0 # Storage Volume=/var/lib/quadlets/vsftpd/log:/var/log/vsftpd:Z Volume=/var/lib/quadlets/vsftpd/data:/var/lib/vsftpd:Z -Volume=/var/lib/virtiofs/data/storage:/data Volume=/etc/quadlets/vsftpd/vsftpd.conf.d:/etc/vsftpd:ro Volume=/run/quadlets/vsftpd/cache/passwd:/etc/passwd:Z Volume=/run/quadlets/vsftpd/cache/group:/etc/group:Z Volume=/run/quadlets/vsftpd/cache/shadow:/etc/shadow:Z Volume=/run/quadlets/vsftpd/tls:/etc/vsftpd/tls:Z +# Heads up ! You need to mount the volumes you want to share in a drop-in configuration file ! # Network Network=host