From 8c8be464dae02dcb430e5d0138f9f5ce61f9c311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= Date: Fri, 24 Jul 2026 08:40:16 +0000 Subject: [PATCH] fix health probe + listen address --- cookbooks/vllm/README.md | 11 +++---- .../vllm/config/examples/llmsnap/config.yaml | 12 ++++---- .../vllm/config/examples/models/qwen05.env | 16 ++++++++++ .../vllm/config/examples/models/qwen05.yaml | 10 ------- .../vllm/config/examples/models/smol135.env | 16 ++++++++++ .../vllm/config/examples/models/smol135.yaml | 10 ------- cookbooks/vllm/config/examples/vllm.env | 30 +++++++++++++++++-- cookbooks/vllm/vllm-model@.container | 30 +++++++++++++++---- cookbooks/vllm/vllm.image | 4 +-- 9 files changed, 99 insertions(+), 40 deletions(-) diff --git a/cookbooks/vllm/README.md b/cookbooks/vllm/README.md index a8b6457..52a2371 100644 --- a/cookbooks/vllm/README.md +++ b/cookbooks/vllm/README.md @@ -93,14 +93,15 @@ as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true` > file. For a new model `` you only add three small pieces: > > 1. `models/.yaml` — its vLLM engine config (see below). - > 2. `vllm-model@.container` — a **symlink** to `vllm-model@.container` - > (so Quadlet instantiates the unit) **and** a - > `dropins/vllm-model@.container.d/10-port.conf` drop-in picking a new - > fixed `PublishPort=127.0.0.1:58NN:8000`. + > 2. `models/.env` — its own fixed host-loopback API port + > `COOKBOOK_VLLM_MODEL_PORT=58NN` (fed to vLLM's `--host 127.0.0.1 --port` + > by the shared template; the unit runs `Network=host`, so there is no + > `PublishPort`). Do **not** call it `VLLM_PORT`: that variable is vLLM's + > internal port, not the API port. > 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the > `systemctl … vllm-model@.service` `cmd`/`cmdStop`. > - > Keep the port in sync across the drop-in and `config.yaml`. + > Keep the port in sync across `models/.env` and `config.yaml`. - `models/.yaml` — the **vLLM engine configuration** for one model (`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model diff --git a/cookbooks/vllm/config/examples/llmsnap/config.yaml b/cookbooks/vllm/config/examples/llmsnap/config.yaml index ca0c416..7ec8c59 100644 --- a/cookbooks/vllm/config/examples/llmsnap/config.yaml +++ b/cookbooks/vllm/config/examples/llmsnap/config.yaml @@ -15,9 +15,11 @@ # vllm-polkit.sh). # # Consequences for each model entry below: -# - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet -# publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic -# ${PORT}: a Quadlet cannot consume llmsnap's runtime port. +# - `proxy:` is REQUIRED and points at the FIXED host-loopback port the model's +# vLLM API server binds (`--host 127.0.0.1 --port 58NN`, from +# COOKBOOK_VLLM_MODEL_PORT in models/.env; the unit runs Network=host +# so there is no PublishPort). We do NOT use llmsnap's dynamic ${PORT}: a +# Quadlet cannot consume llmsnap's runtime port. # - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the # command as a child process for the model's whole lifetime, so we: # 1. `systemctl start vllm-model@.service` (returns once the container is up; @@ -50,14 +52,14 @@ models: # Tiny real instruct model: has a chat template AND a tool-call parser, so it # exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts). qwen05: - proxy: http://127.0.0.1:5801 # PublishPort of vllm-model@qwen05 + proxy: http://127.0.0.1:5801 # --port of vllm-model@qwen05 (COOKBOOK_VLLM_MODEL_PORT) cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done' cmdStop: systemctl stop vllm-model@qwen05.service # Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely # tested rather than reloading the same weights. smol135: - proxy: http://127.0.0.1:5802 # PublishPort of vllm-model@smol135 + proxy: http://127.0.0.1:5802 # --port of vllm-model@smol135 (COOKBOOK_VLLM_MODEL_PORT) cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done' cmdStop: systemctl stop vllm-model@smol135.service diff --git a/cookbooks/vllm/config/examples/models/qwen05.env b/cookbooks/vllm/config/examples/models/qwen05.env index e69de29..dfdd515 100644 --- a/cookbooks/vllm/config/examples/models/qwen05.env +++ b/cookbooks/vllm/config/examples/models/qwen05.env @@ -0,0 +1,16 @@ +## +## Per-instance fixed host-loopback port for the qwen05 model's API server. +## +## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host +## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's +## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5801` +## in config.yaml (llmsnap runs Network=host and reaches the model here). Each +## model gets its OWN port so two models can be resident at once (GPU sleep +## mode). Keep it in sync with config.yaml. +## +## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed +## executor) and does NOT set the API server port — reusing it as the API port +## collides with vLLM's internal usage. See +## https://docs.vllm.ai/en/stable/configuration/env_vars/ +## +COOKBOOK_VLLM_MODEL_PORT=5801 diff --git a/cookbooks/vllm/config/examples/models/qwen05.yaml b/cookbooks/vllm/config/examples/models/qwen05.yaml index 78cfa71..cc4634b 100644 --- a/cookbooks/vllm/config/examples/models/qwen05.yaml +++ b/cookbooks/vllm/config/examples/models/qwen05.yaml @@ -18,16 +18,6 @@ dtype: bfloat16 max-model-len: 4096 max-num-seqs: 4 -## -## Per-instance fixed host-loopback port for the qwen05 model. -## -## Referenced by llmsnap's per-model -## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and -## reaches the model here). Each model gets its OWN port so two models can be -## resident at once (GPU sleep mode). Keep it in sync with config.yaml. -## -port: 5801 - # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager # mode skips it (fine for a smoke test). enforce-eager: true diff --git a/cookbooks/vllm/config/examples/models/smol135.env b/cookbooks/vllm/config/examples/models/smol135.env index e69de29..d6e8f37 100644 --- a/cookbooks/vllm/config/examples/models/smol135.env +++ b/cookbooks/vllm/config/examples/models/smol135.env @@ -0,0 +1,16 @@ +## +## Per-instance fixed host-loopback port for the smol135 model's API server. +## +## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host +## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's +## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5802` +## in config.yaml (llmsnap runs Network=host and reaches the model here). Each +## model gets its OWN port so two models can be resident at once (GPU sleep +## mode). Keep it in sync with config.yaml. +## +## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed +## executor) and does NOT set the API server port — reusing it as the API port +## collides with vLLM's internal usage. See +## https://docs.vllm.ai/en/stable/configuration/env_vars/ +## +COOKBOOK_VLLM_MODEL_PORT=5802 diff --git a/cookbooks/vllm/config/examples/models/smol135.yaml b/cookbooks/vllm/config/examples/models/smol135.yaml index 4ac7042..05d0f33 100644 --- a/cookbooks/vllm/config/examples/models/smol135.yaml +++ b/cookbooks/vllm/config/examples/models/smol135.yaml @@ -17,16 +17,6 @@ dtype: bfloat16 max-model-len: 4096 max-num-seqs: 4 -## -## Per-instance fixed host-loopback port for the smol135 model. -## -## Referenced by llmsnap's per-model -## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and -## reaches the model here). Each model gets its OWN port so two models can be -## resident at once (GPU sleep mode). Keep it in sync with config.yaml. -## -port: 5802 - # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager # mode skips it (fine for a smoke test). enforce-eager: true diff --git a/cookbooks/vllm/config/examples/vllm.env b/cookbooks/vllm/config/examples/vllm.env index 0996206..93d5c9e 100644 --- a/cookbooks/vllm/config/examples/vllm.env +++ b/cookbooks/vllm/config/examples/vllm.env @@ -6,11 +6,37 @@ ## # Hugging Face token, used to download gated / private model weights. -HF_TOKEN=hf_replace_me +#HF_TOKEN=hf_replace_me # Optional: speed up weight downloads (requires hf_transfer in the image). #HF_HUB_ENABLE_HF_TRANSFER=1 +# Keep vLLM's INTERNAL inter-worker traffic off the LAN. With Network=host, a +# multi-worker instance (tensor/pipeline parallel) otherwise exposes a fan of +# `VLLM::Worker` listeners on the primary routable IP. Two independent knobs are +# needed because vLLM binds these ports through two different backends, and ONLY +# the second one is what you actually see in `netstat`: +# +# 1. VLLM_HOST_IP -> vLLM's own ZMQ path (get_ip(), MessageQueue). In a +# single-node run its remote socket is not even bound (local readers use +# IPC), so this alone has NO visible effect — but it is the documented knob +# and correct to pin. https://docs.vllm.ai/en/stable/configuration/env_vars/ +# +# 2. GLOO_SOCKET_IFNAME / NCCL_SOCKET_IFNAME -> the torch distributed process +# group (gloo on CPU, nccl on GPU). vLLM builds ProcessGroupGloo WITHOUT an +# interface (distributed/utils.py: init_gloo_process_group), so gloo falls +# back to resolving the hostname and binds the default-route interface (the +# LAN IP). These are torch/NCCL env vars, NOT routed through VLLM_HOST_IP — +# this is THE fix for the `VLLM::Worker` listeners. `lo` = loopback iface. +# +# All valid ONLY because this is a SINGLE-NODE deployment (every worker on this +# host, loopback reachable). Do NOT set these on a real multi-node cluster. +# Distinct from the API server bind, which is --host/--port in +# vllm-model@.container (COOKBOOK_VLLM_MODEL_PORT). +VLLM_HOST_IP=127.0.0.1 +GLOO_SOCKET_IFNAME=lo +NCCL_SOCKET_IFNAME=lo + # VLLM container image # # CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used @@ -21,4 +47,4 @@ HF_TOKEN=hf_replace_me # architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64, # `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time. # See SPECS.md for the production (GPU) reference. -VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0 +COOKBOOK_VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0 diff --git a/cookbooks/vllm/vllm-model@.container b/cookbooks/vllm/vllm-model@.container index db86a33..6862a37 100644 --- a/cookbooks/vllm/vllm-model@.container +++ b/cookbooks/vllm/vllm-model@.container @@ -22,8 +22,9 @@ PartOf=vllm.target # `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE # this file: # - engine parameters -> /etc/quadlets/vllm/models/%i.yaml (mounted below) -# - fixed loopback port -> a per-instance drop-in sets PublishPort -# (dropins/vllm-model@%i.container.d/10-port.conf) +# - fixed loopback port -> COOKBOOK_VLLM_MODEL_PORT in models/%i.env, fed to +# vLLM's `--port` in Exec= below. This is a REAL API-server port (Network=host, +# so PublishPort does not apply and VLLM_PORT is NOT the API port). ContainerName=vllm-%i Image=vllm.image AutoUpdate=local @@ -45,14 +46,31 @@ Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z EnvironmentFile=/etc/quadlets/vllm/vllm.env EnvironmentFile=/etc/quadlets/vllm/models/%i.env -# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM -# reads every engine parameter from the mounted config file. -Exec=--config /etc/vllm/config.yaml +# The CPU image ENTRYPOINT is `vllm serve`; these flags are appended to it. vLLM +# reads every engine parameter from the mounted config file; the API server bind +# address is the ONE thing kept on the command line because it is per-instance +# (host loopback + fixed port) and CLI args passed after --config win. systemd +# expands ${COOKBOOK_VLLM_MODEL_PORT} from models/%i.env (loaded into the service +# environment in [Service] below), exactly like Image=vllm.image expands +# COOKBOOK_VLLM_IMAGE. --host 127.0.0.1 keeps the API off external interfaces +# (Network=host); Traefik + API key is the only front door. +Exec=--config /etc/vllm/config.yaml --host 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT} + +# Readiness: vLLM serves /health once the engine is up. +HealthCmd=python3 -c "import urllib.request, os; port = os.environ.get('COOKBOOK_VLLM_MODEL_PORT'); urllib.request.urlopen(f'http://127.0.0.1:{port}/health')" +HealthInterval=30s +HealthTimeout=10s +HealthStartPeriod=600s +HealthRetries=3 [Service] # vLLM cold start (weights download on first run + slow CPU load). llmsnap does # its own HTTP health polling on top (healthCheckTimeout in config.yaml). TimeoutStartSec=1800 -# Load the VLLM_IMAGE environment variable +# Load COOKBOOK_VLLM_IMAGE (for the Image=vllm.image indirection) and +# COOKBOOK_VLLM_MODEL_PORT (for the ${...} expansion in Exec=) into the SERVICE +# environment. systemd only expands variables it sees here; the [Container] +# EnvironmentFile= lines above feed the CONTAINER, not the unit's own expansion. EnvironmentFile=/etc/quadlets/vllm/vllm.env +EnvironmentFile=/etc/quadlets/vllm/models/%i.env diff --git a/cookbooks/vllm/vllm.image b/cookbooks/vllm/vllm.image index ce9dff6..23c8647 100644 --- a/cookbooks/vllm/vllm.image +++ b/cookbooks/vllm/vllm.image @@ -8,8 +8,8 @@ Documentation=https://docs.vllm.ai/ PartOf=vllm.target [Image] -Image=${VLLM_IMAGE} +Image=${COOKBOOK_VLLM_IMAGE} [Service] -# Load the VLLM_IMAGE environment variable +# Load the COOKBOOK_VLLM_IMAGE environment variable EnvironmentFile=/etc/quadlets/vllm/vllm.env