fix health probe + listen address

cookbooks/vllm/README.md

@@ -93,14 +93,15 @@ as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true`
   > file. For a new model `<name>` you only add three small pieces:
   >
   > 1. `models/<name>.yaml` — its vLLM engine config (see below).
-  > 2. `vllm-model@<name>.container` — a **symlink** to `vllm-model@.container`
+  > 2. `models/<name>.env` — its own fixed host-loopback API port
-  >    (so Quadlet instantiates the unit) **and** a
+  >    `COOKBOOK_VLLM_MODEL_PORT=58NN` (fed to vLLM's `--host 127.0.0.1 --port`
-  >    `dropins/vllm-model@<name>.container.d/10-port.conf` drop-in picking a new
+  >    by the shared template; the unit runs `Network=host`, so there is no
-  >    fixed `PublishPort=127.0.0.1:58NN:8000`.
+  >    `PublishPort`). Do **not** call it `VLLM_PORT`: that variable is vLLM's
+  >    internal port, not the API port.
   > 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the
   >    `systemctl … vllm-model@<name>.service` `cmd`/`cmdStop`.
   >
-  > Keep the port in sync across the drop-in and `config.yaml`.
+  > Keep the port in sync across `models/<name>.env` and `config.yaml`.
 
 - `models/<name>.yaml` — the **vLLM engine configuration** for one model
   (`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model

cookbooks/vllm/config/examples/llmsnap/config.yaml

@@ -15,9 +15,11 @@
 # vllm-polkit.sh).
 #
 # Consequences for each model entry below:
-#   - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet
+#   - `proxy:` is REQUIRED and points at the FIXED host-loopback port the model's
-#     publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic
+#     vLLM API server binds (`--host 127.0.0.1 --port 58NN`, from
-#     ${PORT}: a Quadlet cannot consume llmsnap's runtime port.
+#     COOKBOOK_VLLM_MODEL_PORT in models/<model>.env; the unit runs Network=host
+#     so there is no PublishPort). We do NOT use llmsnap's dynamic ${PORT}: a
+#     Quadlet cannot consume llmsnap's runtime port.
 #   - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the
 #     command as a child process for the model's whole lifetime, so we:
 #       1. `systemctl start vllm-model@<model>.service` (returns once the container is up;
@@ -50,14 +52,14 @@ models:
   # Tiny real instruct model: has a chat template AND a tool-call parser, so it
   # exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts).
   qwen05:
-    proxy: http://127.0.0.1:5801                 # PublishPort of vllm-model@qwen05
+    proxy: http://127.0.0.1:5801                 # --port of vllm-model@qwen05 (COOKBOOK_VLLM_MODEL_PORT)
     cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done'
     cmdStop: systemctl stop vllm-model@qwen05.service
 
   # Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely
   # tested rather than reloading the same weights.
   smol135:
-    proxy: http://127.0.0.1:5802                 # PublishPort of vllm-model@smol135
+    proxy: http://127.0.0.1:5802                 # --port of vllm-model@smol135 (COOKBOOK_VLLM_MODEL_PORT)
     cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done'
     cmdStop: systemctl stop vllm-model@smol135.service
 

cookbooks/vllm/config/examples/models/qwen05.env

@@ -0,0 +1,16 @@
+##
+## Per-instance fixed host-loopback port for the qwen05 model's API server.
+##
+## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host
+## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's
+## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5801`
+## in config.yaml (llmsnap runs Network=host and reaches the model here). Each
+## model gets its OWN port so two models can be resident at once (GPU sleep
+## mode). Keep it in sync with config.yaml.
+##
+## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed
+## executor) and does NOT set the API server port — reusing it as the API port
+## collides with vLLM's internal usage. See
+## https://docs.vllm.ai/en/stable/configuration/env_vars/
+##
+COOKBOOK_VLLM_MODEL_PORT=5801

cookbooks/vllm/config/examples/models/qwen05.yaml

@@ -18,16 +18,6 @@ dtype: bfloat16
 max-model-len: 4096
 max-num-seqs: 4
 
-##
-## Per-instance fixed host-loopback port for the qwen05 model.
-##
-## Referenced by llmsnap's per-model
-## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and
-## reaches the model here). Each model gets its OWN port so two models can be
-## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
-##
-port: 5801
-
 # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
 # mode skips it (fine for a smoke test).
 enforce-eager: true

cookbooks/vllm/config/examples/models/smol135.env

@@ -0,0 +1,16 @@
+##
+## Per-instance fixed host-loopback port for the smol135 model's API server.
+##
+## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host
+## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's
+## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5802`
+## in config.yaml (llmsnap runs Network=host and reaches the model here). Each
+## model gets its OWN port so two models can be resident at once (GPU sleep
+## mode). Keep it in sync with config.yaml.
+##
+## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed
+## executor) and does NOT set the API server port — reusing it as the API port
+## collides with vLLM's internal usage. See
+## https://docs.vllm.ai/en/stable/configuration/env_vars/
+##
+COOKBOOK_VLLM_MODEL_PORT=5802

cookbooks/vllm/config/examples/models/smol135.yaml

@@ -17,16 +17,6 @@ dtype: bfloat16
 max-model-len: 4096
 max-num-seqs: 4
 
-##
-## Per-instance fixed host-loopback port for the smol135 model.
-##
-## Referenced by llmsnap's per-model
-## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and
-## reaches the model here). Each model gets its OWN port so two models can be
-## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
-##
-port: 5802
-
 # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
 # mode skips it (fine for a smoke test).
 enforce-eager: true

cookbooks/vllm/config/examples/vllm.env

@@ -6,11 +6,37 @@
 ##
 
 # Hugging Face token, used to download gated / private model weights.
-HF_TOKEN=hf_replace_me
+#HF_TOKEN=hf_replace_me
 
 # Optional: speed up weight downloads (requires hf_transfer in the image).
 #HF_HUB_ENABLE_HF_TRANSFER=1
 
+# Keep vLLM's INTERNAL inter-worker traffic off the LAN. With Network=host, a
+# multi-worker instance (tensor/pipeline parallel) otherwise exposes a fan of
+# `VLLM::Worker` listeners on the primary routable IP. Two independent knobs are
+# needed because vLLM binds these ports through two different backends, and ONLY
+# the second one is what you actually see in `netstat`:
+#
+#   1. VLLM_HOST_IP -> vLLM's own ZMQ path (get_ip(), MessageQueue). In a
+#      single-node run its remote socket is not even bound (local readers use
+#      IPC), so this alone has NO visible effect — but it is the documented knob
+#      and correct to pin. https://docs.vllm.ai/en/stable/configuration/env_vars/
+#
+#   2. GLOO_SOCKET_IFNAME / NCCL_SOCKET_IFNAME -> the torch distributed process
+#      group (gloo on CPU, nccl on GPU). vLLM builds ProcessGroupGloo WITHOUT an
+#      interface (distributed/utils.py: init_gloo_process_group), so gloo falls
+#      back to resolving the hostname and binds the default-route interface (the
+#      LAN IP). These are torch/NCCL env vars, NOT routed through VLLM_HOST_IP —
+#      this is THE fix for the `VLLM::Worker` listeners. `lo` = loopback iface.
+#
+# All valid ONLY because this is a SINGLE-NODE deployment (every worker on this
+# host, loopback reachable). Do NOT set these on a real multi-node cluster.
+# Distinct from the API server bind, which is --host/--port in
+# vllm-model@.container (COOKBOOK_VLLM_MODEL_PORT).
+VLLM_HOST_IP=127.0.0.1
+GLOO_SOCKET_IFNAME=lo
+NCCL_SOCKET_IFNAME=lo
+
 # VLLM container image
 #
 # CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used
@@ -21,4 +47,4 @@ HF_TOKEN=hf_replace_me
 # architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64,
 # `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time.
 # See SPECS.md for the production (GPU) reference.
-VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0
+COOKBOOK_VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0

cookbooks/vllm/vllm-model@.container

@@ -22,8 +22,9 @@ PartOf=vllm.target
 # `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE
 # this file:
 #   - engine parameters  -> /etc/quadlets/vllm/models/%i.yaml (mounted below)
-#   - fixed loopback port -> a per-instance drop-in sets PublishPort
+#   - fixed loopback port -> COOKBOOK_VLLM_MODEL_PORT in models/%i.env, fed to
-#     (dropins/vllm-model@%i.container.d/10-port.conf)
+#     vLLM's `--port` in Exec= below. This is a REAL API-server port (Network=host,
+#     so PublishPort does not apply and VLLM_PORT is NOT the API port).
 ContainerName=vllm-%i
 Image=vllm.image
 AutoUpdate=local
@@ -45,14 +46,31 @@ Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z
 EnvironmentFile=/etc/quadlets/vllm/vllm.env
 EnvironmentFile=/etc/quadlets/vllm/models/%i.env
 
-# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM
+# The CPU image ENTRYPOINT is `vllm serve`; these flags are appended to it. vLLM
-# reads every engine parameter from the mounted config file.
+# reads every engine parameter from the mounted config file; the API server bind
-Exec=--config /etc/vllm/config.yaml
+# address is the ONE thing kept on the command line because it is per-instance
+# (host loopback + fixed port) and CLI args passed after --config win. systemd
+# expands ${COOKBOOK_VLLM_MODEL_PORT} from models/%i.env (loaded into the service
+# environment in [Service] below), exactly like Image=vllm.image expands
+# COOKBOOK_VLLM_IMAGE. --host 127.0.0.1 keeps the API off external interfaces
+# (Network=host); Traefik + API key is the only front door.
+Exec=--config /etc/vllm/config.yaml --host 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}
+
+# Readiness: vLLM serves /health once the engine is up.
+HealthCmd=python3 -c "import urllib.request, os; port = os.environ.get('COOKBOOK_VLLM_MODEL_PORT'); urllib.request.urlopen(f'http://127.0.0.1:{port}/health')"
+HealthInterval=30s
+HealthTimeout=10s
+HealthStartPeriod=600s
+HealthRetries=3
 
 [Service]
 # vLLM cold start (weights download on first run + slow CPU load). llmsnap does
 # its own HTTP health polling on top (healthCheckTimeout in config.yaml).
 TimeoutStartSec=1800
 
-# Load the VLLM_IMAGE environment variable
+# Load COOKBOOK_VLLM_IMAGE (for the Image=vllm.image indirection) and
+# COOKBOOK_VLLM_MODEL_PORT (for the ${...} expansion in Exec=) into the SERVICE
+# environment. systemd only expands variables it sees here; the [Container]
+# EnvironmentFile= lines above feed the CONTAINER, not the unit's own expansion.
 EnvironmentFile=/etc/quadlets/vllm/vllm.env
+EnvironmentFile=/etc/quadlets/vllm/models/%i.env

cookbooks/vllm/vllm.image

@@ -8,8 +8,8 @@ Documentation=https://docs.vllm.ai/
 PartOf=vllm.target
 
 [Image]
-Image=${VLLM_IMAGE}
+Image=${COOKBOOK_VLLM_IMAGE}
 
 [Service]
-# Load the VLLM_IMAGE environment variable
+# Load the COOKBOOK_VLLM_IMAGE environment variable
 EnvironmentFile=/etc/quadlets/vllm/vllm.env