fix leaking environment variable polluting make variables in sub-makes

cookbooks/base/Makefile

@@ -23,9 +23,9 @@ pre-requisites::
 install-actions-pre::
 	@set -Eeuo pipefail; \
 	run() { echo $$*; "$$@"; }; \
-	if ! getent group itix-svc >/dev/null; then \
+	if ! getent group 10000 >/dev/null; then \
 		run groupadd -g 10000 itix-svc; \
 	fi ; \
-	if ! getent passwd itix-svc >/dev/null; then \
+	if ! getent passwd 10000 >/dev/null; then \
 		run useradd -u 10000 -g 10000 -M -d /tmp -c "ITIX Misc. Services" itix-svc; \
 	fi

cookbooks/base/dropins/container.d/10-wait-teardown.conf

@@ -0,0 +1,3 @@
+[Service]
+# Wait for the container to be removed before returning from stop
+ExecStopPost=/bin/sh -c 'cid=$(podman ps -aq --filter label=PODMAN_SYSTEMD_UNIT=%n); [ -n "$cid" ] && podman wait --ignore --condition removing "$cid"; exit 0'

cookbooks/vllm/README.md

@@ -93,14 +93,15 @@ as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true`
   > file. For a new model `<name>` you only add three small pieces:
   >
   > 1. `models/<name>.yaml` — its vLLM engine config (see below).
-  > 2. `vllm-model@<name>.container` — a **symlink** to `vllm-model@.container`
-  >    (so Quadlet instantiates the unit) **and** a
-  >    `dropins/vllm-model@<name>.container.d/10-port.conf` drop-in picking a new
-  >    fixed `PublishPort=127.0.0.1:58NN:8000`.
+  > 2. `models/<name>.env` — its own fixed host-loopback API port
+  >    `COOKBOOK_VLLM_MODEL_PORT=58NN` (fed to vLLM's `--host 127.0.0.1 --port`
+  >    by the shared template; the unit runs `Network=host`, so there is no
+  >    `PublishPort`). Do **not** call it `VLLM_PORT`: that variable is vLLM's
+  >    internal port, not the API port.
   > 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the
   >    `systemctl … vllm-model@<name>.service` `cmd`/`cmdStop`.
   >
-  > Keep the port in sync across the drop-in and `config.yaml`.
+  > Keep the port in sync across `models/<name>.env` and `config.yaml`.
 
 - `models/<name>.yaml` — the **vLLM engine configuration** for one model
   (`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model

cookbooks/vllm/config/examples/llmsnap/config.yaml

@@ -15,9 +15,11 @@
 # vllm-polkit.sh).
 #
 # Consequences for each model entry below:
-#   - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet
-#     publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic
-#     ${PORT}: a Quadlet cannot consume llmsnap's runtime port.
+#   - `proxy:` is REQUIRED and points at the FIXED host-loopback port the model's
+#     vLLM API server binds (`--host 127.0.0.1 --port 58NN`, from
+#     COOKBOOK_VLLM_MODEL_PORT in models/<model>.env; the unit runs Network=host
+#     so there is no PublishPort). We do NOT use llmsnap's dynamic ${PORT}: a
+#     Quadlet cannot consume llmsnap's runtime port.
 #   - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the
 #     command as a child process for the model's whole lifetime, so we:
 #       1. `systemctl start vllm-model@<model>.service` (returns once the container is up;
@@ -50,14 +52,14 @@ models:
   # Tiny real instruct model: has a chat template AND a tool-call parser, so it
   # exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts).
   qwen05:
-    proxy: http://127.0.0.1:5801                 # PublishPort of vllm-model@qwen05
+    proxy: http://127.0.0.1:5801                 # --port of vllm-model@qwen05 (COOKBOOK_VLLM_MODEL_PORT)
     cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done'
     cmdStop: systemctl stop vllm-model@qwen05.service
 
   # Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely
   # tested rather than reloading the same weights.
   smol135:
-    proxy: http://127.0.0.1:5802                 # PublishPort of vllm-model@smol135
+    proxy: http://127.0.0.1:5802                 # --port of vllm-model@smol135 (COOKBOOK_VLLM_MODEL_PORT)
     cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done'
     cmdStop: systemctl stop vllm-model@smol135.service
 

cookbooks/vllm/config/examples/models/qwen05.env

@@ -0,0 +1,16 @@
+##
+## Per-instance fixed host-loopback port for the qwen05 model's API server.
+##
+## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host
+## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's
+## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5801`
+## in config.yaml (llmsnap runs Network=host and reaches the model here). Each
+## model gets its OWN port so two models can be resident at once (GPU sleep
+## mode). Keep it in sync with config.yaml.
+##
+## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed
+## executor) and does NOT set the API server port — reusing it as the API port
+## collides with vLLM's internal usage. See
+## https://docs.vllm.ai/en/stable/configuration/env_vars/
+##
+COOKBOOK_VLLM_MODEL_PORT=5801

cookbooks/vllm/config/examples/models/qwen05.yaml

@@ -18,16 +18,6 @@ dtype: bfloat16
 max-model-len: 4096
 max-num-seqs: 4
 
-##
-## Per-instance fixed host-loopback port for the qwen05 model.
-##
-## Referenced by llmsnap's per-model
-## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and
-## reaches the model here). Each model gets its OWN port so two models can be
-## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
-##
-port: 5801
-
 # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
 # mode skips it (fine for a smoke test).
 enforce-eager: true

cookbooks/vllm/config/examples/models/smol135.env

@@ -0,0 +1,16 @@
+##
+## Per-instance fixed host-loopback port for the smol135 model's API server.
+##
+## Fed to vLLM's `--port` flag (see vllm-model@.container: `Exec=... --host
+## 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}`) and reused by the unit's
+## HealthCmd. Referenced by llmsnap's per-model `proxy: http://127.0.0.1:5802`
+## in config.yaml (llmsnap runs Network=host and reaches the model here). Each
+## model gets its OWN port so two models can be resident at once (GPU sleep
+## mode). Keep it in sync with config.yaml.
+##
+## NOT `VLLM_PORT`: that variable is for vLLM's INTERNAL use (ZMQ / distributed
+## executor) and does NOT set the API server port — reusing it as the API port
+## collides with vLLM's internal usage. See
+## https://docs.vllm.ai/en/stable/configuration/env_vars/
+##
+COOKBOOK_VLLM_MODEL_PORT=5802

cookbooks/vllm/config/examples/models/smol135.yaml

@@ -17,16 +17,6 @@ dtype: bfloat16
 max-model-len: 4096
 max-num-seqs: 4
 
-##
-## Per-instance fixed host-loopback port for the smol135 model.
-##
-## Referenced by llmsnap's per-model
-## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and
-## reaches the model here). Each model gets its OWN port so two models can be
-## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
-##
-port: 5802
-
 # --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
 # mode skips it (fine for a smoke test).
 enforce-eager: true

cookbooks/vllm/config/examples/vllm.env

@@ -6,11 +6,37 @@
 ##
 
 # Hugging Face token, used to download gated / private model weights.
-HF_TOKEN=hf_replace_me
+#HF_TOKEN=hf_replace_me
 
 # Optional: speed up weight downloads (requires hf_transfer in the image).
 #HF_HUB_ENABLE_HF_TRANSFER=1
 
+# Keep vLLM's INTERNAL inter-worker traffic off the LAN. With Network=host, a
+# multi-worker instance (tensor/pipeline parallel) otherwise exposes a fan of
+# `VLLM::Worker` listeners on the primary routable IP. Two independent knobs are
+# needed because vLLM binds these ports through two different backends, and ONLY
+# the second one is what you actually see in `netstat`:
+#
+#   1. VLLM_HOST_IP -> vLLM's own ZMQ path (get_ip(), MessageQueue). In a
+#      single-node run its remote socket is not even bound (local readers use
+#      IPC), so this alone has NO visible effect — but it is the documented knob
+#      and correct to pin. https://docs.vllm.ai/en/stable/configuration/env_vars/
+#
+#   2. GLOO_SOCKET_IFNAME / NCCL_SOCKET_IFNAME -> the torch distributed process
+#      group (gloo on CPU, nccl on GPU). vLLM builds ProcessGroupGloo WITHOUT an
+#      interface (distributed/utils.py: init_gloo_process_group), so gloo falls
+#      back to resolving the hostname and binds the default-route interface (the
+#      LAN IP). These are torch/NCCL env vars, NOT routed through VLLM_HOST_IP —
+#      this is THE fix for the `VLLM::Worker` listeners. `lo` = loopback iface.
+#
+# All valid ONLY because this is a SINGLE-NODE deployment (every worker on this
+# host, loopback reachable). Do NOT set these on a real multi-node cluster.
+# Distinct from the API server bind, which is --host/--port in
+# vllm-model@.container (COOKBOOK_VLLM_MODEL_PORT).
+VLLM_HOST_IP=127.0.0.1
+GLOO_SOCKET_IFNAME=lo
+NCCL_SOCKET_IFNAME=lo
+
 # VLLM container image
 #
 # CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used
@@ -21,4 +47,4 @@ HF_TOKEN=hf_replace_me
 # architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64,
 # `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time.
 # See SPECS.md for the production (GPU) reference.
-VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0
+COOKBOOK_VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0

cookbooks/vllm/dropins/vllm-model@.container.d/20-no-wait-teardown.conf

@@ -0,0 +1,17 @@
+[Service]
+# Opt these model units OUT of the global wait-teardown drop-in shipped by the
+# "base" cookbook (container.d/10-wait-teardown.conf), which makes `systemctl
+# stop` block until Podman has finished tearing the container down.
+#
+# That global wait keeps a unit in the `deactivating` state for the whole
+# teardown. It is the wrong behavior for the model units: llmsnap owns their
+# lifecycle and drives rapid stop/start swaps over D-Bus, keying off
+# `systemctl is-active` (which flips to inactive as soon as deactivation starts)
+# and off the exit code of `systemctl stop` (its cmdStop). A widened
+# `deactivating` window lets a subsequent `start` cancel the still-pending stop
+# job, so cmdStop returns non-zero, the swap fails and `make test` fails with it.
+#
+# An empty assignment clears the inherited ExecStopPost (the global wait).
+# Quadlet re-emits its own `podman rm` cleanup after the drop-ins, and the models
+# always run with --rm, so the container is still removed — we only drop the wait.
+ExecStopPost=

cookbooks/vllm/vllm-model@.container

@@ -22,8 +22,9 @@ PartOf=vllm.target
 # `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE
 # this file:
 #   - engine parameters  -> /etc/quadlets/vllm/models/%i.yaml (mounted below)
-#   - fixed loopback port -> a per-instance drop-in sets PublishPort
-#     (dropins/vllm-model@%i.container.d/10-port.conf)
+#   - fixed loopback port -> COOKBOOK_VLLM_MODEL_PORT in models/%i.env, fed to
+#     vLLM's `--port` in Exec= below. This is a REAL API-server port (Network=host,
+#     so PublishPort does not apply and VLLM_PORT is NOT the API port).
 ContainerName=vllm-%i
 Image=vllm.image
 AutoUpdate=local
@@ -45,14 +46,31 @@ Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z
 EnvironmentFile=/etc/quadlets/vllm/vllm.env
 EnvironmentFile=/etc/quadlets/vllm/models/%i.env
 
-# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM
-# reads every engine parameter from the mounted config file.
-Exec=--config /etc/vllm/config.yaml
+# The CPU image ENTRYPOINT is `vllm serve`; these flags are appended to it. vLLM
+# reads every engine parameter from the mounted config file; the API server bind
+# address is the ONE thing kept on the command line because it is per-instance
+# (host loopback + fixed port) and CLI args passed after --config win. systemd
+# expands ${COOKBOOK_VLLM_MODEL_PORT} from models/%i.env (loaded into the service
+# environment in [Service] below), exactly like Image=vllm.image expands
+# COOKBOOK_VLLM_IMAGE. --host 127.0.0.1 keeps the API off external interfaces
+# (Network=host); Traefik + API key is the only front door.
+Exec=--config /etc/vllm/config.yaml --host 127.0.0.1 --port ${COOKBOOK_VLLM_MODEL_PORT}
+
+# Readiness: vLLM serves /health once the engine is up.
+HealthCmd=python3 -c "import urllib.request, os; port = os.environ.get('COOKBOOK_VLLM_MODEL_PORT'); urllib.request.urlopen(f'http://127.0.0.1:{port}/health')"
+HealthInterval=30s
+HealthTimeout=10s
+HealthStartPeriod=600s
+HealthRetries=3
 
 [Service]
 # vLLM cold start (weights download on first run + slow CPU load). llmsnap does
 # its own HTTP health polling on top (healthCheckTimeout in config.yaml).
 TimeoutStartSec=1800
 
-# Load the VLLM_IMAGE environment variable
+# Load COOKBOOK_VLLM_IMAGE (for the Image=vllm.image indirection) and
+# COOKBOOK_VLLM_MODEL_PORT (for the ${...} expansion in Exec=) into the SERVICE
+# environment. systemd only expands variables it sees here; the [Container]
+# EnvironmentFile= lines above feed the CONTAINER, not the unit's own expansion.
 EnvironmentFile=/etc/quadlets/vllm/vllm.env
+EnvironmentFile=/etc/quadlets/vllm/models/%i.env

cookbooks/vllm/vllm.image

@@ -8,8 +8,8 @@ Documentation=https://docs.vllm.ai/
 PartOf=vllm.target
 
 [Image]
-Image=${VLLM_IMAGE}
+Image=${COOKBOOK_VLLM_IMAGE}
 
 [Service]
-# Load the VLLM_IMAGE environment variable
+# Load the COOKBOOK_VLLM_IMAGE environment variable
 EnvironmentFile=/etc/quadlets/vllm/vllm.env

scripts/butane.blocklist

@@ -11,3 +11,5 @@
 /etc/sysctl.d
 /etc/profile.d
 /etc/sysconfig
+/etc/polkit-1
+/etc/polkit-1/rules.d

scripts/common.mk

@@ -115,8 +115,8 @@ TARGET_EXAMPLES_PROFILED_FILES = $(patsubst profile.d/examples/%, $(TARGET_CHROO
 TARGET_EXAMPLES_POLKITD_FILES = $(patsubst polkit-rules.d/examples/%, $(TARGET_CHROOT)/etc/polkit-1/rules.d/60-%, $(EXAMPLES_POLKITD_FILES))
 
 # Example quadlet and systemd drop-ins files
-EXAMPLES_QUADLET_DROPINS_FILES := $(shell if [ -d dropins ]; then find dropins -mindepth 1 -type f | grep -E '\.(container|volume|network|pod|build|image)\.d/' 2>/dev/null; fi)
-EXAMPLES_SYSTEMD_DROPINS_FILES := $(shell if [ -d dropins ]; then find dropins -mindepth 1 -type f | grep -E '\.(service|target|timer|mount)\.d/' 2>/dev/null; fi)
+EXAMPLES_QUADLET_DROPINS_FILES := $(shell if [ -d dropins ]; then find dropins -mindepth 1 -type f | grep -E '\.?(container|volume|network|pod|build|image)\.d/' 2>/dev/null; fi)
+EXAMPLES_SYSTEMD_DROPINS_FILES := $(shell if [ -d dropins ]; then find dropins -mindepth 1 -type f | grep -E '\.?(service|target|timer|mount)\.d/' 2>/dev/null; fi)
 TARGET_EXAMPLES_QUADLET_DROPINS_FILES = $(patsubst dropins/%, $(TARGET_CHROOT)/etc/containers/systemd/%, $(EXAMPLES_QUADLET_DROPINS_FILES))
 TARGET_EXAMPLES_SYSTEMD_DROPINS_FILES = $(patsubst dropins/%, $(TARGET_CHROOT)/etc/systemd/system/%, $(EXAMPLES_SYSTEMD_DROPINS_FILES))
 
@@ -189,7 +189,7 @@ dryrun:
 	fi
 
 # Create the base directories needed for installation.
-$(TARGET_CHROOT)/etc/containers/systemd $(TARGET_CHROOT)/etc/systemd/system $(TARGET_CHROOT)/etc/tmpfiles.d $(TARGET_CHROOT)/etc/sysctl.d $(TARGET_CHROOT)/etc/profile.d:
+$(TARGET_CHROOT)/etc/containers/systemd $(TARGET_CHROOT)/etc/systemd/system $(TARGET_CHROOT)/etc/tmpfiles.d $(TARGET_CHROOT)/etc/sysctl.d $(TARGET_CHROOT)/etc/profile.d $(TARGET_CHROOT)/etc/polkit-1/rules.d:
 	install -D -d -m 0755 -o root -g root $@
 
 # Create the directory to store quadlet configuration files.
@@ -436,7 +436,12 @@ build/$(PROJECT_NAME).tar.gz build/$(PROJECT_NAME).bu build/$(PROJECT_NAME)-exam
 				# Dependency is up-to-date. \
 				continue; \
 			fi ; \
-			run $(MAKE) -C $(COOKBOOKS_DIR)/$$dep build/$$dep.ign build/$$dep-examples.ign ; \
+			# NB: this recipe exports SYSTEMD_{ENABLE,START}_UNITS (see the \
+			# target-specific `export` lines below the rule) for generate-tarball.sh. \
+			# Strip them here so the dependency's own `?=` recomputes them instead of \
+			# inheriting *our* units and mislabelling its metadata.json / tarball. \
+			run env -u SYSTEMD_ENABLE_UNITS -u SYSTEMD_START_UNITS -u SYSTEMD_MAIN_UNIT_NAMES \
+				$(MAKE) -C $(COOKBOOKS_DIR)/$$dep build/$$dep.ign build/$$dep-examples.ign ; \
 		done; \
 		run make install-config; \
 		YQ_FILES="$$(if [ -f "overlay.bu" ]; then echo "- overlay.bu"; else echo "-"; fi)"; \