add vllm cookbook

cookbooks/vllm/Makefile

@@ -0,0 +1,123 @@
+##
+## Makefile for the vLLM + llmsnap cookbook
+##
+
+# vLLM quadlet is mapped to the 10032 user (vllm) and 10000 group (itix-svc)
+PROJECT_UID = 10032
+PROJECT_GID = 10000
+
+# The vLLM *models* run *rootful*: they need the NVIDIA GPU through CDI
+# (nvidia.com/gpu=all) and --security-opt label=disable, which require root
+# privileges (similar to the samba and vsftpd cookbooks). llmsnap itself,
+# however, runs as a dedicated *non-root* container (User=8100): it is a pure
+# control plane that only starts/stops the model units over D-Bus (see
+# llmsnap.container and other/base/install-tools.d/vllm-polkit.sh).
+
+# - traefik: TLS termination (automatic Let's Encrypt certificate), API-key
+#            authentication and the /v1 path allowlist in front of llmsnap
+#            (see other/traefik/vllm.yaml).
+DEPENDENCIES = traefik
+
+# Add empty directory for AI models and llmsnap config
+TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/models
+TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap
+
+# Include common Makefile
+include ../../scripts/common.mk
+
+.PHONY: test
+
+# vLLM runs as root
+$(TARGET_CHROOT)/etc/quadlets/vllm/models:
+	install -m 0755 -o root -g root -D -d $@
+
+# But llmsnap runs as a dedicated non-root user
+$(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap:
+	install -m 0700 -o $(PROJECT_UID) -g $(PROJECT_GID) -D -d $@
+
+# All vLLM config files are owned by root
+$(filter models/%.yaml, $(TARGET_CONFIG_FILES) $(TARGET_EXAMPLES_CONFIG_FILES)):
+	install -o root -g root -m 0644 $< $@
+
+##
+## `make test` — smoke-test the chain and measure model start/swap times.
+##
+## Talks straight to llmsnap on the host loopback (127.0.0.1:8000), bypassing
+## Traefik and its API key. llmsnap holds each /v1/chat/completions request until
+## the target model is actually serving, so the request's wall-clock time IS the
+## time llmsnap needed to bring the model up:
+##   - COLD START: from every model unit stopped -> model serving. We stop all
+##     vllm-model@*.service first (needs root, hence `test: pre-requisites`) so
+##     the number is reproducible run to run.
+##   - SWAP: request the other model while one is loaded. llmsnap's exclusive
+##     group cold-swaps on CPU (stop previous unit, start next); on the GPU
+##     config it is a sleep/wake instead (see SPECS.md), which this same
+##     measurement captures transparently.
+##
+## The model list is discovered from llmsnap itself (GET /v1/models), so it stays
+## in sync with config.yaml with no hard-coded names.
+##
+
+# llmsnap control-plane endpoint (host loopback; bypasses Traefik / API key).
+test: LLMSNAP_URL ?= http://127.0.0.1:8000
+# How long to wait for llmsnap to answer /v1/models before giving up.
+test: TEST_READY_TIMEOUT ?= 60
+# Per-request cap: a first-run cold start also pulls the image + downloads
+# weights + loads slowly on CPU (matches llmsnap's healthCheckTimeout).
+test: TEST_LOAD_TIMEOUT ?= 1800
+
+test: pre-requisites
+	@run() { echo "+ $$*" >&2; "$$@"; }; \
+	set -Eeuo pipefail; \
+	url="$(LLMSNAP_URL)"; \
+	resp="$$(mktemp /tmp/vllm-test-resp-XXXXXX)"; \
+	trap 'rm -f "$$resp"' EXIT; \
+	echo "==> Waiting for llmsnap at $$url ..."; \
+	deadline=$$(( $$(date +%s) + $(TEST_READY_TIMEOUT) )); \
+	until curl -sSf -o /dev/null "$$url/v1/models"; do \
+		if [ "$$(date +%s)" -ge "$$deadline" ]; then \
+			echo "llmsnap did not become ready within $(TEST_READY_TIMEOUT)s" >&2; \
+			exit 1; \
+		fi; \
+		sleep 2; \
+	done; \
+	models="$$(curl -sSf "$$url/v1/models" | yq -p=json '.data[].id')"; \
+	if [ -z "$$models" ]; then echo "llmsnap exposes no models" >&2; exit 1; fi; \
+	echo "llmsnap is ready; models: $$(echo $$models | tr '\n' ' ')"; \
+	reset_models() { \
+		local m; \
+		for m in $$models; do run systemctl stop "vllm-model@$$m.service" || true; done; \
+		sleep 5; \
+	}; \
+	timed_request() { \
+		local m="$$1" start end code; \
+		start="$$(date +%s.%N)"; \
+		code="$$(curl -sS -o "$$resp" -w '%{http_code}' --max-time $(TEST_LOAD_TIMEOUT) \
+			-X POST "$$url/v1/chat/completions" -H 'Content-Type: application/json' \
+			-d "{\"model\":\"$$m\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1}")"; \
+		end="$$(date +%s.%N)"; \
+		if [ "$$code" != "200" ]; then \
+			echo "request for '$$m' failed (HTTP $$code):" >&2; cat "$$resp" >&2; echo >&2; \
+			exit 1; \
+		fi; \
+		awk -v s="$$start" -v e="$$end" 'BEGIN { printf "%.1f", e - s }'; \
+	}; \
+	echo; echo "==> Cold start times (each measured from all models stopped)"; \
+	for m in $$models; do \
+		reset_models; \
+		printf '  cold start  %-24s ' "$$m"; \
+		echo "$$(timed_request "$$m")s"; \
+	done; \
+	echo; echo "==> Swap times (switch model while another is loaded)"; \
+	reset_models; \
+	first="$$(echo $$models | awk '{print $$1}')"; \
+	echo "  (warming up $$first ...)"; timed_request "$$first" >/dev/null; \
+	n="$$(echo $$models | wc -w)"; prev="$$first"; swaps=0; \
+	for m in $$models $$models; do \
+		[ "$$m" = "$$prev" ] && continue; \
+		printf '  swap  %-16s -> %-16s ' "$$prev" "$$m"; \
+		echo "$$(timed_request "$$m")s"; \
+		prev="$$m"; swaps=$$((swaps + 1)); \
+		[ "$$swaps" -ge "$$n" ] && break; \
+	done; \
+	echo; echo "All timing tests completed."

cookbooks/vllm/README.md

@@ -0,0 +1,181 @@
+# Podman Quadlet: vLLM
+
+## Overview
+
+This cookbook serves large language models with [vLLM](https://docs.vllm.ai/) on a
+single NVIDIA GPU, and multiplexes several models on that GPU using
+[llmsnap](https://github.com/napmany/llmsnap).
+
+Rather than a cold restart on every model switch (~128 s), llmsnap uses vLLM's
+**sleep mode**: the outgoing model is put to sleep (its weights moved from VRAM to
+host RAM) and woken up on demand (~11 s). Only one model is awake at a time, which
+lets two large models share a 24 GB GPU (e.g. Qwen3-30B-A3B and Ministral-3-14B on
+an L4).
+
+This cookbook:
+
+- Defines **each model as a first-class Podman Quadlet**, instantiated from a
+  single **templated** unit `vllm-model@.container` → `vllm-model@<model>.service`
+  (e.g. `vllm-model@qwen05.service`) that references a shared `vllm.image`. Every
+  model parameter lives in a per-model engine config (`models/<model>.yaml`), so
+  the instances differ only by their fixed loopback port (a small per-instance
+  drop-in). The models are **not** started at boot: llmsnap owns their lifecycle.
+- Runs **llmsnap as a containerized, non-root control plane** (`llmsnap.container`,
+  `User=10032`, `Network=host`, bound to the **host loopback** `127.0.0.1:8000`).
+  The image is built locally (`llmsnap.build`) from the version-pinned,
+  checksum-verified llmsnap release. llmsnap no longer runs `podman` or touches
+  the GPU: it only **starts/stops the model units over the host D-Bus system bus**
+  (`systemctl start/stop vllm-model@<model>.service`) and swaps between them via
+  sleep/wake.
+- Grants that non-root control plane the right to manage **only** the
+  `vllm-*.service` units through a **polkit rule**.
+  Everything else on the host stays off-limits.
+- Publishes the service through **Traefik** (requires the `traefik` cookbook),
+  which terminates TLS with an automatic Let's Encrypt certificate, authenticates
+  callers by **API key** (`Authorization: Bearer`, via the
+  `api-key-and-token-middleware` community plugin) and applies a strict `/v1/*`
+  path allowlist (`other/traefik/vllm.yaml`).
+
+> This is designed for a remote gateway (e.g. a separate LiteLLM host) to consume
+> the models over the network, while llmsnap itself never leaves the loopback.
+
+## Privilege model (models rootful, llmsnap not)
+
+The **vLLM models run rootful**: accessing the NVIDIA GPU through CDI
+(`--device nvidia.com/gpu=all`) together with `--security-opt label=disable`
+requires root privileges, similar to the `samba` and `vsftpd` cookbooks. Each
+`vllm-model@<model>.service` is a rootful Podman container started by systemd.
+
+**llmsnap itself runs unprivileged.** Because the models are separate units,
+llmsnap needs neither `podman` nor the GPU — it is a pure control plane. It runs
+as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true`,
+`NoNewPrivileges`) whose only host reach is the D-Bus system-bus socket. A scoped
+**polkit rule** lets uid 10032 start/stop `vllm-*.service` and nothing else.
+
+> `SecurityLabelDisable=true` is set on `llmsnap.container` only so the confined
+> container can reach the host D-Bus socket (SELinux `container_t` → `system_dbusd_t`).
+> The privilege boundary is the non-root uid + the scoped polkit rule, not SELinux
+> confinement.
+
+## Prerequisites
+
+- The `base` and `traefik` cookbooks (installed automatically as dependencies).
+  `base` provides the fedora base image used to build the llmsnap control-plane
+  image.
+- A working **systemd + D-Bus + polkit** host (the default on Fedora CoreOS). The
+  containerized, non-root llmsnap drives the model units over the D-Bus system bus.
+- An **NVIDIA GPU** with the driver + Container Toolkit and a working CDI spec at
+  `/etc/cdi/nvidia.yaml` (`nvidia.com/gpu=all`) — for the production (GPU) config.
+  This is host-level setup, out of scope of this cookbook. The shipped **CPU
+  smoke-test** config needs no GPU. **On a GPU-less VM the CPU models launch and
+  answer** (slowly); the CUDA production config is documented in `SPECS.md`.
+- Configuration files provided by the user (copied from the examples):
+  - `/etc/quadlets/vllm/config.yaml` — the llmsnap model definitions.
+  - `/etc/quadlets/vllm/models/<name>.yaml` — the vLLM engine configuration for
+    each model (`vllm serve --config`).
+  - `/etc/quadlets/vllm/vllm.env` — secrets (`HF_TOKEN`) injected into vLLM.
+- Traefik must expose an `https` entryPoint and a `le` (Let's Encrypt)
+  `certificatesResolver`, and declare the `traefik-api-token-middleware` plugin
+  in its static config (`experimental.plugins`) — all provided by the traefik
+  cookbook example config. See `other/traefik/vllm.yaml` and the traefik cookbook.
+
+## Configuration
+
+- `config.yaml` — llmsnap configuration. Each model entry no longer holds a
+  `podman run` line; it holds a **`proxy:`** (the fixed loopback port its Quadlet
+  publishes) and a **`cmd:`/`cmdStop:`** that only `systemctl start/stop` the
+  matching `vllm-model@<model>.service` (with a foreground `is-active` bridge —
+  see the header of the shipped `config.yaml`). The sleep/wake endpoints and the
+  swap group are unchanged. See <https://github.com/napmany/llmsnap>.
+
+  > **Adding/editing a model:** the model unit itself is a single templated
+  > Quadlet (`vllm-model@.container`), so you do **not** copy a whole container
+  > file. For a new model `<name>` you only add three small pieces:
+  >
+  > 1. `models/<name>.yaml` — its vLLM engine config (see below).
+  > 2. `vllm-model@<name>.container` — a **symlink** to `vllm-model@.container`
+  >    (so Quadlet instantiates the unit) **and** a
+  >    `dropins/vllm-model@<name>.container.d/10-port.conf` drop-in picking a new
+  >    fixed `PublishPort=127.0.0.1:58NN:8000`.
+  > 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the
+  >    `systemctl … vllm-model@<name>.service` `cmd`/`cmdStop`.
+  >
+  > Keep the port in sync across the drop-in and `config.yaml`.
+
+- `models/<name>.yaml` — the **vLLM engine configuration** for one model
+  (`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model
+  Quadlet mounts its file read-only at `/etc/vllm/config.yaml` and runs
+  `vllm serve --config …`, so every engine parameter lives here rather than on
+  the container command line. Keys are the long CLI flags with the leading `--`
+  stripped; store-true flags (e.g. `--enforce-eager`) become `true`. See
+  <https://docs.vllm.ai/en/latest/configuration/serve_args.html>.
+
+  > **Architecture note (GPU):** the CUDA image tag is architecture-specific
+  > (`:v0.24.0-aarch64` for arm64, `:v0.24.0` for amd64). Set the `VLLM_IMAGE` environment
+  > variable in `vllm.env` to match your host.
+  > The shipped `vllm.image` uses the multi-arch CPU smoke-test image.
+
+- `vllm.env` — secrets injected into every vLLM container as environment
+  variables (installed `0600 root:root`):
+
+  | Variable   | Purpose                                       |
+  | ---------- | --------------------------------------------- |
+  | `HF_TOKEN` | Hugging Face token to download model weights  |
+
+- `other/traefik/vllm.yaml` — Traefik router + the `vllm-api-token` middleware.
+  Edit the `Host(...)` FQDN and replace the placeholder API keys in the
+  middleware `tokens:` list (raw tokens, without the `Bearer ` prefix). Generate
+  a key with `echo "sk-$(openssl rand -hex 32)"`.
+
+## Data
+
+- `/var/lib/quadlets/vllm/cache` — Hugging Face weights cache. Re-downloadable, so
+  it is stored under `/var/lib/quadlets` (non-precious) and created by
+  `tmpfiles.d/vllm.conf`.
+
+## Ports
+
+- TCP `8000` — llmsnap, bound on `127.0.0.1` only, exposed through Traefik.
+- TCP `5801`, `5802`, … — fixed vLLM model ports (host loopback, one per model,
+  published by each model's `vllm-model@<model>.container.d` port drop-in and referenced by `proxy:` in
+  `config.yaml`).
+- TCP `443` — public HTTPS ingress (Traefik).
+
+## UID / GID
+
+- **llmsnap** control plane: uid/gid **10032** (non-root `vllm` container +
+  system user; see "Privilege model" above).
+- **vLLM models**: `root` (rootful Podman for GPU/CDI access).
+
+## Usage
+
+In a separate terminal, follow the logs.
+
+```sh
+sudo make tail-logs
+```
+
+Install the cookbook (installs `base`, `traefik`, builds the llmsnap image,
+installs the polkit rule and starts `vllm.target`).
+
+```sh
+sudo make clean install
+```
+
+Check that llmsnap is up (once a model is awake):
+
+```sh
+curl -sSf http://127.0.0.1:8000/v1/models
+```
+
+Then, through Traefik with a valid API key:
+
+```sh
+curl -sSf -H 'Host: vllm' http://127.0.0.1/v1/models -H "Authorization: Bearer secret123"
+```
+
+Finally, remove the quadlets, their configuration and their data.
+
+```sh
+sudo make uninstall clean
+```

cookbooks/vllm/config/container/Containerfile

@@ -0,0 +1,42 @@
+# Minimal llmsnap control-plane image.
+#
+# In this cookbook llmsnap is a PURE CONTROL PLANE: it does NOT run the vLLM
+# models itself. Each model is a separate Podman Quadlet — an instance of the
+# vllm-model@.container template -> vllm-model@<model>.service; llmsnap only
+# starts/stops those systemd units over
+# the host D-Bus system bus (see llmsnap.container and the polkit rule installed
+# by other/base/install-tools.d/vllm-polkit.sh).
+#
+# The image therefore only needs: the llmsnap binary + `systemctl` (to drive the
+# host units over D-Bus) + CA certificates. It needs NO podman and NO GPU, which
+# is exactly what lets llmsnap run as an unprivileged, capability-dropped
+# container.
+FROM quay.io/fedora/fedora:44
+
+# systemctl (systemd package) is invoked by the model `cmd`s in config.yaml to
+# start/stop the vllm-model@<model>.service units over D-Bus. curl/tar to fetch llmsnap.
+RUN dnf install -y systemd ca-certificates curl tar gzip \
+ && dnf clean all
+
+# Supply-chain hygiene: llmsnap is a low-visibility GitHub release. Pin the
+# version and verify the tarball against a local copy of SHA256SUMS before trusting
+# it. Architecture is detected from the build host (aarch64 -> arm64, x86_64 ->
+# amd64), so the same Containerfile builds on both.
+ARG LLMSNAP_VERSION=0.0.5
+ADD llmsnap_${LLMSNAP_VERSION}_checksums.txt /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt
+RUN set -Eeuo pipefail; \
+    declare -A arch_map=( [x86_64]=amd64 [aarch64]=arm64 ); \
+    arch="${arch_map[$(uname -m)]:?unsupported architecture: $(uname -m)}"; \
+    base="https://github.com/napmany/llmsnap/releases/download/v${LLMSNAP_VERSION}"; \
+    tarball="llmsnap_${LLMSNAP_VERSION}_linux_${arch}.tar.gz"; \
+    cd /tmp; \
+    curl -sSfL -o "$tarball" "$base/$tarball"; \
+    grep " $tarball\$" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt | sha256sum -c -; \
+    tar -xzf "$tarball" -C /usr/local/bin llmsnap; \
+    rm -f "$tarball" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt; \
+    chmod 0755 /usr/local/bin/llmsnap; \
+    /usr/local/bin/llmsnap --version
+
+# llmsnap flags are appended by the Quadlet (Exec=).
+ENTRYPOINT ["/usr/local/bin/llmsnap"]
+CMD []

cookbooks/vllm/config/container/llmsnap_0.0.5_checksums.txt

@@ -0,0 +1,6 @@
+61983d3478654c9853f619ce56d67c34d96a671fb8d21cac826a165ab106810c  llmsnap_0.0.5_darwin_amd64.tar.gz
+0a72121d2c0818d7f5faf21db07cd81f9d1ce35412763fdb27fea9caf74d608d  llmsnap_0.0.5_darwin_arm64.tar.gz
+e2eb5a65cacc25583913c23ce27b35af2e315fc5fd6d795ce78f6b43279c7413  llmsnap_0.0.5_freebsd_amd64.tar.gz
+57712924e0f9dbe7b9eae5056861e7d52d3aa644a52d23c78989c231b90ae4cc  llmsnap_0.0.5_linux_amd64.tar.gz
+adaa0882bb113a82e8a2072abdb0c7fb7c1c4ea1701511f4ef0d562ad5f40661  llmsnap_0.0.5_linux_arm64.tar.gz
+a2c3dbb32baeb3cb0cc4eaeafcbb06aeafbb35951a56957c89e9e454a8dace88  llmsnap_0.0.5_windows_amd64.zip

cookbooks/vllm/config/examples/llmsnap/config.yaml

@@ -0,0 +1,72 @@
+# llmsnap — CPU smoke-test configuration.
+#
+# Purpose: validate the traefik -> llmsnap -> vLLM chain end to end WITHOUT a GPU.
+# The models are intentionally tiny (0.5B / 135M) so they download and load fast and
+# answer near-instantly on CPU. The point is NOT useful output, only that the chain
+# is wired correctly (routing, API key, /v1/* allowlist, model swap).
+#
+# ARCHITECTURE (Q1 + Q3): unlike a plain llama-swap setup, the models are NOT
+# launched by llmsnap as `podman run` children. Each model is a first-class Podman
+# Quadlet — an instance of the shared vllm-model@.container template, i.e.
+# vllm-model@<model>.service (see vllm-model@.container). llmsnap runs as a
+# CONTAINERIZED, NON-ROOT control plane (see llmsnap.container) and only
+# starts/stops those systemd units over the host D-Bus system bus. A polkit rule
+# authorizes it for `vllm-*.service` and nothing else (other/base/install-tools.d/
+# vllm-polkit.sh).
+#
+# Consequences for each model entry below:
+#   - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet
+#     publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic
+#     ${PORT}: a Quadlet cannot consume llmsnap's runtime port.
+#   - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the
+#     command as a child process for the model's whole lifetime, so we:
+#       1. `systemctl start vllm-model@<model>.service` (returns once the container is up;
+#          llmsnap then health-polls `proxy` until vLLM answers), then
+#       2. block with `while systemctl -q is-active ...; do sleep 2; done` so the
+#          child stays alive exactly as long as the unit does. This uses ONLY D-Bus
+#          (no podman socket in the llmsnap container -> preserves least privilege).
+#     `is-active` is a read-only query (not gated by polkit); only start/stop are.
+#   - `cmdStop:` stops the unit; the container exits, `is-active` turns false, the
+#     bridge returns, and llmsnap considers the model stopped.
+#
+# Differences vs the production (GPU) config (see SPECS.md):
+#   - CPU vLLM image (vllm.image -> public.ecr.aws vLLM CI build), NOT the CUDA
+#     `vllm/vllm-openai` image; the model Quadlets carry no `--device
+#     nvidia.com/gpu=all` / `--security-opt label=disable`.
+#   - NO sleep mode: `--enable-sleep-mode` + /sleep + /wake_up are CUDA/ROCm ONLY.
+#     On CPU, llmsnap falls back to a plain cold swap (stop the previous unit, start
+#     the next), driven by the `exclusive: true` group below.
+#   - No GPU-only flags (--gpu-memory-utilization, --kv-cache-dtype fp8) and no
+#     VLLM_SERVER_DEV_MODE (its /sleep,/wake_up dev endpoints are GPU-only too).
+#   - Public models (no HF_TOKEN needed), so this works without vllm.env.
+#   - `--enforce-eager`: this CPU image's torch inductor JIT fails at warmup; eager
+#     mode skips it. See models/qwen05.yaml.
+
+healthCheckTimeout: 1800         # CPU is slow: also covers the FIRST-run image pull
+                                 # + HF weight download + slow CPU model load.
+logLevel: info
+
+models:
+  # Tiny real instruct model: has a chat template AND a tool-call parser, so it
+  # exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts).
+  qwen05:
+    proxy: http://127.0.0.1:5801                 # PublishPort of vllm-model@qwen05
+    cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done'
+    cmdStop: systemctl stop vllm-model@qwen05.service
+
+  # Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely
+  # tested rather than reloading the same weights.
+  smol135:
+    proxy: http://127.0.0.1:5802                 # PublishPort of vllm-model@smol135
+    cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done'
+    cmdStop: systemctl stop vllm-model@smol135.service
+
+groups:
+  cpu:
+    # No `swap:`/sleep on CPU. `exclusive: true` keeps a single model running at a
+    # time: switching to the other member cold-stops the previous unit (cmdStop)
+    # and starts the next. This validates llmsnap's swap sequencing on CPU.
+    exclusive: true
+    members:
+      - qwen05
+      - smol135

cookbooks/vllm/config/examples/models/qwen05.yaml

@@ -0,0 +1,37 @@
+##
+## vLLM engine configuration — qwen05 (CPU smoke test)
+##
+## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a
+## long CLI flag with the leading `--` stripped; store-true flags (e.g.
+## --enforce-eager) become `true`. CLI args passed after --config still win, so
+## the Quadlet keeps only `--config` and lets this file carry every parameter.
+##
+## Installed to /etc/quadlets/vllm/models/qwen05.yaml and mounted read-only into
+## the container at /etc/vllm/config.yaml.
+##
+
+# Tiny real instruct model: has a chat template AND a tool-call parser, so it
+# exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts).
+model: Qwen/Qwen2.5-0.5B-Instruct
+served-model-name: qwen05
+dtype: bfloat16
+max-model-len: 4096
+max-num-seqs: 4
+
+##
+## Per-instance fixed host-loopback port for the qwen05 model.
+##
+## Referenced by llmsnap's per-model
+## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and
+## reaches the model here). Each model gets its OWN port so two models can be
+## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
+##
+port: 5801
+
+# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
+# mode skips it (fine for a smoke test).
+enforce-eager: true
+
+# Tool-calling support (Qwen2 uses the hermes parser template).
+enable-auto-tool-choice: true
+tool-call-parser: hermes

cookbooks/vllm/config/examples/models/smol135.yaml

@@ -0,0 +1,32 @@
+##
+## vLLM engine configuration — smol135 (CPU smoke test)
+##
+## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a
+## long CLI flag with the leading `--` stripped; store-true flags (e.g.
+## --enforce-eager) become `true`.
+##
+## Installed to /etc/quadlets/vllm/models/smol135.yaml and mounted read-only into
+## the container at /etc/vllm/config.yaml.
+##
+
+# Ultra-light model of a DIFFERENT architecture (Llama vs Qwen2) so the swap is
+# genuinely exercised. No tool-call parser (SmolLM has no tool template).
+model: HuggingFaceTB/SmolLM2-135M-Instruct
+served-model-name: smol135
+dtype: bfloat16
+max-model-len: 4096
+max-num-seqs: 4
+
+##
+## Per-instance fixed host-loopback port for the smol135 model.
+##
+## Referenced by llmsnap's per-model
+## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and
+## reaches the model here). Each model gets its OWN port so two models can be
+## resident at once (GPU sleep mode). Keep it in sync with config.yaml.
+##
+port: 5802
+
+# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager
+# mode skips it (fine for a smoke test).
+enforce-eager: true

cookbooks/vllm/config/examples/vllm.env

@@ -0,0 +1,24 @@
+##
+## vLLM environment file
+##
+## Injected into every vLLM container launched by llmsnap (via --env-file in
+## config.yaml). Installed with 0600 root:root permissions as it holds a secret.
+##
+
+# Hugging Face token, used to download gated / private model weights.
+HF_TOKEN=hf_replace_me
+
+# Optional: speed up weight downloads (requires hf_transfer in the image).
+#HF_HUB_ENABLE_HF_TRANSFER=1
+
+# VLLM container image
+#
+# CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used
+# by the shipped config.yaml / model Quadlets to validate the whole chain WITHOUT
+# a GPU. Its ENTRYPOINT is `vllm serve`, so the model Quadlets only append flags.
+#
+# For a real GPU deployment, swap this for the CUDA image, whose tag is
+# architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64,
+# `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time.
+# See SPECS.md for the production (GPU) reference.
+VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0

cookbooks/vllm/llmsnap.build

@@ -0,0 +1,11 @@
+[Unit]
+Description=llmsnap control-plane image build (llmsnap + systemctl over D-Bus)
+Documentation=https://github.com/napmany/llmsnap
+Wants=network-online.target fedora-image.service
+After=network-online.target fedora-image.service
+Requires=fedora-image.service
+
+[Build]
+File=/etc/quadlets/vllm/container/Containerfile
+ImageTag=localhost/llmsnap:latest
+SetWorkingDirectory=/etc/quadlets/vllm/container

cookbooks/vllm/llmsnap.container

@@ -0,0 +1,73 @@
+[Unit]
+Description=llmsnap — vLLM control plane (drives model units over D-Bus)
+Documentation=https://github.com/napmany/llmsnap
+
+# Only start once the configuration has been provided by the user.
+ConditionPathExists=/etc/quadlets/vllm/llmsnap/config.yaml
+
+# Build the image first; the system D-Bus bus must be up to reach systemd.
+Wants=llmsnap-build.service network-online.target
+After=llmsnap-build.service network-online.target dbus.service
+
+# Start/stop this unit when the target is started/stopped.
+PartOf=vllm.target
+
+[Container]
+ContainerName=llmsnap
+Image=localhost/llmsnap:latest
+AutoUpdate=local
+
+# Run as the vllm user (uid 10032) and itix-svc group (gid 10000). User 10032 (vllm)
+# is authorized by polkit to manage the vllm-*.service model units over the host D-Bus bus.
+User=10032
+Group=10000
+
+# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik,
+# which runs Network=host) and reach each model on the host loopback
+# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would
+# make 127.0.0.1 the container's own loopback, not the host's.
+Network=host
+
+# Drive the host systemd over the system D-Bus bus. polkit authorizes uid 8100
+# for org.freedesktop.systemd1.manage-units on vllm-*.service ONLY.
+Volume=/run/dbus/system_bus_socket:/run/dbus/system_bus_socket
+Environment=DBUS_SYSTEM_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket
+
+# llmsnap configuration (read-only).
+Volume=/etc/quadlets/vllm/llmsnap/config.yaml:/etc/llmsnap/config.yaml:ro,Z
+
+# SELinux: a confined container is denied access to the host D-Bus socket
+# (container_t -> system_dbusd_t). Disabling the label keeps the socket reachable
+# without shipping a custom SELinux module. The privilege boundary is enforced by
+# the non-root uid + the scoped polkit rule, not by SELinux confinement here.
+SecurityLabelDisable=true
+
+# Hardening: llmsnap is only an HTTP proxy + a D-Bus client. It needs no extra
+# capabilities and no writable root filesystem.
+NoNewPrivileges=true
+DropCapability=ALL
+ReadOnly=true
+Tmpfs=/tmp
+
+# `systemctl` (the model cmd:/cmdStop: in config.yaml) refuses to talk to the
+# host manager unless sd_booted() succeeds, i.e. unless /run/systemd/system
+# EXISTS in this mount namespace. With ReadOnly=true, Podman mounts a fresh empty
+# tmpfs over /run, so that path is absent and every systemctl call aborts early
+# with "System has not been booted with systemd (PID 1)" / EHOSTDOWN — *before*
+# it ever tries the D-Bus socket. This empty tmpfs just makes the directory
+# exist so the check passes; the actual start/stop/is-active still go over the
+# mounted host system bus (DBUS_SYSTEM_BUS_ADDRESS), gated by the polkit rule.
+Tmpfs=/run/systemd/system
+
+# ENTRYPOINT is the llmsnap binary; these are its arguments.
+Exec=--config /etc/llmsnap/config.yaml --listen 127.0.0.1:8000
+
+[Service]
+Restart=always
+RestartSec=5
+# Give in-flight model swaps time to settle on shutdown.
+TimeoutStopSec=60
+
+[Install]
+# Start with the target (llmsnap is the control plane; the models are not).
+WantedBy=vllm.target

cookbooks/vllm/other/traefik/vllm.yaml

@@ -0,0 +1,51 @@
+# vLLM (llmsnap) ingress — Traefik dynamic configuration.
+# Deployed by the "traefik" cookbook hook to /etc/quadlets/traefik/conf.d/vllm.yaml.
+#
+# This replaces the original HAProxy front-end. It keeps the same security model:
+#
+#   1. TLS termination with an automatic Let's Encrypt certificate.
+#   2. API-key authentication (Authorization: Bearer <key>), enforced by the
+#      "api-key-and-token-middleware" community plugin. A missing/invalid key is
+#      rejected with HTTP 403.
+#   3. A strict path allowlist: only the OpenAI inference surface (/v1/*) is
+#      exposed. Everything else — llmsnap's /upstream/* passthrough (hence vLLM's
+#      /sleep, /wake_up, /collective_rpc in VLLM_SERVER_DEV_MODE) and the whole
+#      management/UI surface — is NOT routed and therefore returns 404.
+#
+# Prerequisites in the traefik cookbook:
+#   - the "traefik-api-token-middleware" plugin declared in traefik.yaml
+#     (experimental.plugins) — provided by the traefik cookbook example config.
+http:
+  middlewares:
+    # Bearer-token authentication. Extracts the token from "Authorization: Bearer
+    # <token>" and checks it against the allowlist below. On success the header is
+    # stripped before forwarding, so the key never reaches llmsnap/vLLM.
+    vllm-api-token:
+      plugin:
+        traefik-api-token-middleware:
+          bearerHeader: true
+          bearerHeaderName: "Authorization"
+          # Disable the X-API-TOKEN header form: OpenAI clients only send Bearer.
+          authenticationHeader: false
+          authenticationErrorMsg: "Access Denied"
+          removeHeadersOnSuccess: true
+          # Allowed API keys. Generate one with: echo "sk-$(openssl rand -hex 32)"
+          # List the RAW token here (without the "Bearer " prefix).
+          tokens:
+            - "secret123"
+  routers:
+    vllm:
+      # Adjust the Host to your public FQDN. Only /v1/* is routed (path allowlist).
+      rule: "Host(`vllm`) && PathPrefix(`/v1/`)"
+      entryPoints:
+      - http
+      middlewares:
+      - vllm-api-token
+      service: "vllm"
+  services:
+    vllm:
+      loadBalancer:
+        servers:
+        # llmsnap listens on the host loopback only; Traefik (Network=host)
+        # reaches it directly. llmsnap then proxies to the awake vLLM model.
+        - url: "http://127.0.0.1:8000"

cookbooks/vllm/overlay.bu

@@ -0,0 +1,9 @@
+variant: fcos
+version: 1.4.0
+passwd:
+  users:
+  - name: vllm
+    uid: 10032
+    gecos: vLLM inference service
+    home_dir: /var/lib/quadlets/vllm
+    primary_group: itix-svc

cookbooks/vllm/polkit-rules.d/vllm-llmsnap.rules

@@ -0,0 +1,16 @@
+// Allow the llmsnap control-plane container (system user "llmsnap", uid 8100) to
+// start/stop the vLLM model units over D-Bus, and nothing else. The default
+// policy still applies to every other action and unit.
+//
+// Note: the ^vllm-.* glob also covers vllm-image.service (the image pull), which
+// is harmless. It does NOT cover llmsnap.service itself (so llmsnap cannot
+// restart/stop its own control plane) nor any non-vLLM unit on the host.
+polkit.addRule(function(action, subject) {
+    if (action.id == "org.freedesktop.systemd1.manage-units" &&
+        subject.user == "vllm") {
+        var unit = action.lookup("unit");
+        if (unit && /^vllm-.*\.service$/.test(unit)) {
+            return polkit.Result.YES;
+        }
+    }
+});

cookbooks/vllm/tmpfiles.d/vllm.conf

@@ -0,0 +1,5 @@
+# Hugging Face cache for the vLLM model weights. It is re-downloadable, hence
+# stored under /var/lib/quadlets (non-precious data). Written by the rootful vLLM
+# containers launched by llmsnap, so it is owned by root.
+d /var/lib/quadlets/vllm 0755 root root -
+d /var/lib/quadlets/vllm/cache 0755 root root -

cookbooks/vllm/vllm-model@.container

@@ -0,0 +1,58 @@
+[Unit]
+Description=vLLM model — %i (CPU smoke test)
+Documentation=https://docs.vllm.ai/
+
+# Only usable once the cookbook has been configured: this model's own engine config (models/%i.yaml, mounted below) must exist.
+ConditionPathExists=/etc/quadlets/vllm/models/%i.yaml
+ConditionPathExists=/etc/quadlets/vllm/models/%i.env
+
+# IMPORTANT: model units are deliberately NOT started at boot and are NOT pulled
+# in by vllm.target (no [Install], and the target does not Want/Require them).
+# llmsnap owns their lifecycle: it runs `systemctl start/stop
+# vllm-model@%i.service` on demand and its swap group keeps a single model
+# running at a time (a GPU only fits one). `PartOf=` only makes the model stop
+# when the target stops.
+PartOf=vllm.target
+
+# `Image=vllm.image` makes Quadlet add the dependency on vllm-image.service, so
+# the image is pulled before the model starts.
+
+[Container]
+# One template serves every model: systemd expands %i to the instance name (e.g.
+# `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE
+# this file:
+#   - engine parameters  -> /etc/quadlets/vllm/models/%i.yaml (mounted below)
+#   - fixed loopback port -> a per-instance drop-in sets PublishPort
+#     (dropins/vllm-model@%i.container.d/10-port.conf)
+ContainerName=vllm-%i
+Image=vllm.image
+AutoUpdate=local
+
+# Hugging Face weights cache (re-downloadable, non-precious). Shared by all models.
+Volume=/var/lib/quadlets/vllm/cache:/root/.cache/huggingface:z
+Environment=VLLM_CPU_KVCACHE_SPACE=4
+ShmSize=4g
+
+# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik,
+# which runs Network=host) and reach each model on the host loopback
+# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would
+# make 127.0.0.1 the container's own loopback, not the host's.
+Network=host
+
+# vLLM engine configuration (read-only). Every model parameter lives in this
+# per-model file instead of on the command line; see config/examples/models/.
+Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z
+EnvironmentFile=/etc/quadlets/vllm/vllm.env
+EnvironmentFile=/etc/quadlets/vllm/models/%i.env
+
+# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM
+# reads every engine parameter from the mounted config file.
+Exec=--config /etc/vllm/config.yaml
+
+[Service]
+# vLLM cold start (weights download on first run + slow CPU load). llmsnap does
+# its own HTTP health polling on top (healthCheckTimeout in config.yaml).
+TimeoutStartSec=1800
+
+# Load the VLLM_IMAGE environment variable
+EnvironmentFile=/etc/quadlets/vllm/vllm.env

cookbooks/vllm/vllm.image

@@ -0,0 +1,15 @@
+[Unit]
+Description=podman pull vLLM image
+Documentation=https://docs.vllm.ai/
+
+# Pull/remove the image together with the vLLM target. The image is also pulled
+# on demand: the vllm-model@.container template references `Image=vllm.image`, so each instance's
+# generated service depends on vllm-image.service and triggers the pull.
+PartOf=vllm.target
+
+[Image]
+Image=${VLLM_IMAGE}
+
+[Service]
+# Load the VLLM_IMAGE environment variable
+EnvironmentFile=/etc/quadlets/vllm/vllm.env

cookbooks/vllm/vllm.target

@@ -0,0 +1,15 @@
+[Unit]
+Description=vLLM Inferencing Service
+Documentation=man:systemd.target(5)
+
+# The control plane. The vLLM models (vllm-model@<model>.service) are NOT started here:
+# llmsnap starts/stops them on demand and its swap group keeps one running at a
+# time. Starting the target brings up llmsnap; llmsnap brings up the models.
+Requires=llmsnap.service
+After=llmsnap.service
+
+# Allow isolation - can stop/start this target independently
+AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target