diff --git a/cookbooks/vllm/Makefile b/cookbooks/vllm/Makefile new file mode 100644 index 0000000..f584299 --- /dev/null +++ b/cookbooks/vllm/Makefile @@ -0,0 +1,123 @@ +## +## Makefile for the vLLM + llmsnap cookbook +## + +# vLLM quadlet is mapped to the 10032 user (vllm) and 10000 group (itix-svc) +PROJECT_UID = 10032 +PROJECT_GID = 10000 + +# The vLLM *models* run *rootful*: they need the NVIDIA GPU through CDI +# (nvidia.com/gpu=all) and --security-opt label=disable, which require root +# privileges (similar to the samba and vsftpd cookbooks). llmsnap itself, +# however, runs as a dedicated *non-root* container (User=8100): it is a pure +# control plane that only starts/stops the model units over D-Bus (see +# llmsnap.container and other/base/install-tools.d/vllm-polkit.sh). + +# - traefik: TLS termination (automatic Let's Encrypt certificate), API-key +# authentication and the /v1 path allowlist in front of llmsnap +# (see other/traefik/vllm.yaml). +DEPENDENCIES = traefik + +# Add empty directory for AI models and llmsnap config +TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/models +TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap + +# Include common Makefile +include ../../scripts/common.mk + +.PHONY: test + +# vLLM runs as root +$(TARGET_CHROOT)/etc/quadlets/vllm/models: + install -m 0755 -o root -g root -D -d $@ + +# But llmsnap runs as a dedicated non-root user +$(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap: + install -m 0700 -o $(PROJECT_UID) -g $(PROJECT_GID) -D -d $@ + +# All vLLM config files are owned by root +$(filter models/%.yaml, $(TARGET_CONFIG_FILES) $(TARGET_EXAMPLES_CONFIG_FILES)): + install -o root -g root -m 0644 $< $@ + +## +## `make test` — smoke-test the chain and measure model start/swap times. +## +## Talks straight to llmsnap on the host loopback (127.0.0.1:8000), bypassing +## Traefik and its API key. llmsnap holds each /v1/chat/completions request until +## the target model is actually serving, so the request's wall-clock time IS the +## time llmsnap needed to bring the model up: +## - COLD START: from every model unit stopped -> model serving. We stop all +## vllm-model@*.service first (needs root, hence `test: pre-requisites`) so +## the number is reproducible run to run. +## - SWAP: request the other model while one is loaded. llmsnap's exclusive +## group cold-swaps on CPU (stop previous unit, start next); on the GPU +## config it is a sleep/wake instead (see SPECS.md), which this same +## measurement captures transparently. +## +## The model list is discovered from llmsnap itself (GET /v1/models), so it stays +## in sync with config.yaml with no hard-coded names. +## + +# llmsnap control-plane endpoint (host loopback; bypasses Traefik / API key). +test: LLMSNAP_URL ?= http://127.0.0.1:8000 +# How long to wait for llmsnap to answer /v1/models before giving up. +test: TEST_READY_TIMEOUT ?= 60 +# Per-request cap: a first-run cold start also pulls the image + downloads +# weights + loads slowly on CPU (matches llmsnap's healthCheckTimeout). +test: TEST_LOAD_TIMEOUT ?= 1800 + +test: pre-requisites + @run() { echo "+ $$*" >&2; "$$@"; }; \ + set -Eeuo pipefail; \ + url="$(LLMSNAP_URL)"; \ + resp="$$(mktemp /tmp/vllm-test-resp-XXXXXX)"; \ + trap 'rm -f "$$resp"' EXIT; \ + echo "==> Waiting for llmsnap at $$url ..."; \ + deadline=$$(( $$(date +%s) + $(TEST_READY_TIMEOUT) )); \ + until curl -sSf -o /dev/null "$$url/v1/models"; do \ + if [ "$$(date +%s)" -ge "$$deadline" ]; then \ + echo "llmsnap did not become ready within $(TEST_READY_TIMEOUT)s" >&2; \ + exit 1; \ + fi; \ + sleep 2; \ + done; \ + models="$$(curl -sSf "$$url/v1/models" | yq -p=json '.data[].id')"; \ + if [ -z "$$models" ]; then echo "llmsnap exposes no models" >&2; exit 1; fi; \ + echo "llmsnap is ready; models: $$(echo $$models | tr '\n' ' ')"; \ + reset_models() { \ + local m; \ + for m in $$models; do run systemctl stop "vllm-model@$$m.service" || true; done; \ + sleep 5; \ + }; \ + timed_request() { \ + local m="$$1" start end code; \ + start="$$(date +%s.%N)"; \ + code="$$(curl -sS -o "$$resp" -w '%{http_code}' --max-time $(TEST_LOAD_TIMEOUT) \ + -X POST "$$url/v1/chat/completions" -H 'Content-Type: application/json' \ + -d "{\"model\":\"$$m\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1}")"; \ + end="$$(date +%s.%N)"; \ + if [ "$$code" != "200" ]; then \ + echo "request for '$$m' failed (HTTP $$code):" >&2; cat "$$resp" >&2; echo >&2; \ + exit 1; \ + fi; \ + awk -v s="$$start" -v e="$$end" 'BEGIN { printf "%.1f", e - s }'; \ + }; \ + echo; echo "==> Cold start times (each measured from all models stopped)"; \ + for m in $$models; do \ + reset_models; \ + printf ' cold start %-24s ' "$$m"; \ + echo "$$(timed_request "$$m")s"; \ + done; \ + echo; echo "==> Swap times (switch model while another is loaded)"; \ + reset_models; \ + first="$$(echo $$models | awk '{print $$1}')"; \ + echo " (warming up $$first ...)"; timed_request "$$first" >/dev/null; \ + n="$$(echo $$models | wc -w)"; prev="$$first"; swaps=0; \ + for m in $$models $$models; do \ + [ "$$m" = "$$prev" ] && continue; \ + printf ' swap %-16s -> %-16s ' "$$prev" "$$m"; \ + echo "$$(timed_request "$$m")s"; \ + prev="$$m"; swaps=$$((swaps + 1)); \ + [ "$$swaps" -ge "$$n" ] && break; \ + done; \ + echo; echo "All timing tests completed." diff --git a/cookbooks/vllm/README.md b/cookbooks/vllm/README.md new file mode 100644 index 0000000..a8b6457 --- /dev/null +++ b/cookbooks/vllm/README.md @@ -0,0 +1,181 @@ +# Podman Quadlet: vLLM + +## Overview + +This cookbook serves large language models with [vLLM](https://docs.vllm.ai/) on a +single NVIDIA GPU, and multiplexes several models on that GPU using +[llmsnap](https://github.com/napmany/llmsnap). + +Rather than a cold restart on every model switch (~128 s), llmsnap uses vLLM's +**sleep mode**: the outgoing model is put to sleep (its weights moved from VRAM to +host RAM) and woken up on demand (~11 s). Only one model is awake at a time, which +lets two large models share a 24 GB GPU (e.g. Qwen3-30B-A3B and Ministral-3-14B on +an L4). + +This cookbook: + +- Defines **each model as a first-class Podman Quadlet**, instantiated from a + single **templated** unit `vllm-model@.container` → `vllm-model@.service` + (e.g. `vllm-model@qwen05.service`) that references a shared `vllm.image`. Every + model parameter lives in a per-model engine config (`models/.yaml`), so + the instances differ only by their fixed loopback port (a small per-instance + drop-in). The models are **not** started at boot: llmsnap owns their lifecycle. +- Runs **llmsnap as a containerized, non-root control plane** (`llmsnap.container`, + `User=10032`, `Network=host`, bound to the **host loopback** `127.0.0.1:8000`). + The image is built locally (`llmsnap.build`) from the version-pinned, + checksum-verified llmsnap release. llmsnap no longer runs `podman` or touches + the GPU: it only **starts/stops the model units over the host D-Bus system bus** + (`systemctl start/stop vllm-model@.service`) and swaps between them via + sleep/wake. +- Grants that non-root control plane the right to manage **only** the + `vllm-*.service` units through a **polkit rule**. + Everything else on the host stays off-limits. +- Publishes the service through **Traefik** (requires the `traefik` cookbook), + which terminates TLS with an automatic Let's Encrypt certificate, authenticates + callers by **API key** (`Authorization: Bearer`, via the + `api-key-and-token-middleware` community plugin) and applies a strict `/v1/*` + path allowlist (`other/traefik/vllm.yaml`). + +> This is designed for a remote gateway (e.g. a separate LiteLLM host) to consume +> the models over the network, while llmsnap itself never leaves the loopback. + +## Privilege model (models rootful, llmsnap not) + +The **vLLM models run rootful**: accessing the NVIDIA GPU through CDI +(`--device nvidia.com/gpu=all`) together with `--security-opt label=disable` +requires root privileges, similar to the `samba` and `vsftpd` cookbooks. Each +`vllm-model@.service` is a rootful Podman container started by systemd. + +**llmsnap itself runs unprivileged.** Because the models are separate units, +llmsnap needs neither `podman` nor the GPU — it is a pure control plane. It runs +as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true`, +`NoNewPrivileges`) whose only host reach is the D-Bus system-bus socket. A scoped +**polkit rule** lets uid 10032 start/stop `vllm-*.service` and nothing else. + +> `SecurityLabelDisable=true` is set on `llmsnap.container` only so the confined +> container can reach the host D-Bus socket (SELinux `container_t` → `system_dbusd_t`). +> The privilege boundary is the non-root uid + the scoped polkit rule, not SELinux +> confinement. + +## Prerequisites + +- The `base` and `traefik` cookbooks (installed automatically as dependencies). + `base` provides the fedora base image used to build the llmsnap control-plane + image. +- A working **systemd + D-Bus + polkit** host (the default on Fedora CoreOS). The + containerized, non-root llmsnap drives the model units over the D-Bus system bus. +- An **NVIDIA GPU** with the driver + Container Toolkit and a working CDI spec at + `/etc/cdi/nvidia.yaml` (`nvidia.com/gpu=all`) — for the production (GPU) config. + This is host-level setup, out of scope of this cookbook. The shipped **CPU + smoke-test** config needs no GPU. **On a GPU-less VM the CPU models launch and + answer** (slowly); the CUDA production config is documented in `SPECS.md`. +- Configuration files provided by the user (copied from the examples): + - `/etc/quadlets/vllm/config.yaml` — the llmsnap model definitions. + - `/etc/quadlets/vllm/models/.yaml` — the vLLM engine configuration for + each model (`vllm serve --config`). + - `/etc/quadlets/vllm/vllm.env` — secrets (`HF_TOKEN`) injected into vLLM. +- Traefik must expose an `https` entryPoint and a `le` (Let's Encrypt) + `certificatesResolver`, and declare the `traefik-api-token-middleware` plugin + in its static config (`experimental.plugins`) — all provided by the traefik + cookbook example config. See `other/traefik/vllm.yaml` and the traefik cookbook. + +## Configuration + +- `config.yaml` — llmsnap configuration. Each model entry no longer holds a + `podman run` line; it holds a **`proxy:`** (the fixed loopback port its Quadlet + publishes) and a **`cmd:`/`cmdStop:`** that only `systemctl start/stop` the + matching `vllm-model@.service` (with a foreground `is-active` bridge — + see the header of the shipped `config.yaml`). The sleep/wake endpoints and the + swap group are unchanged. See . + + > **Adding/editing a model:** the model unit itself is a single templated + > Quadlet (`vllm-model@.container`), so you do **not** copy a whole container + > file. For a new model `` you only add three small pieces: + > + > 1. `models/.yaml` — its vLLM engine config (see below). + > 2. `vllm-model@.container` — a **symlink** to `vllm-model@.container` + > (so Quadlet instantiates the unit) **and** a + > `dropins/vllm-model@.container.d/10-port.conf` drop-in picking a new + > fixed `PublishPort=127.0.0.1:58NN:8000`. + > 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the + > `systemctl … vllm-model@.service` `cmd`/`cmdStop`. + > + > Keep the port in sync across the drop-in and `config.yaml`. + +- `models/.yaml` — the **vLLM engine configuration** for one model + (`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model + Quadlet mounts its file read-only at `/etc/vllm/config.yaml` and runs + `vllm serve --config …`, so every engine parameter lives here rather than on + the container command line. Keys are the long CLI flags with the leading `--` + stripped; store-true flags (e.g. `--enforce-eager`) become `true`. See + . + + > **Architecture note (GPU):** the CUDA image tag is architecture-specific + > (`:v0.24.0-aarch64` for arm64, `:v0.24.0` for amd64). Set the `VLLM_IMAGE` environment + > variable in `vllm.env` to match your host. + > The shipped `vllm.image` uses the multi-arch CPU smoke-test image. + +- `vllm.env` — secrets injected into every vLLM container as environment + variables (installed `0600 root:root`): + + | Variable | Purpose | + | ---------- | --------------------------------------------- | + | `HF_TOKEN` | Hugging Face token to download model weights | + +- `other/traefik/vllm.yaml` — Traefik router + the `vllm-api-token` middleware. + Edit the `Host(...)` FQDN and replace the placeholder API keys in the + middleware `tokens:` list (raw tokens, without the `Bearer ` prefix). Generate + a key with `echo "sk-$(openssl rand -hex 32)"`. + +## Data + +- `/var/lib/quadlets/vllm/cache` — Hugging Face weights cache. Re-downloadable, so + it is stored under `/var/lib/quadlets` (non-precious) and created by + `tmpfiles.d/vllm.conf`. + +## Ports + +- TCP `8000` — llmsnap, bound on `127.0.0.1` only, exposed through Traefik. +- TCP `5801`, `5802`, … — fixed vLLM model ports (host loopback, one per model, + published by each model's `vllm-model@.container.d` port drop-in and referenced by `proxy:` in + `config.yaml`). +- TCP `443` — public HTTPS ingress (Traefik). + +## UID / GID + +- **llmsnap** control plane: uid/gid **10032** (non-root `vllm` container + + system user; see "Privilege model" above). +- **vLLM models**: `root` (rootful Podman for GPU/CDI access). + +## Usage + +In a separate terminal, follow the logs. + +```sh +sudo make tail-logs +``` + +Install the cookbook (installs `base`, `traefik`, builds the llmsnap image, +installs the polkit rule and starts `vllm.target`). + +```sh +sudo make clean install +``` + +Check that llmsnap is up (once a model is awake): + +```sh +curl -sSf http://127.0.0.1:8000/v1/models +``` + +Then, through Traefik with a valid API key: + +```sh +curl -sSf -H 'Host: vllm' http://127.0.0.1/v1/models -H "Authorization: Bearer secret123" +``` + +Finally, remove the quadlets, their configuration and their data. + +```sh +sudo make uninstall clean +``` diff --git a/cookbooks/vllm/config/container/Containerfile b/cookbooks/vllm/config/container/Containerfile new file mode 100644 index 0000000..d3082dc --- /dev/null +++ b/cookbooks/vllm/config/container/Containerfile @@ -0,0 +1,42 @@ +# Minimal llmsnap control-plane image. +# +# In this cookbook llmsnap is a PURE CONTROL PLANE: it does NOT run the vLLM +# models itself. Each model is a separate Podman Quadlet — an instance of the +# vllm-model@.container template -> vllm-model@.service; llmsnap only +# starts/stops those systemd units over +# the host D-Bus system bus (see llmsnap.container and the polkit rule installed +# by other/base/install-tools.d/vllm-polkit.sh). +# +# The image therefore only needs: the llmsnap binary + `systemctl` (to drive the +# host units over D-Bus) + CA certificates. It needs NO podman and NO GPU, which +# is exactly what lets llmsnap run as an unprivileged, capability-dropped +# container. +FROM quay.io/fedora/fedora:44 + +# systemctl (systemd package) is invoked by the model `cmd`s in config.yaml to +# start/stop the vllm-model@.service units over D-Bus. curl/tar to fetch llmsnap. +RUN dnf install -y systemd ca-certificates curl tar gzip \ + && dnf clean all + +# Supply-chain hygiene: llmsnap is a low-visibility GitHub release. Pin the +# version and verify the tarball against a local copy of SHA256SUMS before trusting +# it. Architecture is detected from the build host (aarch64 -> arm64, x86_64 -> +# amd64), so the same Containerfile builds on both. +ARG LLMSNAP_VERSION=0.0.5 +ADD llmsnap_${LLMSNAP_VERSION}_checksums.txt /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt +RUN set -Eeuo pipefail; \ + declare -A arch_map=( [x86_64]=amd64 [aarch64]=arm64 ); \ + arch="${arch_map[$(uname -m)]:?unsupported architecture: $(uname -m)}"; \ + base="https://github.com/napmany/llmsnap/releases/download/v${LLMSNAP_VERSION}"; \ + tarball="llmsnap_${LLMSNAP_VERSION}_linux_${arch}.tar.gz"; \ + cd /tmp; \ + curl -sSfL -o "$tarball" "$base/$tarball"; \ + grep " $tarball\$" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt | sha256sum -c -; \ + tar -xzf "$tarball" -C /usr/local/bin llmsnap; \ + rm -f "$tarball" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt; \ + chmod 0755 /usr/local/bin/llmsnap; \ + /usr/local/bin/llmsnap --version + +# llmsnap flags are appended by the Quadlet (Exec=). +ENTRYPOINT ["/usr/local/bin/llmsnap"] +CMD [] diff --git a/cookbooks/vllm/config/container/llmsnap_0.0.5_checksums.txt b/cookbooks/vllm/config/container/llmsnap_0.0.5_checksums.txt new file mode 100644 index 0000000..51e4d57 --- /dev/null +++ b/cookbooks/vllm/config/container/llmsnap_0.0.5_checksums.txt @@ -0,0 +1,6 @@ +61983d3478654c9853f619ce56d67c34d96a671fb8d21cac826a165ab106810c llmsnap_0.0.5_darwin_amd64.tar.gz +0a72121d2c0818d7f5faf21db07cd81f9d1ce35412763fdb27fea9caf74d608d llmsnap_0.0.5_darwin_arm64.tar.gz +e2eb5a65cacc25583913c23ce27b35af2e315fc5fd6d795ce78f6b43279c7413 llmsnap_0.0.5_freebsd_amd64.tar.gz +57712924e0f9dbe7b9eae5056861e7d52d3aa644a52d23c78989c231b90ae4cc llmsnap_0.0.5_linux_amd64.tar.gz +adaa0882bb113a82e8a2072abdb0c7fb7c1c4ea1701511f4ef0d562ad5f40661 llmsnap_0.0.5_linux_arm64.tar.gz +a2c3dbb32baeb3cb0cc4eaeafcbb06aeafbb35951a56957c89e9e454a8dace88 llmsnap_0.0.5_windows_amd64.zip diff --git a/cookbooks/vllm/config/examples/llmsnap/config.yaml b/cookbooks/vllm/config/examples/llmsnap/config.yaml new file mode 100644 index 0000000..ca0c416 --- /dev/null +++ b/cookbooks/vllm/config/examples/llmsnap/config.yaml @@ -0,0 +1,72 @@ +# llmsnap — CPU smoke-test configuration. +# +# Purpose: validate the traefik -> llmsnap -> vLLM chain end to end WITHOUT a GPU. +# The models are intentionally tiny (0.5B / 135M) so they download and load fast and +# answer near-instantly on CPU. The point is NOT useful output, only that the chain +# is wired correctly (routing, API key, /v1/* allowlist, model swap). +# +# ARCHITECTURE (Q1 + Q3): unlike a plain llama-swap setup, the models are NOT +# launched by llmsnap as `podman run` children. Each model is a first-class Podman +# Quadlet — an instance of the shared vllm-model@.container template, i.e. +# vllm-model@.service (see vllm-model@.container). llmsnap runs as a +# CONTAINERIZED, NON-ROOT control plane (see llmsnap.container) and only +# starts/stops those systemd units over the host D-Bus system bus. A polkit rule +# authorizes it for `vllm-*.service` and nothing else (other/base/install-tools.d/ +# vllm-polkit.sh). +# +# Consequences for each model entry below: +# - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet +# publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic +# ${PORT}: a Quadlet cannot consume llmsnap's runtime port. +# - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the +# command as a child process for the model's whole lifetime, so we: +# 1. `systemctl start vllm-model@.service` (returns once the container is up; +# llmsnap then health-polls `proxy` until vLLM answers), then +# 2. block with `while systemctl -q is-active ...; do sleep 2; done` so the +# child stays alive exactly as long as the unit does. This uses ONLY D-Bus +# (no podman socket in the llmsnap container -> preserves least privilege). +# `is-active` is a read-only query (not gated by polkit); only start/stop are. +# - `cmdStop:` stops the unit; the container exits, `is-active` turns false, the +# bridge returns, and llmsnap considers the model stopped. +# +# Differences vs the production (GPU) config (see SPECS.md): +# - CPU vLLM image (vllm.image -> public.ecr.aws vLLM CI build), NOT the CUDA +# `vllm/vllm-openai` image; the model Quadlets carry no `--device +# nvidia.com/gpu=all` / `--security-opt label=disable`. +# - NO sleep mode: `--enable-sleep-mode` + /sleep + /wake_up are CUDA/ROCm ONLY. +# On CPU, llmsnap falls back to a plain cold swap (stop the previous unit, start +# the next), driven by the `exclusive: true` group below. +# - No GPU-only flags (--gpu-memory-utilization, --kv-cache-dtype fp8) and no +# VLLM_SERVER_DEV_MODE (its /sleep,/wake_up dev endpoints are GPU-only too). +# - Public models (no HF_TOKEN needed), so this works without vllm.env. +# - `--enforce-eager`: this CPU image's torch inductor JIT fails at warmup; eager +# mode skips it. See models/qwen05.yaml. + +healthCheckTimeout: 1800 # CPU is slow: also covers the FIRST-run image pull + # + HF weight download + slow CPU model load. +logLevel: info + +models: + # Tiny real instruct model: has a chat template AND a tool-call parser, so it + # exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts). + qwen05: + proxy: http://127.0.0.1:5801 # PublishPort of vllm-model@qwen05 + cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done' + cmdStop: systemctl stop vllm-model@qwen05.service + + # Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely + # tested rather than reloading the same weights. + smol135: + proxy: http://127.0.0.1:5802 # PublishPort of vllm-model@smol135 + cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done' + cmdStop: systemctl stop vllm-model@smol135.service + +groups: + cpu: + # No `swap:`/sleep on CPU. `exclusive: true` keeps a single model running at a + # time: switching to the other member cold-stops the previous unit (cmdStop) + # and starts the next. This validates llmsnap's swap sequencing on CPU. + exclusive: true + members: + - qwen05 + - smol135 diff --git a/cookbooks/vllm/config/examples/models/qwen05.env b/cookbooks/vllm/config/examples/models/qwen05.env new file mode 100644 index 0000000..e69de29 diff --git a/cookbooks/vllm/config/examples/models/qwen05.yaml b/cookbooks/vllm/config/examples/models/qwen05.yaml new file mode 100644 index 0000000..78cfa71 --- /dev/null +++ b/cookbooks/vllm/config/examples/models/qwen05.yaml @@ -0,0 +1,37 @@ +## +## vLLM engine configuration — qwen05 (CPU smoke test) +## +## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a +## long CLI flag with the leading `--` stripped; store-true flags (e.g. +## --enforce-eager) become `true`. CLI args passed after --config still win, so +## the Quadlet keeps only `--config` and lets this file carry every parameter. +## +## Installed to /etc/quadlets/vllm/models/qwen05.yaml and mounted read-only into +## the container at /etc/vllm/config.yaml. +## + +# Tiny real instruct model: has a chat template AND a tool-call parser, so it +# exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts). +model: Qwen/Qwen2.5-0.5B-Instruct +served-model-name: qwen05 +dtype: bfloat16 +max-model-len: 4096 +max-num-seqs: 4 + +## +## Per-instance fixed host-loopback port for the qwen05 model. +## +## Referenced by llmsnap's per-model +## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and +## reaches the model here). Each model gets its OWN port so two models can be +## resident at once (GPU sleep mode). Keep it in sync with config.yaml. +## +port: 5801 + +# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager +# mode skips it (fine for a smoke test). +enforce-eager: true + +# Tool-calling support (Qwen2 uses the hermes parser template). +enable-auto-tool-choice: true +tool-call-parser: hermes diff --git a/cookbooks/vllm/config/examples/models/smol135.env b/cookbooks/vllm/config/examples/models/smol135.env new file mode 100644 index 0000000..e69de29 diff --git a/cookbooks/vllm/config/examples/models/smol135.yaml b/cookbooks/vllm/config/examples/models/smol135.yaml new file mode 100644 index 0000000..4ac7042 --- /dev/null +++ b/cookbooks/vllm/config/examples/models/smol135.yaml @@ -0,0 +1,32 @@ +## +## vLLM engine configuration — smol135 (CPU smoke test) +## +## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a +## long CLI flag with the leading `--` stripped; store-true flags (e.g. +## --enforce-eager) become `true`. +## +## Installed to /etc/quadlets/vllm/models/smol135.yaml and mounted read-only into +## the container at /etc/vllm/config.yaml. +## + +# Ultra-light model of a DIFFERENT architecture (Llama vs Qwen2) so the swap is +# genuinely exercised. No tool-call parser (SmolLM has no tool template). +model: HuggingFaceTB/SmolLM2-135M-Instruct +served-model-name: smol135 +dtype: bfloat16 +max-model-len: 4096 +max-num-seqs: 4 + +## +## Per-instance fixed host-loopback port for the smol135 model. +## +## Referenced by llmsnap's per-model +## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and +## reaches the model here). Each model gets its OWN port so two models can be +## resident at once (GPU sleep mode). Keep it in sync with config.yaml. +## +port: 5802 + +# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager +# mode skips it (fine for a smoke test). +enforce-eager: true diff --git a/cookbooks/vllm/config/examples/vllm.env b/cookbooks/vllm/config/examples/vllm.env new file mode 100644 index 0000000..0996206 --- /dev/null +++ b/cookbooks/vllm/config/examples/vllm.env @@ -0,0 +1,24 @@ +## +## vLLM environment file +## +## Injected into every vLLM container launched by llmsnap (via --env-file in +## config.yaml). Installed with 0600 root:root permissions as it holds a secret. +## + +# Hugging Face token, used to download gated / private model weights. +HF_TOKEN=hf_replace_me + +# Optional: speed up weight downloads (requires hf_transfer in the image). +#HF_HUB_ENABLE_HF_TRANSFER=1 + +# VLLM container image +# +# CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used +# by the shipped config.yaml / model Quadlets to validate the whole chain WITHOUT +# a GPU. Its ENTRYPOINT is `vllm serve`, so the model Quadlets only append flags. +# +# For a real GPU deployment, swap this for the CUDA image, whose tag is +# architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64, +# `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time. +# See SPECS.md for the production (GPU) reference. +VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0 diff --git a/cookbooks/vllm/llmsnap.build b/cookbooks/vllm/llmsnap.build new file mode 100644 index 0000000..667268d --- /dev/null +++ b/cookbooks/vllm/llmsnap.build @@ -0,0 +1,11 @@ +[Unit] +Description=llmsnap control-plane image build (llmsnap + systemctl over D-Bus) +Documentation=https://github.com/napmany/llmsnap +Wants=network-online.target fedora-image.service +After=network-online.target fedora-image.service +Requires=fedora-image.service + +[Build] +File=/etc/quadlets/vllm/container/Containerfile +ImageTag=localhost/llmsnap:latest +SetWorkingDirectory=/etc/quadlets/vllm/container diff --git a/cookbooks/vllm/llmsnap.container b/cookbooks/vllm/llmsnap.container new file mode 100644 index 0000000..d0247cf --- /dev/null +++ b/cookbooks/vllm/llmsnap.container @@ -0,0 +1,73 @@ +[Unit] +Description=llmsnap — vLLM control plane (drives model units over D-Bus) +Documentation=https://github.com/napmany/llmsnap + +# Only start once the configuration has been provided by the user. +ConditionPathExists=/etc/quadlets/vllm/llmsnap/config.yaml + +# Build the image first; the system D-Bus bus must be up to reach systemd. +Wants=llmsnap-build.service network-online.target +After=llmsnap-build.service network-online.target dbus.service + +# Start/stop this unit when the target is started/stopped. +PartOf=vllm.target + +[Container] +ContainerName=llmsnap +Image=localhost/llmsnap:latest +AutoUpdate=local + +# Run as the vllm user (uid 10032) and itix-svc group (gid 10000). User 10032 (vllm) +# is authorized by polkit to manage the vllm-*.service model units over the host D-Bus bus. +User=10032 +Group=10000 + +# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik, +# which runs Network=host) and reach each model on the host loopback +# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would +# make 127.0.0.1 the container's own loopback, not the host's. +Network=host + +# Drive the host systemd over the system D-Bus bus. polkit authorizes uid 8100 +# for org.freedesktop.systemd1.manage-units on vllm-*.service ONLY. +Volume=/run/dbus/system_bus_socket:/run/dbus/system_bus_socket +Environment=DBUS_SYSTEM_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket + +# llmsnap configuration (read-only). +Volume=/etc/quadlets/vllm/llmsnap/config.yaml:/etc/llmsnap/config.yaml:ro,Z + +# SELinux: a confined container is denied access to the host D-Bus socket +# (container_t -> system_dbusd_t). Disabling the label keeps the socket reachable +# without shipping a custom SELinux module. The privilege boundary is enforced by +# the non-root uid + the scoped polkit rule, not by SELinux confinement here. +SecurityLabelDisable=true + +# Hardening: llmsnap is only an HTTP proxy + a D-Bus client. It needs no extra +# capabilities and no writable root filesystem. +NoNewPrivileges=true +DropCapability=ALL +ReadOnly=true +Tmpfs=/tmp + +# `systemctl` (the model cmd:/cmdStop: in config.yaml) refuses to talk to the +# host manager unless sd_booted() succeeds, i.e. unless /run/systemd/system +# EXISTS in this mount namespace. With ReadOnly=true, Podman mounts a fresh empty +# tmpfs over /run, so that path is absent and every systemctl call aborts early +# with "System has not been booted with systemd (PID 1)" / EHOSTDOWN — *before* +# it ever tries the D-Bus socket. This empty tmpfs just makes the directory +# exist so the check passes; the actual start/stop/is-active still go over the +# mounted host system bus (DBUS_SYSTEM_BUS_ADDRESS), gated by the polkit rule. +Tmpfs=/run/systemd/system + +# ENTRYPOINT is the llmsnap binary; these are its arguments. +Exec=--config /etc/llmsnap/config.yaml --listen 127.0.0.1:8000 + +[Service] +Restart=always +RestartSec=5 +# Give in-flight model swaps time to settle on shutdown. +TimeoutStopSec=60 + +[Install] +# Start with the target (llmsnap is the control plane; the models are not). +WantedBy=vllm.target diff --git a/cookbooks/vllm/other/traefik/vllm.yaml b/cookbooks/vllm/other/traefik/vllm.yaml new file mode 100644 index 0000000..74b8731 --- /dev/null +++ b/cookbooks/vllm/other/traefik/vllm.yaml @@ -0,0 +1,51 @@ +# vLLM (llmsnap) ingress — Traefik dynamic configuration. +# Deployed by the "traefik" cookbook hook to /etc/quadlets/traefik/conf.d/vllm.yaml. +# +# This replaces the original HAProxy front-end. It keeps the same security model: +# +# 1. TLS termination with an automatic Let's Encrypt certificate. +# 2. API-key authentication (Authorization: Bearer ), enforced by the +# "api-key-and-token-middleware" community plugin. A missing/invalid key is +# rejected with HTTP 403. +# 3. A strict path allowlist: only the OpenAI inference surface (/v1/*) is +# exposed. Everything else — llmsnap's /upstream/* passthrough (hence vLLM's +# /sleep, /wake_up, /collective_rpc in VLLM_SERVER_DEV_MODE) and the whole +# management/UI surface — is NOT routed and therefore returns 404. +# +# Prerequisites in the traefik cookbook: +# - the "traefik-api-token-middleware" plugin declared in traefik.yaml +# (experimental.plugins) — provided by the traefik cookbook example config. +http: + middlewares: + # Bearer-token authentication. Extracts the token from "Authorization: Bearer + # " and checks it against the allowlist below. On success the header is + # stripped before forwarding, so the key never reaches llmsnap/vLLM. + vllm-api-token: + plugin: + traefik-api-token-middleware: + bearerHeader: true + bearerHeaderName: "Authorization" + # Disable the X-API-TOKEN header form: OpenAI clients only send Bearer. + authenticationHeader: false + authenticationErrorMsg: "Access Denied" + removeHeadersOnSuccess: true + # Allowed API keys. Generate one with: echo "sk-$(openssl rand -hex 32)" + # List the RAW token here (without the "Bearer " prefix). + tokens: + - "secret123" + routers: + vllm: + # Adjust the Host to your public FQDN. Only /v1/* is routed (path allowlist). + rule: "Host(`vllm`) && PathPrefix(`/v1/`)" + entryPoints: + - http + middlewares: + - vllm-api-token + service: "vllm" + services: + vllm: + loadBalancer: + servers: + # llmsnap listens on the host loopback only; Traefik (Network=host) + # reaches it directly. llmsnap then proxies to the awake vLLM model. + - url: "http://127.0.0.1:8000" diff --git a/cookbooks/vllm/overlay.bu b/cookbooks/vllm/overlay.bu new file mode 100644 index 0000000..b519b6d --- /dev/null +++ b/cookbooks/vllm/overlay.bu @@ -0,0 +1,9 @@ +variant: fcos +version: 1.4.0 +passwd: + users: + - name: vllm + uid: 10032 + gecos: vLLM inference service + home_dir: /var/lib/quadlets/vllm + primary_group: itix-svc diff --git a/cookbooks/vllm/polkit-rules.d/vllm-llmsnap.rules b/cookbooks/vllm/polkit-rules.d/vllm-llmsnap.rules new file mode 100644 index 0000000..c17dda4 --- /dev/null +++ b/cookbooks/vllm/polkit-rules.d/vllm-llmsnap.rules @@ -0,0 +1,16 @@ +// Allow the llmsnap control-plane container (system user "llmsnap", uid 8100) to +// start/stop the vLLM model units over D-Bus, and nothing else. The default +// policy still applies to every other action and unit. +// +// Note: the ^vllm-.* glob also covers vllm-image.service (the image pull), which +// is harmless. It does NOT cover llmsnap.service itself (so llmsnap cannot +// restart/stop its own control plane) nor any non-vLLM unit on the host. +polkit.addRule(function(action, subject) { + if (action.id == "org.freedesktop.systemd1.manage-units" && + subject.user == "vllm") { + var unit = action.lookup("unit"); + if (unit && /^vllm-.*\.service$/.test(unit)) { + return polkit.Result.YES; + } + } +}); diff --git a/cookbooks/vllm/tmpfiles.d/vllm.conf b/cookbooks/vllm/tmpfiles.d/vllm.conf new file mode 100644 index 0000000..216bb64 --- /dev/null +++ b/cookbooks/vllm/tmpfiles.d/vllm.conf @@ -0,0 +1,5 @@ +# Hugging Face cache for the vLLM model weights. It is re-downloadable, hence +# stored under /var/lib/quadlets (non-precious data). Written by the rootful vLLM +# containers launched by llmsnap, so it is owned by root. +d /var/lib/quadlets/vllm 0755 root root - +d /var/lib/quadlets/vllm/cache 0755 root root - diff --git a/cookbooks/vllm/vllm-model@.container b/cookbooks/vllm/vllm-model@.container new file mode 100644 index 0000000..db86a33 --- /dev/null +++ b/cookbooks/vllm/vllm-model@.container @@ -0,0 +1,58 @@ +[Unit] +Description=vLLM model — %i (CPU smoke test) +Documentation=https://docs.vllm.ai/ + +# Only usable once the cookbook has been configured: this model's own engine config (models/%i.yaml, mounted below) must exist. +ConditionPathExists=/etc/quadlets/vllm/models/%i.yaml +ConditionPathExists=/etc/quadlets/vllm/models/%i.env + +# IMPORTANT: model units are deliberately NOT started at boot and are NOT pulled +# in by vllm.target (no [Install], and the target does not Want/Require them). +# llmsnap owns their lifecycle: it runs `systemctl start/stop +# vllm-model@%i.service` on demand and its swap group keeps a single model +# running at a time (a GPU only fits one). `PartOf=` only makes the model stop +# when the target stops. +PartOf=vllm.target + +# `Image=vllm.image` makes Quadlet add the dependency on vllm-image.service, so +# the image is pulled before the model starts. + +[Container] +# One template serves every model: systemd expands %i to the instance name (e.g. +# `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE +# this file: +# - engine parameters -> /etc/quadlets/vllm/models/%i.yaml (mounted below) +# - fixed loopback port -> a per-instance drop-in sets PublishPort +# (dropins/vllm-model@%i.container.d/10-port.conf) +ContainerName=vllm-%i +Image=vllm.image +AutoUpdate=local + +# Hugging Face weights cache (re-downloadable, non-precious). Shared by all models. +Volume=/var/lib/quadlets/vllm/cache:/root/.cache/huggingface:z +Environment=VLLM_CPU_KVCACHE_SPACE=4 +ShmSize=4g + +# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik, +# which runs Network=host) and reach each model on the host loopback +# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would +# make 127.0.0.1 the container's own loopback, not the host's. +Network=host + +# vLLM engine configuration (read-only). Every model parameter lives in this +# per-model file instead of on the command line; see config/examples/models/. +Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z +EnvironmentFile=/etc/quadlets/vllm/vllm.env +EnvironmentFile=/etc/quadlets/vllm/models/%i.env + +# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM +# reads every engine parameter from the mounted config file. +Exec=--config /etc/vllm/config.yaml + +[Service] +# vLLM cold start (weights download on first run + slow CPU load). llmsnap does +# its own HTTP health polling on top (healthCheckTimeout in config.yaml). +TimeoutStartSec=1800 + +# Load the VLLM_IMAGE environment variable +EnvironmentFile=/etc/quadlets/vllm/vllm.env diff --git a/cookbooks/vllm/vllm.image b/cookbooks/vllm/vllm.image new file mode 100644 index 0000000..ce9dff6 --- /dev/null +++ b/cookbooks/vllm/vllm.image @@ -0,0 +1,15 @@ +[Unit] +Description=podman pull vLLM image +Documentation=https://docs.vllm.ai/ + +# Pull/remove the image together with the vLLM target. The image is also pulled +# on demand: the vllm-model@.container template references `Image=vllm.image`, so each instance's +# generated service depends on vllm-image.service and triggers the pull. +PartOf=vllm.target + +[Image] +Image=${VLLM_IMAGE} + +[Service] +# Load the VLLM_IMAGE environment variable +EnvironmentFile=/etc/quadlets/vllm/vllm.env diff --git a/cookbooks/vllm/vllm.target b/cookbooks/vllm/vllm.target new file mode 100644 index 0000000..8cf9db4 --- /dev/null +++ b/cookbooks/vllm/vllm.target @@ -0,0 +1,15 @@ +[Unit] +Description=vLLM Inferencing Service +Documentation=man:systemd.target(5) + +# The control plane. The vLLM models (vllm-model@.service) are NOT started here: +# llmsnap starts/stops them on demand and its swap group keeps one running at a +# time. Starting the target brings up llmsnap; llmsnap brings up the models. +Requires=llmsnap.service +After=llmsnap.service + +# Allow isolation - can stop/start this target independently +AllowIsolate=yes + +[Install] +WantedBy=multi-user.target