Compare commits
3 Commits
2c9f6fc8f2
...
25f6177ddd
| Author | SHA1 | Date |
|---|---|---|
 Nicolas Massé
|
25f6177ddd | 2 weeks ago |
|
Nicolas Massé
|
3328fb6ca7 | 2 weeks ago |
|
Nicolas Massé
|
bd606ada30 | 2 weeks ago |
25 changed files with 896 additions and 22 deletions
@ -0,0 +1,123 @@ |
|||
##
|
|||
## Makefile for the vLLM + llmsnap cookbook
|
|||
##
|
|||
|
|||
# vLLM quadlet is mapped to the 10032 user (vllm) and 10000 group (itix-svc)
|
|||
PROJECT_UID = 10032 |
|||
PROJECT_GID = 10000 |
|||
|
|||
# The vLLM *models* run *rootful*: they need the NVIDIA GPU through CDI
|
|||
# (nvidia.com/gpu=all) and --security-opt label=disable, which require root
|
|||
# privileges (similar to the samba and vsftpd cookbooks). llmsnap itself,
|
|||
# however, runs as a dedicated *non-root* container (User=8100): it is a pure
|
|||
# control plane that only starts/stops the model units over D-Bus (see
|
|||
# llmsnap.container and other/base/install-tools.d/vllm-polkit.sh).
|
|||
|
|||
# - traefik: TLS termination (automatic Let's Encrypt certificate), API-key
|
|||
# authentication and the /v1 path allowlist in front of llmsnap
|
|||
# (see other/traefik/vllm.yaml).
|
|||
DEPENDENCIES = traefik |
|||
|
|||
# Add empty directory for AI models and llmsnap config
|
|||
TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/models |
|||
TARGET_FILES += $(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap |
|||
|
|||
# Include common Makefile
|
|||
include ../../scripts/common.mk |
|||
|
|||
.PHONY: test |
|||
|
|||
# vLLM runs as root
|
|||
$(TARGET_CHROOT)/etc/quadlets/vllm/models: |
|||
install -m 0755 -o root -g root -D -d $@ |
|||
|
|||
# But llmsnap runs as a dedicated non-root user
|
|||
$(TARGET_CHROOT)/etc/quadlets/vllm/llmsnap: |
|||
install -m 0700 -o $(PROJECT_UID) -g $(PROJECT_GID) -D -d $@ |
|||
|
|||
# All vLLM config files are owned by root
|
|||
$(filter models/%.yaml, $(TARGET_CONFIG_FILES) $(TARGET_EXAMPLES_CONFIG_FILES)): |
|||
install -o root -g root -m 0644 $< $@ |
|||
|
|||
##
|
|||
## `make test` — smoke-test the chain and measure model start/swap times.
|
|||
##
|
|||
## Talks straight to llmsnap on the host loopback (127.0.0.1:8000), bypassing
|
|||
## Traefik and its API key. llmsnap holds each /v1/chat/completions request until
|
|||
## the target model is actually serving, so the request's wall-clock time IS the
|
|||
## time llmsnap needed to bring the model up:
|
|||
## - COLD START: from every model unit stopped -> model serving. We stop all
|
|||
## vllm-model@*.service first (needs root, hence `test: pre-requisites`) so
|
|||
## the number is reproducible run to run.
|
|||
## - SWAP: request the other model while one is loaded. llmsnap's exclusive
|
|||
## group cold-swaps on CPU (stop previous unit, start next); on the GPU
|
|||
## config it is a sleep/wake instead (see SPECS.md), which this same
|
|||
## measurement captures transparently.
|
|||
##
|
|||
## The model list is discovered from llmsnap itself (GET /v1/models), so it stays
|
|||
## in sync with config.yaml with no hard-coded names.
|
|||
##
|
|||
|
|||
# llmsnap control-plane endpoint (host loopback; bypasses Traefik / API key).
|
|||
test: LLMSNAP_URL ?= http://127.0.0.1:8000 |
|||
# How long to wait for llmsnap to answer /v1/models before giving up.
|
|||
test: TEST_READY_TIMEOUT ?= 60 |
|||
# Per-request cap: a first-run cold start also pulls the image + downloads
|
|||
# weights + loads slowly on CPU (matches llmsnap's healthCheckTimeout).
|
|||
test: TEST_LOAD_TIMEOUT ?= 1800 |
|||
|
|||
test: pre-requisites |
|||
@run() { echo "+ $$*" >&2; "$$@"; }; \
|
|||
set -Eeuo pipefail; \
|
|||
url="$(LLMSNAP_URL)"; \
|
|||
resp="$$(mktemp /tmp/vllm-test-resp-XXXXXX)"; \
|
|||
trap 'rm -f "$$resp"' EXIT; \
|
|||
echo "==> Waiting for llmsnap at $$url ..."; \
|
|||
deadline=$$(( $$(date +%s) + $(TEST_READY_TIMEOUT) )); \
|
|||
until curl -sSf -o /dev/null "$$url/v1/models"; do \
|
|||
if [ "$$(date +%s)" -ge "$$deadline" ]; then \
|
|||
echo "llmsnap did not become ready within $(TEST_READY_TIMEOUT)s" >&2; \
|
|||
exit 1; \
|
|||
fi; \
|
|||
sleep 2; \
|
|||
done; \
|
|||
models="$$(curl -sSf "$$url/v1/models" | yq -p=json '.data[].id')"; \
|
|||
if [ -z "$$models" ]; then echo "llmsnap exposes no models" >&2; exit 1; fi; \
|
|||
echo "llmsnap is ready; models: $$(echo $$models | tr '\n' ' ')"; \
|
|||
reset_models() { \
|
|||
local m; \
|
|||
for m in $$models; do run systemctl stop "vllm-model@$$m.service" || true; done; \
|
|||
sleep 5; \
|
|||
}; \
|
|||
timed_request() { \
|
|||
local m="$$1" start end code; \
|
|||
start="$$(date +%s.%N)"; \
|
|||
code="$$(curl -sS -o "$$resp" -w '%{http_code}' --max-time $(TEST_LOAD_TIMEOUT) \
|
|||
-X POST "$$url/v1/chat/completions" -H 'Content-Type: application/json' \
|
|||
-d "{\"model\":\"$$m\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1}")"; \
|
|||
end="$$(date +%s.%N)"; \
|
|||
if [ "$$code" != "200" ]; then \
|
|||
echo "request for '$$m' failed (HTTP $$code):" >&2; cat "$$resp" >&2; echo >&2; \
|
|||
exit 1; \
|
|||
fi; \
|
|||
awk -v s="$$start" -v e="$$end" 'BEGIN { printf "%.1f", e - s }'; \
|
|||
}; \
|
|||
echo; echo "==> Cold start times (each measured from all models stopped)"; \
|
|||
for m in $$models; do \
|
|||
reset_models; \
|
|||
printf ' cold start %-24s ' "$$m"; \
|
|||
echo "$$(timed_request "$$m")s"; \
|
|||
done; \
|
|||
echo; echo "==> Swap times (switch model while another is loaded)"; \
|
|||
reset_models; \
|
|||
first="$$(echo $$models | awk '{print $$1}')"; \
|
|||
echo " (warming up $$first ...)"; timed_request "$$first" >/dev/null; \
|
|||
n="$$(echo $$models | wc -w)"; prev="$$first"; swaps=0; \
|
|||
for m in $$models $$models; do \
|
|||
[ "$$m" = "$$prev" ] && continue; \
|
|||
printf ' swap %-16s -> %-16s ' "$$prev" "$$m"; \
|
|||
echo "$$(timed_request "$$m")s"; \
|
|||
prev="$$m"; swaps=$$((swaps + 1)); \
|
|||
[ "$$swaps" -ge "$$n" ] && break; \
|
|||
done; \
|
|||
echo; echo "All timing tests completed." |
|||
@ -0,0 +1,181 @@ |
|||
# Podman Quadlet: vLLM |
|||
|
|||
## Overview |
|||
|
|||
This cookbook serves large language models with [vLLM](https://docs.vllm.ai/) on a |
|||
single NVIDIA GPU, and multiplexes several models on that GPU using |
|||
[llmsnap](https://github.com/napmany/llmsnap). |
|||
|
|||
Rather than a cold restart on every model switch (~128 s), llmsnap uses vLLM's |
|||
**sleep mode**: the outgoing model is put to sleep (its weights moved from VRAM to |
|||
host RAM) and woken up on demand (~11 s). Only one model is awake at a time, which |
|||
lets two large models share a 24 GB GPU (e.g. Qwen3-30B-A3B and Ministral-3-14B on |
|||
an L4). |
|||
|
|||
This cookbook: |
|||
|
|||
- Defines **each model as a first-class Podman Quadlet**, instantiated from a |
|||
single **templated** unit `vllm-model@.container` → `vllm-model@<model>.service` |
|||
(e.g. `vllm-model@qwen05.service`) that references a shared `vllm.image`. Every |
|||
model parameter lives in a per-model engine config (`models/<model>.yaml`), so |
|||
the instances differ only by their fixed loopback port (a small per-instance |
|||
drop-in). The models are **not** started at boot: llmsnap owns their lifecycle. |
|||
- Runs **llmsnap as a containerized, non-root control plane** (`llmsnap.container`, |
|||
`User=10032`, `Network=host`, bound to the **host loopback** `127.0.0.1:8000`). |
|||
The image is built locally (`llmsnap.build`) from the version-pinned, |
|||
checksum-verified llmsnap release. llmsnap no longer runs `podman` or touches |
|||
the GPU: it only **starts/stops the model units over the host D-Bus system bus** |
|||
(`systemctl start/stop vllm-model@<model>.service`) and swaps between them via |
|||
sleep/wake. |
|||
- Grants that non-root control plane the right to manage **only** the |
|||
`vllm-*.service` units through a **polkit rule**. |
|||
Everything else on the host stays off-limits. |
|||
- Publishes the service through **Traefik** (requires the `traefik` cookbook), |
|||
which terminates TLS with an automatic Let's Encrypt certificate, authenticates |
|||
callers by **API key** (`Authorization: Bearer`, via the |
|||
`api-key-and-token-middleware` community plugin) and applies a strict `/v1/*` |
|||
path allowlist (`other/traefik/vllm.yaml`). |
|||
|
|||
> This is designed for a remote gateway (e.g. a separate LiteLLM host) to consume |
|||
> the models over the network, while llmsnap itself never leaves the loopback. |
|||
|
|||
## Privilege model (models rootful, llmsnap not) |
|||
|
|||
The **vLLM models run rootful**: accessing the NVIDIA GPU through CDI |
|||
(`--device nvidia.com/gpu=all`) together with `--security-opt label=disable` |
|||
requires root privileges, similar to the `samba` and `vsftpd` cookbooks. Each |
|||
`vllm-model@<model>.service` is a rootful Podman container started by systemd. |
|||
|
|||
**llmsnap itself runs unprivileged.** Because the models are separate units, |
|||
llmsnap needs neither `podman` nor the GPU — it is a pure control plane. It runs |
|||
as a **non-root container** (`User=10032`, `DropCapability=ALL`, `ReadOnly=true`, |
|||
`NoNewPrivileges`) whose only host reach is the D-Bus system-bus socket. A scoped |
|||
**polkit rule** lets uid 10032 start/stop `vllm-*.service` and nothing else. |
|||
|
|||
> `SecurityLabelDisable=true` is set on `llmsnap.container` only so the confined |
|||
> container can reach the host D-Bus socket (SELinux `container_t` → `system_dbusd_t`). |
|||
> The privilege boundary is the non-root uid + the scoped polkit rule, not SELinux |
|||
> confinement. |
|||
|
|||
## Prerequisites |
|||
|
|||
- The `base` and `traefik` cookbooks (installed automatically as dependencies). |
|||
`base` provides the fedora base image used to build the llmsnap control-plane |
|||
image. |
|||
- A working **systemd + D-Bus + polkit** host (the default on Fedora CoreOS). The |
|||
containerized, non-root llmsnap drives the model units over the D-Bus system bus. |
|||
- An **NVIDIA GPU** with the driver + Container Toolkit and a working CDI spec at |
|||
`/etc/cdi/nvidia.yaml` (`nvidia.com/gpu=all`) — for the production (GPU) config. |
|||
This is host-level setup, out of scope of this cookbook. The shipped **CPU |
|||
smoke-test** config needs no GPU. **On a GPU-less VM the CPU models launch and |
|||
answer** (slowly); the CUDA production config is documented in `SPECS.md`. |
|||
- Configuration files provided by the user (copied from the examples): |
|||
- `/etc/quadlets/vllm/config.yaml` — the llmsnap model definitions. |
|||
- `/etc/quadlets/vllm/models/<name>.yaml` — the vLLM engine configuration for |
|||
each model (`vllm serve --config`). |
|||
- `/etc/quadlets/vllm/vllm.env` — secrets (`HF_TOKEN`) injected into vLLM. |
|||
- Traefik must expose an `https` entryPoint and a `le` (Let's Encrypt) |
|||
`certificatesResolver`, and declare the `traefik-api-token-middleware` plugin |
|||
in its static config (`experimental.plugins`) — all provided by the traefik |
|||
cookbook example config. See `other/traefik/vllm.yaml` and the traefik cookbook. |
|||
|
|||
## Configuration |
|||
|
|||
- `config.yaml` — llmsnap configuration. Each model entry no longer holds a |
|||
`podman run` line; it holds a **`proxy:`** (the fixed loopback port its Quadlet |
|||
publishes) and a **`cmd:`/`cmdStop:`** that only `systemctl start/stop` the |
|||
matching `vllm-model@<model>.service` (with a foreground `is-active` bridge — |
|||
see the header of the shipped `config.yaml`). The sleep/wake endpoints and the |
|||
swap group are unchanged. See <https://github.com/napmany/llmsnap>. |
|||
|
|||
> **Adding/editing a model:** the model unit itself is a single templated |
|||
> Quadlet (`vllm-model@.container`), so you do **not** copy a whole container |
|||
> file. For a new model `<name>` you only add three small pieces: |
|||
> |
|||
> 1. `models/<name>.yaml` — its vLLM engine config (see below). |
|||
> 2. `vllm-model@<name>.container` — a **symlink** to `vllm-model@.container` |
|||
> (so Quadlet instantiates the unit) **and** a |
|||
> `dropins/vllm-model@<name>.container.d/10-port.conf` drop-in picking a new |
|||
> fixed `PublishPort=127.0.0.1:58NN:8000`. |
|||
> 3. an entry in `config.yaml` with `proxy: http://127.0.0.1:58NN` and the |
|||
> `systemctl … vllm-model@<name>.service` `cmd`/`cmdStop`. |
|||
> |
|||
> Keep the port in sync across the drop-in and `config.yaml`. |
|||
|
|||
- `models/<name>.yaml` — the **vLLM engine configuration** for one model |
|||
(`model:`, `served-model-name:`, `dtype:`, `max-model-len:`, …). Each model |
|||
Quadlet mounts its file read-only at `/etc/vllm/config.yaml` and runs |
|||
`vllm serve --config …`, so every engine parameter lives here rather than on |
|||
the container command line. Keys are the long CLI flags with the leading `--` |
|||
stripped; store-true flags (e.g. `--enforce-eager`) become `true`. See |
|||
<https://docs.vllm.ai/en/latest/configuration/serve_args.html>. |
|||
|
|||
> **Architecture note (GPU):** the CUDA image tag is architecture-specific |
|||
> (`:v0.24.0-aarch64` for arm64, `:v0.24.0` for amd64). Set the `VLLM_IMAGE` environment |
|||
> variable in `vllm.env` to match your host. |
|||
> The shipped `vllm.image` uses the multi-arch CPU smoke-test image. |
|||
|
|||
- `vllm.env` — secrets injected into every vLLM container as environment |
|||
variables (installed `0600 root:root`): |
|||
|
|||
| Variable | Purpose | |
|||
| ---------- | --------------------------------------------- | |
|||
| `HF_TOKEN` | Hugging Face token to download model weights | |
|||
|
|||
- `other/traefik/vllm.yaml` — Traefik router + the `vllm-api-token` middleware. |
|||
Edit the `Host(...)` FQDN and replace the placeholder API keys in the |
|||
middleware `tokens:` list (raw tokens, without the `Bearer ` prefix). Generate |
|||
a key with `echo "sk-$(openssl rand -hex 32)"`. |
|||
|
|||
## Data |
|||
|
|||
- `/var/lib/quadlets/vllm/cache` — Hugging Face weights cache. Re-downloadable, so |
|||
it is stored under `/var/lib/quadlets` (non-precious) and created by |
|||
`tmpfiles.d/vllm.conf`. |
|||
|
|||
## Ports |
|||
|
|||
- TCP `8000` — llmsnap, bound on `127.0.0.1` only, exposed through Traefik. |
|||
- TCP `5801`, `5802`, … — fixed vLLM model ports (host loopback, one per model, |
|||
published by each model's `vllm-model@<model>.container.d` port drop-in and referenced by `proxy:` in |
|||
`config.yaml`). |
|||
- TCP `443` — public HTTPS ingress (Traefik). |
|||
|
|||
## UID / GID |
|||
|
|||
- **llmsnap** control plane: uid/gid **10032** (non-root `vllm` container + |
|||
system user; see "Privilege model" above). |
|||
- **vLLM models**: `root` (rootful Podman for GPU/CDI access). |
|||
|
|||
## Usage |
|||
|
|||
In a separate terminal, follow the logs. |
|||
|
|||
```sh |
|||
sudo make tail-logs |
|||
``` |
|||
|
|||
Install the cookbook (installs `base`, `traefik`, builds the llmsnap image, |
|||
installs the polkit rule and starts `vllm.target`). |
|||
|
|||
```sh |
|||
sudo make clean install |
|||
``` |
|||
|
|||
Check that llmsnap is up (once a model is awake): |
|||
|
|||
```sh |
|||
curl -sSf http://127.0.0.1:8000/v1/models |
|||
``` |
|||
|
|||
Then, through Traefik with a valid API key: |
|||
|
|||
```sh |
|||
curl -sSf -H 'Host: vllm' http://127.0.0.1/v1/models -H "Authorization: Bearer secret123" |
|||
``` |
|||
|
|||
Finally, remove the quadlets, their configuration and their data. |
|||
|
|||
```sh |
|||
sudo make uninstall clean |
|||
``` |
|||
@ -0,0 +1,42 @@ |
|||
# Minimal llmsnap control-plane image. |
|||
# |
|||
# In this cookbook llmsnap is a PURE CONTROL PLANE: it does NOT run the vLLM |
|||
# models itself. Each model is a separate Podman Quadlet — an instance of the |
|||
# vllm-model@.container template -> vllm-model@<model>.service; llmsnap only |
|||
# starts/stops those systemd units over |
|||
# the host D-Bus system bus (see llmsnap.container and the polkit rule installed |
|||
# by other/base/install-tools.d/vllm-polkit.sh). |
|||
# |
|||
# The image therefore only needs: the llmsnap binary + `systemctl` (to drive the |
|||
# host units over D-Bus) + CA certificates. It needs NO podman and NO GPU, which |
|||
# is exactly what lets llmsnap run as an unprivileged, capability-dropped |
|||
# container. |
|||
FROM quay.io/fedora/fedora:44 |
|||
|
|||
# systemctl (systemd package) is invoked by the model `cmd`s in config.yaml to |
|||
# start/stop the vllm-model@<model>.service units over D-Bus. curl/tar to fetch llmsnap. |
|||
RUN dnf install -y systemd ca-certificates curl tar gzip \ |
|||
&& dnf clean all |
|||
|
|||
# Supply-chain hygiene: llmsnap is a low-visibility GitHub release. Pin the |
|||
# version and verify the tarball against a local copy of SHA256SUMS before trusting |
|||
# it. Architecture is detected from the build host (aarch64 -> arm64, x86_64 -> |
|||
# amd64), so the same Containerfile builds on both. |
|||
ARG LLMSNAP_VERSION=0.0.5 |
|||
ADD llmsnap_${LLMSNAP_VERSION}_checksums.txt /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt |
|||
RUN set -Eeuo pipefail; \ |
|||
declare -A arch_map=( [x86_64]=amd64 [aarch64]=arm64 ); \ |
|||
arch="${arch_map[$(uname -m)]:?unsupported architecture: $(uname -m)}"; \ |
|||
base="https://github.com/napmany/llmsnap/releases/download/v${LLMSNAP_VERSION}"; \ |
|||
tarball="llmsnap_${LLMSNAP_VERSION}_linux_${arch}.tar.gz"; \ |
|||
cd /tmp; \ |
|||
curl -sSfL -o "$tarball" "$base/$tarball"; \ |
|||
grep " $tarball\$" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt | sha256sum -c -; \ |
|||
tar -xzf "$tarball" -C /usr/local/bin llmsnap; \ |
|||
rm -f "$tarball" /tmp/llmsnap_${LLMSNAP_VERSION}_checksums.txt; \ |
|||
chmod 0755 /usr/local/bin/llmsnap; \ |
|||
/usr/local/bin/llmsnap --version |
|||
|
|||
# llmsnap flags are appended by the Quadlet (Exec=). |
|||
ENTRYPOINT ["/usr/local/bin/llmsnap"] |
|||
CMD [] |
|||
@ -0,0 +1,6 @@ |
|||
61983d3478654c9853f619ce56d67c34d96a671fb8d21cac826a165ab106810c llmsnap_0.0.5_darwin_amd64.tar.gz |
|||
0a72121d2c0818d7f5faf21db07cd81f9d1ce35412763fdb27fea9caf74d608d llmsnap_0.0.5_darwin_arm64.tar.gz |
|||
e2eb5a65cacc25583913c23ce27b35af2e315fc5fd6d795ce78f6b43279c7413 llmsnap_0.0.5_freebsd_amd64.tar.gz |
|||
57712924e0f9dbe7b9eae5056861e7d52d3aa644a52d23c78989c231b90ae4cc llmsnap_0.0.5_linux_amd64.tar.gz |
|||
adaa0882bb113a82e8a2072abdb0c7fb7c1c4ea1701511f4ef0d562ad5f40661 llmsnap_0.0.5_linux_arm64.tar.gz |
|||
a2c3dbb32baeb3cb0cc4eaeafcbb06aeafbb35951a56957c89e9e454a8dace88 llmsnap_0.0.5_windows_amd64.zip |
|||
@ -0,0 +1,72 @@ |
|||
# llmsnap — CPU smoke-test configuration. |
|||
# |
|||
# Purpose: validate the traefik -> llmsnap -> vLLM chain end to end WITHOUT a GPU. |
|||
# The models are intentionally tiny (0.5B / 135M) so they download and load fast and |
|||
# answer near-instantly on CPU. The point is NOT useful output, only that the chain |
|||
# is wired correctly (routing, API key, /v1/* allowlist, model swap). |
|||
# |
|||
# ARCHITECTURE (Q1 + Q3): unlike a plain llama-swap setup, the models are NOT |
|||
# launched by llmsnap as `podman run` children. Each model is a first-class Podman |
|||
# Quadlet — an instance of the shared vllm-model@.container template, i.e. |
|||
# vllm-model@<model>.service (see vllm-model@.container). llmsnap runs as a |
|||
# CONTAINERIZED, NON-ROOT control plane (see llmsnap.container) and only |
|||
# starts/stops those systemd units over the host D-Bus system bus. A polkit rule |
|||
# authorizes it for `vllm-*.service` and nothing else (other/base/install-tools.d/ |
|||
# vllm-polkit.sh). |
|||
# |
|||
# Consequences for each model entry below: |
|||
# - `proxy:` is REQUIRED and points at the FIXED loopback port the Quadlet |
|||
# publishes (PublishPort=127.0.0.1:58NN:8000). We do NOT use llmsnap's dynamic |
|||
# ${PORT}: a Quadlet cannot consume llmsnap's runtime port. |
|||
# - `cmd:` is a FOREGROUND bridge, not the server itself. llmsnap supervises the |
|||
# command as a child process for the model's whole lifetime, so we: |
|||
# 1. `systemctl start vllm-model@<model>.service` (returns once the container is up; |
|||
# llmsnap then health-polls `proxy` until vLLM answers), then |
|||
# 2. block with `while systemctl -q is-active ...; do sleep 2; done` so the |
|||
# child stays alive exactly as long as the unit does. This uses ONLY D-Bus |
|||
# (no podman socket in the llmsnap container -> preserves least privilege). |
|||
# `is-active` is a read-only query (not gated by polkit); only start/stop are. |
|||
# - `cmdStop:` stops the unit; the container exits, `is-active` turns false, the |
|||
# bridge returns, and llmsnap considers the model stopped. |
|||
# |
|||
# Differences vs the production (GPU) config (see SPECS.md): |
|||
# - CPU vLLM image (vllm.image -> public.ecr.aws vLLM CI build), NOT the CUDA |
|||
# `vllm/vllm-openai` image; the model Quadlets carry no `--device |
|||
# nvidia.com/gpu=all` / `--security-opt label=disable`. |
|||
# - NO sleep mode: `--enable-sleep-mode` + /sleep + /wake_up are CUDA/ROCm ONLY. |
|||
# On CPU, llmsnap falls back to a plain cold swap (stop the previous unit, start |
|||
# the next), driven by the `exclusive: true` group below. |
|||
# - No GPU-only flags (--gpu-memory-utilization, --kv-cache-dtype fp8) and no |
|||
# VLLM_SERVER_DEV_MODE (its /sleep,/wake_up dev endpoints are GPU-only too). |
|||
# - Public models (no HF_TOKEN needed), so this works without vllm.env. |
|||
# - `--enforce-eager`: this CPU image's torch inductor JIT fails at warmup; eager |
|||
# mode skips it. See models/qwen05.yaml. |
|||
|
|||
healthCheckTimeout: 1800 # CPU is slow: also covers the FIRST-run image pull |
|||
# + HF weight download + slow CPU model load. |
|||
logLevel: info |
|||
|
|||
models: |
|||
# Tiny real instruct model: has a chat template AND a tool-call parser, so it |
|||
# exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts). |
|||
qwen05: |
|||
proxy: http://127.0.0.1:5801 # PublishPort of vllm-model@qwen05 |
|||
cmd: /bin/sh -c 'systemctl start vllm-model@qwen05.service && while systemctl -q is-active vllm-model@qwen05.service; do sleep 2; done' |
|||
cmdStop: systemctl stop vllm-model@qwen05.service |
|||
|
|||
# Ultra-light, DIFFERENT architecture (Llama vs Qwen2) so the swap is genuinely |
|||
# tested rather than reloading the same weights. |
|||
smol135: |
|||
proxy: http://127.0.0.1:5802 # PublishPort of vllm-model@smol135 |
|||
cmd: /bin/sh -c 'systemctl start vllm-model@smol135.service && while systemctl -q is-active vllm-model@smol135.service; do sleep 2; done' |
|||
cmdStop: systemctl stop vllm-model@smol135.service |
|||
|
|||
groups: |
|||
cpu: |
|||
# No `swap:`/sleep on CPU. `exclusive: true` keeps a single model running at a |
|||
# time: switching to the other member cold-stops the previous unit (cmdStop) |
|||
# and starts the next. This validates llmsnap's swap sequencing on CPU. |
|||
exclusive: true |
|||
members: |
|||
- qwen05 |
|||
- smol135 |
|||
@ -0,0 +1,37 @@ |
|||
## |
|||
## vLLM engine configuration — qwen05 (CPU smoke test) |
|||
## |
|||
## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a |
|||
## long CLI flag with the leading `--` stripped; store-true flags (e.g. |
|||
## --enforce-eager) become `true`. CLI args passed after --config still win, so |
|||
## the Quadlet keeps only `--config` and lets this file carry every parameter. |
|||
## |
|||
## Installed to /etc/quadlets/vllm/models/qwen05.yaml and mounted read-only into |
|||
## the container at /etc/vllm/config.yaml. |
|||
## |
|||
|
|||
# Tiny real instruct model: has a chat template AND a tool-call parser, so it |
|||
# exercises /v1/chat/completions (what the Traefik /v1/* allowlist fronts). |
|||
model: Qwen/Qwen2.5-0.5B-Instruct |
|||
served-model-name: qwen05 |
|||
dtype: bfloat16 |
|||
max-model-len: 4096 |
|||
max-num-seqs: 4 |
|||
|
|||
## |
|||
## Per-instance fixed host-loopback port for the qwen05 model. |
|||
## |
|||
## Referenced by llmsnap's per-model |
|||
## `proxy: http://127.0.0.1:5801` in config.yaml (llmsnap runs Network=host and |
|||
## reaches the model here). Each model gets its OWN port so two models can be |
|||
## resident at once (GPU sleep mode). Keep it in sync with config.yaml. |
|||
## |
|||
port: 5801 |
|||
|
|||
# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager |
|||
# mode skips it (fine for a smoke test). |
|||
enforce-eager: true |
|||
|
|||
# Tool-calling support (Qwen2 uses the hermes parser template). |
|||
enable-auto-tool-choice: true |
|||
tool-call-parser: hermes |
|||
@ -0,0 +1,32 @@ |
|||
## |
|||
## vLLM engine configuration — smol135 (CPU smoke test) |
|||
## |
|||
## Consumed by `vllm serve --config` (see vllm-model@.container). Each key is a |
|||
## long CLI flag with the leading `--` stripped; store-true flags (e.g. |
|||
## --enforce-eager) become `true`. |
|||
## |
|||
## Installed to /etc/quadlets/vllm/models/smol135.yaml and mounted read-only into |
|||
## the container at /etc/vllm/config.yaml. |
|||
## |
|||
|
|||
# Ultra-light model of a DIFFERENT architecture (Llama vs Qwen2) so the swap is |
|||
# genuinely exercised. No tool-call parser (SmolLM has no tool template). |
|||
model: HuggingFaceTB/SmolLM2-135M-Instruct |
|||
served-model-name: smol135 |
|||
dtype: bfloat16 |
|||
max-model-len: 4096 |
|||
max-num-seqs: 4 |
|||
|
|||
## |
|||
## Per-instance fixed host-loopback port for the smol135 model. |
|||
## |
|||
## Referenced by llmsnap's per-model |
|||
## `proxy: http://127.0.0.1:5802` in config.yaml (llmsnap runs Network=host and |
|||
## reaches the model here). Each model gets its OWN port so two models can be |
|||
## resident at once (GPU sleep mode). Keep it in sync with config.yaml. |
|||
## |
|||
port: 5802 |
|||
|
|||
# --enforce-eager: this CPU image's torch inductor JIT fails at warmup; eager |
|||
# mode skips it (fine for a smoke test). |
|||
enforce-eager: true |
|||
@ -0,0 +1,24 @@ |
|||
## |
|||
## vLLM environment file |
|||
## |
|||
## Injected into every vLLM container launched by llmsnap (via --env-file in |
|||
## config.yaml). Installed with 0600 root:root permissions as it holds a secret. |
|||
## |
|||
|
|||
# Hugging Face token, used to download gated / private model weights. |
|||
HF_TOKEN=hf_replace_me |
|||
|
|||
# Optional: speed up weight downloads (requires hf_transfer in the image). |
|||
#HF_HUB_ENABLE_HF_TRANSFER=1 |
|||
|
|||
# VLLM container image |
|||
# |
|||
# CPU inference image: the OFFICIAL vLLM CI build (multi-arch amd64/arm64), used |
|||
# by the shipped config.yaml / model Quadlets to validate the whole chain WITHOUT |
|||
# a GPU. Its ENTRYPOINT is `vllm serve`, so the model Quadlets only append flags. |
|||
# |
|||
# For a real GPU deployment, swap this for the CUDA image, whose tag is |
|||
# architecture-specific (`docker.io/vllm/vllm-openai:v0.24.0` on amd64, |
|||
# `:v0.24.0-aarch64` on arm64) — Podman cannot template it at generation time. |
|||
# See SPECS.md for the production (GPU) reference. |
|||
VLLM_IMAGE=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.24.0 |
|||
@ -0,0 +1,11 @@ |
|||
[Unit] |
|||
Description=llmsnap control-plane image build (llmsnap + systemctl over D-Bus) |
|||
Documentation=https://github.com/napmany/llmsnap |
|||
Wants=network-online.target fedora-image.service |
|||
After=network-online.target fedora-image.service |
|||
Requires=fedora-image.service |
|||
|
|||
[Build] |
|||
File=/etc/quadlets/vllm/container/Containerfile |
|||
ImageTag=localhost/llmsnap:latest |
|||
SetWorkingDirectory=/etc/quadlets/vllm/container |
|||
@ -0,0 +1,73 @@ |
|||
[Unit] |
|||
Description=llmsnap — vLLM control plane (drives model units over D-Bus) |
|||
Documentation=https://github.com/napmany/llmsnap |
|||
|
|||
# Only start once the configuration has been provided by the user. |
|||
ConditionPathExists=/etc/quadlets/vllm/llmsnap/config.yaml |
|||
|
|||
# Build the image first; the system D-Bus bus must be up to reach systemd. |
|||
Wants=llmsnap-build.service network-online.target |
|||
After=llmsnap-build.service network-online.target dbus.service |
|||
|
|||
# Start/stop this unit when the target is started/stopped. |
|||
PartOf=vllm.target |
|||
|
|||
[Container] |
|||
ContainerName=llmsnap |
|||
Image=localhost/llmsnap:latest |
|||
AutoUpdate=local |
|||
|
|||
# Run as the vllm user (uid 10032) and itix-svc group (gid 10000). User 10032 (vllm) |
|||
# is authorized by polkit to manage the vllm-*.service model units over the host D-Bus bus. |
|||
User=10032 |
|||
Group=10000 |
|||
|
|||
# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik, |
|||
# which runs Network=host) and reach each model on the host loopback |
|||
# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would |
|||
# make 127.0.0.1 the container's own loopback, not the host's. |
|||
Network=host |
|||
|
|||
# Drive the host systemd over the system D-Bus bus. polkit authorizes uid 8100 |
|||
# for org.freedesktop.systemd1.manage-units on vllm-*.service ONLY. |
|||
Volume=/run/dbus/system_bus_socket:/run/dbus/system_bus_socket |
|||
Environment=DBUS_SYSTEM_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket |
|||
|
|||
# llmsnap configuration (read-only). |
|||
Volume=/etc/quadlets/vllm/llmsnap/config.yaml:/etc/llmsnap/config.yaml:ro,Z |
|||
|
|||
# SELinux: a confined container is denied access to the host D-Bus socket |
|||
# (container_t -> system_dbusd_t). Disabling the label keeps the socket reachable |
|||
# without shipping a custom SELinux module. The privilege boundary is enforced by |
|||
# the non-root uid + the scoped polkit rule, not by SELinux confinement here. |
|||
SecurityLabelDisable=true |
|||
|
|||
# Hardening: llmsnap is only an HTTP proxy + a D-Bus client. It needs no extra |
|||
# capabilities and no writable root filesystem. |
|||
NoNewPrivileges=true |
|||
DropCapability=ALL |
|||
ReadOnly=true |
|||
Tmpfs=/tmp |
|||
|
|||
# `systemctl` (the model cmd:/cmdStop: in config.yaml) refuses to talk to the |
|||
# host manager unless sd_booted() succeeds, i.e. unless /run/systemd/system |
|||
# EXISTS in this mount namespace. With ReadOnly=true, Podman mounts a fresh empty |
|||
# tmpfs over /run, so that path is absent and every systemctl call aborts early |
|||
# with "System has not been booted with systemd (PID 1)" / EHOSTDOWN — *before* |
|||
# it ever tries the D-Bus socket. This empty tmpfs just makes the directory |
|||
# exist so the check passes; the actual start/stop/is-active still go over the |
|||
# mounted host system bus (DBUS_SYSTEM_BUS_ADDRESS), gated by the polkit rule. |
|||
Tmpfs=/run/systemd/system |
|||
|
|||
# ENTRYPOINT is the llmsnap binary; these are its arguments. |
|||
Exec=--config /etc/llmsnap/config.yaml --listen 127.0.0.1:8000 |
|||
|
|||
[Service] |
|||
Restart=always |
|||
RestartSec=5 |
|||
# Give in-flight model swaps time to settle on shutdown. |
|||
TimeoutStopSec=60 |
|||
|
|||
[Install] |
|||
# Start with the target (llmsnap is the control plane; the models are not). |
|||
WantedBy=vllm.target |
|||
@ -0,0 +1,51 @@ |
|||
# vLLM (llmsnap) ingress — Traefik dynamic configuration. |
|||
# Deployed by the "traefik" cookbook hook to /etc/quadlets/traefik/conf.d/vllm.yaml. |
|||
# |
|||
# This replaces the original HAProxy front-end. It keeps the same security model: |
|||
# |
|||
# 1. TLS termination with an automatic Let's Encrypt certificate. |
|||
# 2. API-key authentication (Authorization: Bearer <key>), enforced by the |
|||
# "api-key-and-token-middleware" community plugin. A missing/invalid key is |
|||
# rejected with HTTP 403. |
|||
# 3. A strict path allowlist: only the OpenAI inference surface (/v1/*) is |
|||
# exposed. Everything else — llmsnap's /upstream/* passthrough (hence vLLM's |
|||
# /sleep, /wake_up, /collective_rpc in VLLM_SERVER_DEV_MODE) and the whole |
|||
# management/UI surface — is NOT routed and therefore returns 404. |
|||
# |
|||
# Prerequisites in the traefik cookbook: |
|||
# - the "traefik-api-token-middleware" plugin declared in traefik.yaml |
|||
# (experimental.plugins) — provided by the traefik cookbook example config. |
|||
http: |
|||
middlewares: |
|||
# Bearer-token authentication. Extracts the token from "Authorization: Bearer |
|||
# <token>" and checks it against the allowlist below. On success the header is |
|||
# stripped before forwarding, so the key never reaches llmsnap/vLLM. |
|||
vllm-api-token: |
|||
plugin: |
|||
traefik-api-token-middleware: |
|||
bearerHeader: true |
|||
bearerHeaderName: "Authorization" |
|||
# Disable the X-API-TOKEN header form: OpenAI clients only send Bearer. |
|||
authenticationHeader: false |
|||
authenticationErrorMsg: "Access Denied" |
|||
removeHeadersOnSuccess: true |
|||
# Allowed API keys. Generate one with: echo "sk-$(openssl rand -hex 32)" |
|||
# List the RAW token here (without the "Bearer " prefix). |
|||
tokens: |
|||
- "secret123" |
|||
routers: |
|||
vllm: |
|||
# Adjust the Host to your public FQDN. Only /v1/* is routed (path allowlist). |
|||
rule: "Host(`vllm`) && PathPrefix(`/v1/`)" |
|||
entryPoints: |
|||
- http |
|||
middlewares: |
|||
- vllm-api-token |
|||
service: "vllm" |
|||
services: |
|||
vllm: |
|||
loadBalancer: |
|||
servers: |
|||
# llmsnap listens on the host loopback only; Traefik (Network=host) |
|||
# reaches it directly. llmsnap then proxies to the awake vLLM model. |
|||
- url: "http://127.0.0.1:8000" |
|||
@ -0,0 +1,9 @@ |
|||
variant: fcos |
|||
version: 1.4.0 |
|||
passwd: |
|||
users: |
|||
- name: vllm |
|||
uid: 10032 |
|||
gecos: vLLM inference service |
|||
home_dir: /var/lib/quadlets/vllm |
|||
primary_group: itix-svc |
|||
@ -0,0 +1,16 @@ |
|||
// Allow the llmsnap control-plane container (system user "llmsnap", uid 8100) to |
|||
// start/stop the vLLM model units over D-Bus, and nothing else. The default |
|||
// policy still applies to every other action and unit. |
|||
// |
|||
// Note: the ^vllm-.* glob also covers vllm-image.service (the image pull), which |
|||
// is harmless. It does NOT cover llmsnap.service itself (so llmsnap cannot |
|||
// restart/stop its own control plane) nor any non-vLLM unit on the host. |
|||
polkit.addRule(function(action, subject) { |
|||
if (action.id == "org.freedesktop.systemd1.manage-units" && |
|||
subject.user == "vllm") { |
|||
var unit = action.lookup("unit"); |
|||
if (unit && /^vllm-.*\.service$/.test(unit)) { |
|||
return polkit.Result.YES; |
|||
} |
|||
} |
|||
}); |
|||
@ -0,0 +1,5 @@ |
|||
# Hugging Face cache for the vLLM model weights. It is re-downloadable, hence |
|||
# stored under /var/lib/quadlets (non-precious data). Written by the rootful vLLM |
|||
# containers launched by llmsnap, so it is owned by root. |
|||
d /var/lib/quadlets/vllm 0755 root root - |
|||
d /var/lib/quadlets/vllm/cache 0755 root root - |
|||
@ -0,0 +1,58 @@ |
|||
[Unit] |
|||
Description=vLLM model — %i (CPU smoke test) |
|||
Documentation=https://docs.vllm.ai/ |
|||
|
|||
# Only usable once the cookbook has been configured: this model's own engine config (models/%i.yaml, mounted below) must exist. |
|||
ConditionPathExists=/etc/quadlets/vllm/models/%i.yaml |
|||
ConditionPathExists=/etc/quadlets/vllm/models/%i.env |
|||
|
|||
# IMPORTANT: model units are deliberately NOT started at boot and are NOT pulled |
|||
# in by vllm.target (no [Install], and the target does not Want/Require them). |
|||
# llmsnap owns their lifecycle: it runs `systemctl start/stop |
|||
# vllm-model@%i.service` on demand and its swap group keeps a single model |
|||
# running at a time (a GPU only fits one). `PartOf=` only makes the model stop |
|||
# when the target stops. |
|||
PartOf=vllm.target |
|||
|
|||
# `Image=vllm.image` makes Quadlet add the dependency on vllm-image.service, so |
|||
# the image is pulled before the model starts. |
|||
|
|||
[Container] |
|||
# One template serves every model: systemd expands %i to the instance name (e.g. |
|||
# `qwen05` for vllm-model@qwen05.service). The per-model differences live OUTSIDE |
|||
# this file: |
|||
# - engine parameters -> /etc/quadlets/vllm/models/%i.yaml (mounted below) |
|||
# - fixed loopback port -> a per-instance drop-in sets PublishPort |
|||
# (dropins/vllm-model@%i.container.d/10-port.conf) |
|||
ContainerName=vllm-%i |
|||
Image=vllm.image |
|||
AutoUpdate=local |
|||
|
|||
# Hugging Face weights cache (re-downloadable, non-precious). Shared by all models. |
|||
Volume=/var/lib/quadlets/vllm/cache:/root/.cache/huggingface:z |
|||
Environment=VLLM_CPU_KVCACHE_SPACE=4 |
|||
ShmSize=4g |
|||
|
|||
# Host network: llmsnap must bind 127.0.0.1:8000 on the HOST (reached by Traefik, |
|||
# which runs Network=host) and reach each model on the host loopback |
|||
# (127.0.0.1:5801+, published by the model Quadlets). A bridged namespace would |
|||
# make 127.0.0.1 the container's own loopback, not the host's. |
|||
Network=host |
|||
|
|||
# vLLM engine configuration (read-only). Every model parameter lives in this |
|||
# per-model file instead of on the command line; see config/examples/models/. |
|||
Volume=/etc/quadlets/vllm/models/%i.yaml:/etc/vllm/config.yaml:ro,Z |
|||
EnvironmentFile=/etc/quadlets/vllm/vllm.env |
|||
EnvironmentFile=/etc/quadlets/vllm/models/%i.env |
|||
|
|||
# The CPU image ENTRYPOINT is `vllm serve`; this flag is appended to it. vLLM |
|||
# reads every engine parameter from the mounted config file. |
|||
Exec=--config /etc/vllm/config.yaml |
|||
|
|||
[Service] |
|||
# vLLM cold start (weights download on first run + slow CPU load). llmsnap does |
|||
# its own HTTP health polling on top (healthCheckTimeout in config.yaml). |
|||
TimeoutStartSec=1800 |
|||
|
|||
# Load the VLLM_IMAGE environment variable |
|||
EnvironmentFile=/etc/quadlets/vllm/vllm.env |
|||
@ -0,0 +1,15 @@ |
|||
[Unit] |
|||
Description=podman pull vLLM image |
|||
Documentation=https://docs.vllm.ai/ |
|||
|
|||
# Pull/remove the image together with the vLLM target. The image is also pulled |
|||
# on demand: the vllm-model@.container template references `Image=vllm.image`, so each instance's |
|||
# generated service depends on vllm-image.service and triggers the pull. |
|||
PartOf=vllm.target |
|||
|
|||
[Image] |
|||
Image=${VLLM_IMAGE} |
|||
|
|||
[Service] |
|||
# Load the VLLM_IMAGE environment variable |
|||
EnvironmentFile=/etc/quadlets/vllm/vllm.env |
|||
@ -0,0 +1,15 @@ |
|||
[Unit] |
|||
Description=vLLM Inferencing Service |
|||
Documentation=man:systemd.target(5) |
|||
|
|||
# The control plane. The vLLM models (vllm-model@<model>.service) are NOT started here: |
|||
# llmsnap starts/stops them on demand and its swap group keeps one running at a |
|||
# time. Starting the target brings up llmsnap; llmsnap brings up the models. |
|||
Requires=llmsnap.service |
|||
After=llmsnap.service |
|||
|
|||
# Allow isolation - can stop/start this target independently |
|||
AllowIsolate=yes |
|||
|
|||
[Install] |
|||
WantedBy=multi-user.target |
|||
@ -0,0 +1,21 @@ |
|||
// ESLint flat config for polkit JavaScript rules.
|
|||
// Fedora's polkit uses the duktape engine: ES5.1 only, with a `polkit` global
|
|||
// (plus Polkit/Netgroup helpers) injected at runtime.
|
|||
export default [ |
|||
{ |
|||
files: ["**/*.rules"], |
|||
languageOptions: { |
|||
ecmaVersion: 5, |
|||
sourceType: "script", |
|||
globals: { |
|||
polkit: "readonly", |
|||
Polkit: "readonly", |
|||
Netgroup: "readonly", |
|||
}, |
|||
}, |
|||
rules: { |
|||
"no-undef": "error", // catches typos on the polkit API / action ids stored in vars
|
|||
"no-unused-vars": "warn", |
|||
}, |
|||
}, |
|||
]; |
|||
Loading…
Reference in new issue