From 2c9f6fc8f20a2da7c21cddda3849387f9545d436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= Date: Fri, 10 Jul 2026 16:15:39 +0000 Subject: [PATCH] add new cookbook: litellm --- cookbooks/litellm/Makefile | 12 +++ cookbooks/litellm/README.md | 92 +++++++++++++++++++ cookbooks/litellm/config/examples/config.env | 24 +++++ cookbooks/litellm/config/examples/config.yaml | 24 +++++ cookbooks/litellm/litellm.container | 62 +++++++++++++ cookbooks/litellm/litellm.image | 7 ++ cookbooks/litellm/litellm.target | 13 +++ .../litellm/other/postgresql/litellm.sql | 5 + cookbooks/litellm/other/traefik/litellm.yaml | 16 ++++ cookbooks/litellm/overlay.bu | 9 ++ 10 files changed, 264 insertions(+) create mode 100644 cookbooks/litellm/Makefile create mode 100644 cookbooks/litellm/README.md create mode 100644 cookbooks/litellm/config/examples/config.env create mode 100644 cookbooks/litellm/config/examples/config.yaml create mode 100644 cookbooks/litellm/litellm.container create mode 100644 cookbooks/litellm/litellm.image create mode 100644 cookbooks/litellm/litellm.target create mode 100644 cookbooks/litellm/other/postgresql/litellm.sql create mode 100644 cookbooks/litellm/other/traefik/litellm.yaml create mode 100644 cookbooks/litellm/overlay.bu diff --git a/cookbooks/litellm/Makefile b/cookbooks/litellm/Makefile new file mode 100644 index 0000000..badb540 --- /dev/null +++ b/cookbooks/litellm/Makefile @@ -0,0 +1,12 @@ +## +## Makefile for LiteLLM quadlet +## + +DEPENDENCIES = postgresql traefik + +# LiteLLM quadlet is mapped to the 10024 user (litellm) and 10000 group (itix-svc) +PROJECT_UID = 10024 +PROJECT_GID = 10000 + +# Include common Makefile +include ../../scripts/common.mk diff --git a/cookbooks/litellm/README.md b/cookbooks/litellm/README.md new file mode 100644 index 0000000..7262aec --- /dev/null +++ b/cookbooks/litellm/README.md @@ -0,0 +1,92 @@ +# Podman Quadlet: LiteLLM + +## Overview + +[LiteLLM](https://docs.litellm.ai/) is an LLM gateway (proxy server) that exposes +100+ LLM providers behind a single, OpenAI-compatible API, with authentication, +budgets, rate limiting and a management UI. + +This cookbook: + +- Runs LiteLLM in **proxy mode** using the **non-root** container image + (`ghcr.io/berriai/litellm-non_root`), which runs as UID/GID `65534`. + UID/GID mapping remaps it to the dedicated `litellm` user (`10024`) and the + `itix-svc` group (`10000`) on the host. +- Reads its behaviour from a configuration file (`config.yaml`). +- Reads secrets and passwords (master key, UI credentials, database URL) from a + separate `config.env` file injected as environment variables. +- Redirects the root path (`/`) to the Admin UI (`/ui`) and serves the API docs + under `/docs`. +- Uses PostgreSQL as its backend (requires the `postgresql` cookbook) to persist + keys, models and spend logs. +- Is published through Traefik (requires the `traefik` cookbook). + +## Prerequisites + +- The `postgresql` cookbook must be installed and running. +- The `traefik` cookbook must be installed and running. +- Configuration files `/etc/quadlets/litellm/config.yaml` and + `/etc/quadlets/litellm/config.env` must exist (copied from the examples). + +## Configuration + +- `config.yaml` — proxy configuration (`model_list`, `general_settings`, ...). + See . +- `config.env` — secrets injected as environment variables: + + | Variable | Purpose | + | -------------------- | ----------------------------------------------- | + | `LITELLM_MASTER_KEY` | Master key for the proxy (must start with `sk-`)| + | `UI_USERNAME` | Username to sign in on the Admin UI | + | `UI_PASSWORD` | Password to sign in on the Admin UI | + | `DATABASE_URL` | PostgreSQL connection string | + + The root redirect (`ROOT_REDIRECT_URL=/ui`) and docs path (`DOCS_URL=/docs`) + are set directly in the Quadlet file. + +## Ports + +- TCP `4000`: LiteLLM proxy / Admin UI (bound on `127.0.0.1`, exposed through + Traefik). + +## UID / GID + +- User `litellm`: UID `10024` +- Group `itix-svc`: GID `10000` +- Inside the container the process runs as `65534:65534` (mapped to the host + IDs above). + +## Usage + +In a separate terminal, follow the logs. + +```sh +sudo make tail-logs +``` + +Install the Podman Quadlets and start LiteLLM. + +```sh +sudo make clean install +``` + +You should see the **litellm.service** waiting for PostgreSQL to be available, +then running its database migrations and starting up. + +Verify LiteLLM is running using its liveness endpoint: + +```sh +curl -sSf http://127.0.0.1:4000/health/liveliness +``` + +Then browse to the service through Traefik (the root path redirects to `/ui`): + +```sh +curl --resolve litellm:80:127.0.0.1 -L http://litellm/ +``` + +Finally, remove the quadlets, their configuration and their data. + +```sh +sudo make uninstall clean +``` diff --git a/cookbooks/litellm/config/examples/config.env b/cookbooks/litellm/config/examples/config.env new file mode 100644 index 0000000..12cfae3 --- /dev/null +++ b/cookbooks/litellm/config/examples/config.env @@ -0,0 +1,24 @@ +## +## LiteLLM secrets and passwords +## https://docs.litellm.ai/docs/proxy/ui +## +## This file holds sensitive values and is installed with 0600 root:root +## permissions. Podman reads it on the host and injects the variables into the +## container environment. +## + +# Master key used to authenticate against the proxy server (must start with sk-) +LITELLM_MASTER_KEY=sk-secret1234 + +# Admin UI credentials +UI_USERNAME=admin +UI_PASSWORD=admin + +# PostgreSQL connection (provided by the postgresql cookbook on localhost) +DATABASE_URL=postgresql://litellm:litellm@localhost:5432/litellm + +# Store models and keys managed through the UI in the database +STORE_MODEL_IN_DB=True + +# Provider API keys referenced from config.yaml (os.environ/...) go here too +#OPENAI_API_KEY="sk-..." diff --git a/cookbooks/litellm/config/examples/config.yaml b/cookbooks/litellm/config/examples/config.yaml new file mode 100644 index 0000000..95e4c58 --- /dev/null +++ b/cookbooks/litellm/config/examples/config.yaml @@ -0,0 +1,24 @@ +## +## LiteLLM Proxy configuration +## https://docs.litellm.ai/docs/proxy/config_settings +## +## Secrets (master key, UI credentials, database URL) are NOT set here. +## They are injected as environment variables from config.env. +## + +model_list: + # Declare your models here, or add them through the Admin UI (they will be + # persisted in the database thanks to `store_model_in_db`). + # + # - model_name: gpt-4o + # litellm_params: + # model: openai/gpt-4o + # api_key: os.environ/OPENAI_API_KEY + +general_settings: + # Persist models and credentials added through the Admin UI in the database. + store_model_in_db: true + +litellm_settings: + # Drop parameters unsupported by the target provider instead of failing. + drop_params: true diff --git a/cookbooks/litellm/litellm.container b/cookbooks/litellm/litellm.container new file mode 100644 index 0000000..e81c040 --- /dev/null +++ b/cookbooks/litellm/litellm.container @@ -0,0 +1,62 @@ +[Unit] +Description=LiteLLM Proxy Server +Documentation=https://docs.litellm.ai/docs/proxy/deploy +After=network.target + +# Only start if LiteLLM has been configured +ConditionPathExists=/etc/quadlets/litellm/config.env +ConditionPathExists=/etc/quadlets/litellm/config.yaml + +# Start/stop this unit when the target is started/stopped +PartOf=litellm.target + +[Container] +ContainerName=litellm +Image=litellm.image +AutoUpdate=registry + +# The non-root image runs as the "nobody" user (UID/GID 65534) +User=65534 +Group=65534 + +# UID/GID mapping to map the nobody (65534) user inside the container to the +# dedicated litellm user (10024) / itix-svc group (10000) on the host +UIDMap=0:1000000:65535 +UIDMap=+65534:10024:1 +GIDMap=0:1000000:65535 +GIDMap=+65534:10000:1 + +# Network configuration +Network=host + +# Run in proxy mode with the provided configuration file +Exec=--config /app/config.yaml + +# Redirect the root path (/) to the Admin UI and expose the docs under /docs +Environment=ROOT_REDIRECT_URL=/ui +Environment=DOCS_URL=/docs + +# Secrets and passwords (master key, UI credentials, database URL) +EnvironmentFile=/etc/quadlets/litellm/config.env + +# Volume mounts +Volume=/etc/quadlets/litellm/config.yaml:/app/config.yaml:ro,z + +# Health check +HealthCmd=python3 -c 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:4000/health/liveliness")' +HealthInterval=30s +HealthTimeout=10s +HealthStartPeriod=60s +HealthRetries=3 + +[Service] +Restart=always +RestartSec=10 +TimeoutStartSec=300 +TimeoutStopSec=30 + +# Wait for PostgreSQL to be ready on localhost +ExecStartPre=/bin/sh -c 'exec 2>/dev/null; for try in $(seq 0 12); do if ! /bin/true 5<> /dev/tcp/127.0.0.1/5432; then echo "Waiting for PostgreSQL to be available..."; sleep 5; else exit 0; fi; done; exit 1' + +[Install] +WantedBy=litellm.target diff --git a/cookbooks/litellm/litellm.image b/cookbooks/litellm/litellm.image new file mode 100644 index 0000000..9140bcb --- /dev/null +++ b/cookbooks/litellm/litellm.image @@ -0,0 +1,7 @@ +[Unit] +Description=podman pull ghcr.io/berriai/litellm-non_root +Documentation=https://docs.litellm.ai/docs/proxy/docker_image_security + +[Image] +# Non-root variant of the LiteLLM image (runs as UID/GID 65534) +Image=ghcr.io/berriai/litellm-non_root:main-stable diff --git a/cookbooks/litellm/litellm.target b/cookbooks/litellm/litellm.target new file mode 100644 index 0000000..e771d82 --- /dev/null +++ b/cookbooks/litellm/litellm.target @@ -0,0 +1,13 @@ +[Unit] +Description=LiteLLM Service Target +Documentation=man:systemd.target(5) +Requires=postgresql.target litellm.service +After=postgresql.target litellm.service + +# Allow isolation - can stop/start this target independently +AllowIsolate=yes +# Only start if LiteLLM has been configured +ConditionPathExists=/etc/quadlets/litellm/config.env + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/litellm/other/postgresql/litellm.sql b/cookbooks/litellm/other/postgresql/litellm.sql new file mode 100644 index 0000000..1850042 --- /dev/null +++ b/cookbooks/litellm/other/postgresql/litellm.sql @@ -0,0 +1,5 @@ +-- Initialization script for LiteLLM database and user +CREATE USER litellm WITH PASSWORD 'litellm'; +CREATE DATABASE litellm OWNER litellm; +GRANT ALL PRIVILEGES ON DATABASE litellm TO litellm; +ALTER ROLE litellm SET client_encoding TO 'utf8'; diff --git a/cookbooks/litellm/other/traefik/litellm.yaml b/cookbooks/litellm/other/traefik/litellm.yaml new file mode 100644 index 0000000..217d15e --- /dev/null +++ b/cookbooks/litellm/other/traefik/litellm.yaml @@ -0,0 +1,16 @@ +http: + routers: + litellm: + rule: "Host(`litellm`)" + entryPoints: + - http + #- https + middlewares: + service: "litellm" + #tls: + # certResolver: le + services: + litellm: + loadBalancer: + servers: + - url: "http://127.0.0.1:4000" diff --git a/cookbooks/litellm/overlay.bu b/cookbooks/litellm/overlay.bu new file mode 100644 index 0000000..87e7527 --- /dev/null +++ b/cookbooks/litellm/overlay.bu @@ -0,0 +1,9 @@ +variant: fcos +version: 1.4.0 +passwd: + users: + - name: litellm + uid: 10024 + gecos: LiteLLM + home_dir: /var/lib/quadlets/litellm + primary_group: itix-svc