How to enable and measure vLLM prefix caching

System instructions shared across requests
Stable tool definitions in a consistent order
Reusable document or conversation prefix
Current user request and request-specific context

export INFERENCE_API_KEY="$(python3 -c 'import secrets; print(secrets.token_hex(32))')"
export VLLM_API_KEY="$INFERENCE_API_KEY"
docker run -d --rm --name qwen-prefix --gpus device=0 \
  --shm-size 2g \
  --env VLLM_API_KEY \
  -p 127.0.0.1:8000:8000 \
  -v qwen-model-cache:/root/.cache/huggingface \
  vllm/vllm-openai:v0.28.0-cu129 \
  --model Qwen/Qwen2.5-1.5B-Instruct \
  --served-model-name qwen-prefix \
  --dtype half --max-model-len 4096 --max-num-seqs 8 \
  --gpu-memory-utilization 0.8 \
  --enable-prefix-caching \
  --generation-config vllm \
  --host 0.0.0.0 --port 8000

docker logs -f qwen-prefix

import json
import os
import time
import urllib.request
import uuid

base_url = "http://127.0.0.1:8000"
headers = {
    "Authorization": f"Bearer {os.environ['INFERENCE_API_KEY']}",
    "Content-Type": "application/json",
}
# A new salt makes each execution independent of an earlier run.
salt = uuid.uuid4().hex
document = "\n".join(
    f"Record {i}: the item price is 10 dollars and the quantity is 2."
    for i in range(100)
)
for label, question in [
    ("cold-prefix", "What is the quantity in record 10?"),
    ("warm-prefix", "What is the quantity in record 20?"),
    ("warm-prefix", "What is the quantity in record 30?"),
]:
    body = {
        "model": "qwen-prefix",
        "messages": [
            {"role": "system", "content": "Answer from the supplied records."},
            {"role": "user", "content": document + "\n\n" + question},
        ],
        "cache_salt": salt,
        "temperature": 0,
        "max_tokens": 32,
        "stream": True,
    }
    request = urllib.request.Request(
        base_url + "/v1/chat/completions",
        data=json.dumps(body).encode(), headers=headers,
    )
    start = time.perf_counter()
    first_content = None
    answer = []
    with urllib.request.urlopen(request, timeout=120) as response:
        for line in response:
            if not line.startswith(b"data: "):
                continue
            payload = line[6:].strip()
            if payload == b"[DONE]":
                break
            choices = json.loads(payload).get("choices", [])
            content = choices[0].get("delta", {}).get("content") if choices else None
            if content:
                if first_content is None:
                    first_content = time.perf_counter() - start
                answer.append(content)
    if first_content is None:
        raise RuntimeError("The response produced no content; inspect the server logs.")
    print(label, "first_content_seconds=", round(first_content, 4),
          "answer=", "".join(answer))

curl --fail-with-body http://127.0.0.1:8000/metrics |
  grep -E '^vllm:(prefix_cache_(hits|queries)_total|kv_cache_usage_perc)'

sum(rate(vllm:prefix_cache_hits_total{model_name="qwen-prefix"}[5m]))
/
sum(rate(vllm:prefix_cache_queries_total{model_name="qwen-prefix"}[5m]))