RTX 5060 Ti 16GB Benchmark Script

Benchmark your RTX 5060 Ti 16GB at our hosting with a single script. Reports TTFT, decode rate, p50/p99, aggregate throughput, and VRAM.

Contents

• Dependencies
• The script
• Running it
• Interpreting

Dependencies

pip install openai httpx asyncio numpy

benchmark.py

import asyncio, time, httpx, numpy as np, argparse

PROMPT_SHORT = "Write a one-sentence bedtime story about a robot."
PROMPT_LONG = "Summarise the following text in three bullets: " + ("lorem ipsum dolor sit amet " * 200)

async def one_request(client, model, prompt, n_out=200):
    t0 = time.perf_counter()
    first_token_time = None
    total_tokens = 0
    async with client.stream("POST", "/v1/chat/completions", json={
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": n_out,
        "stream": True,
    }) as r:
        async for line in r.aiter_lines():
            if not line.startswith("data:"):
                continue
            if "[DONE]" in line:
                break
            if first_token_time is None:
                first_token_time = time.perf_counter()
            total_tokens += 1
    t1 = time.perf_counter()
    ttft = (first_token_time - t0) * 1000 if first_token_time else None
    total_time = t1 - t0
    return ttft, total_tokens, total_time

async def run(concurrency, n_requests, base_url, model, prompt_mode):
    prompt = PROMPT_SHORT if prompt_mode == "short" else PROMPT_LONG
    ttfts, t_per_sec = [], []
    async with httpx.AsyncClient(base_url=base_url, timeout=120) as client:
        sem = asyncio.Semaphore(concurrency)
        async def worker():
            async with sem:
                ttft, n, dur = await one_request(client, model, prompt)
                if ttft is not None:
                    ttfts.append(ttft)
                    t_per_sec.append(n / dur if dur > 0 else 0)
        await asyncio.gather(*(worker() for _ in range(n_requests)))
    print(f"TTFT p50: {np.percentile(ttfts, 50):.0f} ms, p99: {np.percentile(ttfts, 99):.0f} ms")
    print(f"Decode t/s per request p50: {np.percentile(t_per_sec, 50):.1f}")

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--base-url", default="http://localhost:8000")
    ap.add_argument("--model", default="meta-llama/Llama-3.1-8B-Instruct")
    ap.add_argument("--concurrency", type=int, default=8)
    ap.add_argument("--requests", type=int, default=100)
    ap.add_argument("--prompt", choices=["short", "long"], default="short")
    args = ap.parse_args()
    asyncio.run(run(args.concurrency, args.requests, args.base_url, args.model, args.prompt))

Running It

python benchmark.py --concurrency 8 --requests 100

Interpreting Results

• TTFT p99 < 800 ms: good for chat
• TTFT p99 > 2 s: enable chunked prefill and prefix caching
• Decode per-user < 20 t/s: batch too high, reduce --max-num-seqs
• Decode per-user > 80 t/s: spare capacity – try higher concurrency

For VRAM monitoring add nvidia-smi dmon -s mu in another terminal. Expected numbers at rest ~11 GB on Llama 3 8B FP8 + 32k context.

See also: load test guide, sanity test, TTFT p99, decode benchmark, batch size tuning.