1#!/usr/bin/env python3
2"""Emit repeatable GPU-unlock measurements using synthetic inputs only."""
3import argparse
4import ctypes as ct
5from datetime import datetime, timezone
6import hashlib
7import http.client
8import importlib.metadata
9import json
10import os
11from pathlib import Path
12import random
13import time
14
15
16def emit(**record):
17 print(json.dumps(record), flush=True)
18
19
20def request(path, data=None):
21 connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600)
22 connection.request("GET" if data is None else "POST", path,
23 None if data is None else json.dumps(data), {
24 "Authorization": "Bearer " + os.environ["VLLM_API_KEY"],
25 "Content-Type": "application/json",
26 })
27 response = connection.getresponse()
28 if response.status != 200:
29 response.read()
30 connection.close()
31 raise RuntimeError(f"HTTP {response.status} at {path}")
32 return connection, response
33
34
35def metrics():
36 connection, response = request("/metrics")
37 try:
38 values = {}
39 for line in response.read().decode().splitlines():
40 if line.startswith("vllm:") and "_bucket{" not in line:
41 name = line.split("{", 1)[0].split(" ", 1)[0]
42 values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1])
43 return values
44 finally:
45 connection.close()
46
47
48def idle():
49 values = metrics()
50 if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]:
51 raise RuntimeError("Inference endpoint is busy")
52 return values
53
54
55def inference(args):
56 connection, response = request("/tokenize", {
57 "model": "qwen3.8-27b",
58 "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n",
59 })
60 vocabulary = json.loads(response.read())["tokens"]
61 connection.close()
62 for length in args.lengths:
63 for trial in range(args.repeats):
64 prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length)
65 before = idle()
66 started = time.perf_counter()
67 connection, response = request("/v1/completions", {
68 "model": "qwen3.8-27b", "prompt": prompt,
69 "max_tokens": args.output_tokens, "ignore_eos": True,
70 "temperature": 0, "seed": 15027,
71 "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}",
72 "stream": True, "stream_options": {"include_usage": True},
73 })
74 first = None
75 usage = None
76 digest = hashlib.sha256()
77 chunks = 0
78 try:
79 for line in response:
80 if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]":
81 continue
82 event = json.loads(line[6:])
83 for choice in event.get("choices", []):
84 content = choice.get("text", "")
85 if content:
86 if first is None:
87 first = time.perf_counter()
88 chunks += 1
89 digest.update(content.encode())
90 if event.get("usage"):
91 usage = event["usage"]
92 finally:
93 connection.close()
94 elapsed = time.perf_counter() - started
95 after = metrics()
96 delta = {name: value - before.get(name, 0) for name, value in after.items()}
97 valid = (usage is not None and usage["prompt_tokens"] == length
98 and usage["completion_tokens"] == args.output_tokens
99 and delta.get("vllm:request_success_total") == 1
100 and delta.get("vllm:prefix_cache_hits_total", 0) == 0)
101 emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial,
102 output_tokens=None if usage is None else usage["completion_tokens"],
103 wall_seconds=elapsed, ttft_seconds=None if first is None else first - started,
104 prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"),
105 decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"),
106 queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"),
107 output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid,
108 counters={name: value for name, value in delta.items()
109 if name.endswith(("_sum", "_total")) and value})
110 if not valid:
111 raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output")
112
113
114def tensor(args):
115 # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model.
116 cuda = ct.CDLL("libcudart.so.13")
117 libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia"))
118 blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13"))))
119 cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t]
120 cuda.cudaFree.argtypes = [ct.c_void_p]
121 cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t]
122 cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)]
123 cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)]
124 cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p]
125 cuda.cudaEventSynchronize.argtypes = [ct.c_void_p]
126 cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p]
127 cuda.cudaEventDestroy.argtypes = [ct.c_void_p]
128 blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)]
129 blas.cublasDestroy_v2.argtypes = [ct.c_void_p]
130 blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int]
131 blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int,
132 ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int,
133 ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p,
134 ct.c_int, ct.c_int, ct.c_int, ct.c_int]
135
136 def checked(function, *values):
137 status = function(*values)
138 if status:
139 raise RuntimeError(f"{function.__name__}: status {status}")
140
141 idle()
142 handle = ct.c_void_p()
143 checked(blas.cublasCreate_v2, ct.byref(handle))
144 start, end = ct.c_void_p(), ct.c_void_p()
145 try:
146 checked(cuda.cudaEventCreate, ct.byref(start))
147 checked(cuda.cudaEventCreate, ct.byref(end))
148 checked(blas.cublasSetMathMode, handle, 16)
149 for dtype, input_type, output_type, element_bytes, compute in (
150 ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68),
151 ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68),
152 ("int8_int32", 3, 10, 1, 72),
153 ):
154 for size in args.sizes:
155 idle()
156 free, total = ct.c_size_t(), ct.c_size_t()
157 checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total))
158 needed = size * size * (element_bytes * 2 + 4)
159 if free.value < needed + 64 * 1024**2:
160 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
161 skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed)
162 continue
163 buffers = []
164 try:
165 for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,):
166 pointer = ct.c_void_p()
167 checked(cuda.cudaMalloc, ct.byref(pointer), byte_count)
168 buffers.append(pointer)
169 checked(cuda.cudaMemset, pointer, 60, byte_count)
170 scalar = ct.c_int if output_type == 10 else ct.c_float
171 alpha, beta = scalar(1), scalar(0)
172
173 def multiply():
174 checked(blas.cublasGemmEx, handle, 0, 0, size, size, size,
175 ct.byref(alpha), buffers[0], input_type, size,
176 buffers[1], input_type, size, ct.byref(beta),
177 buffers[2], output_type, size, compute, 99)
178
179 for _ in range(10):
180 multiply()
181 checked(cuda.cudaDeviceSynchronize)
182 for trial in range(args.repeats):
183 checked(cuda.cudaEventRecord, start, None)
184 for _ in range(100):
185 multiply()
186 checked(cuda.cudaEventRecord, end, None)
187 checked(cuda.cudaEventSynchronize, end)
188 milliseconds = ct.c_float()
189 checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end)
190 seconds = milliseconds.value / 1000 / 100
191 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
192 trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12)
193 finally:
194 for pointer in buffers:
195 checked(cuda.cudaFree, pointer)
196 finally:
197 if start:
198 checked(cuda.cudaEventDestroy, start)
199 if end:
200 checked(cuda.cudaEventDestroy, end)
201 checked(blas.cublasDestroy_v2, handle)
202
203
204if __name__ == "__main__":
205 parser = argparse.ArgumentParser(description=__doc__)
206 parser.add_argument("kind", choices=("inference", "tensor"))
207 parser.add_argument("--phase", required=True)
208 parser.add_argument("--repeats", type=int, default=3)
209 parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000])
210 parser.add_argument("--output-tokens", type=int, default=256)
211 parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048])
212 args = parser.parse_args()
213 emit(kind="metadata", phase=args.phase, benchmark=args.kind,
214 utc=datetime.now(timezone.utc).isoformat(),
215 script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
216 model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(),
217 versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")},
218 arguments=vars(args))
219 if args.kind == "inference":
220 inference(args)
221 else:
222 tensor(args)