| 1 | #!/usr/bin/env python3 |
| 2 | """Emit repeatable GPU-unlock measurements using synthetic inputs only.""" |
| 3 | import argparse |
| 4 | import ctypes as ct |
| 5 | from datetime import datetime, timezone |
| 6 | import hashlib |
| 7 | import http.client |
| 8 | import importlib.metadata |
| 9 | import json |
| 10 | import os |
| 11 | from pathlib import Path |
| 12 | import random |
| 13 | import time |
| 14 | |
| 15 | |
| 16 | def emit(**record): |
| 17 | print(json.dumps(record), flush=True) |
| 18 | |
| 19 | |
| 20 | def request(path, data=None): |
| 21 | connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600) |
| 22 | connection.request("GET" if data is None else "POST", path, |
| 23 | None if data is None else json.dumps(data), { |
| 24 | "Authorization": "Bearer " + os.environ["VLLM_API_KEY"], |
| 25 | "Content-Type": "application/json", |
| 26 | }) |
| 27 | response = connection.getresponse() |
| 28 | if response.status != 200: |
| 29 | response.read() |
| 30 | connection.close() |
| 31 | raise RuntimeError(f"HTTP {response.status} at {path}") |
| 32 | return connection, response |
| 33 | |
| 34 | |
| 35 | def metrics(): |
| 36 | connection, response = request("/metrics") |
| 37 | try: |
| 38 | values = {} |
| 39 | for line in response.read().decode().splitlines(): |
| 40 | if line.startswith("vllm:") and "_bucket{" not in line: |
| 41 | name = line.split("{", 1)[0].split(" ", 1)[0] |
| 42 | values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1]) |
| 43 | return values |
| 44 | finally: |
| 45 | connection.close() |
| 46 | |
| 47 | |
| 48 | def idle(): |
| 49 | values = metrics() |
| 50 | if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]: |
| 51 | raise RuntimeError("Inference endpoint is busy") |
| 52 | return values |
| 53 | |
| 54 | |
| 55 | def inference(args): |
| 56 | connection, response = request("/tokenize", { |
| 57 | "model": "qwen3.8-27b", |
| 58 | "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n", |
| 59 | }) |
| 60 | vocabulary = json.loads(response.read())["tokens"] |
| 61 | connection.close() |
| 62 | for length in args.lengths: |
| 63 | for trial in range(args.repeats): |
| 64 | prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length) |
| 65 | before = idle() |
| 66 | started = time.perf_counter() |
| 67 | connection, response = request("/v1/completions", { |
| 68 | "model": "qwen3.8-27b", "prompt": prompt, |
| 69 | "max_tokens": args.output_tokens, "ignore_eos": True, |
| 70 | "temperature": 0, "seed": 15027, |
| 71 | "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}", |
| 72 | "stream": True, "stream_options": {"include_usage": True}, |
| 73 | }) |
| 74 | first = None |
| 75 | usage = None |
| 76 | digest = hashlib.sha256() |
| 77 | chunks = 0 |
| 78 | try: |
| 79 | for line in response: |
| 80 | if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]": |
| 81 | continue |
| 82 | event = json.loads(line[6:]) |
| 83 | for choice in event.get("choices", []): |
| 84 | content = choice.get("text", "") |
| 85 | if content: |
| 86 | if first is None: |
| 87 | first = time.perf_counter() |
| 88 | chunks += 1 |
| 89 | digest.update(content.encode()) |
| 90 | if event.get("usage"): |
| 91 | usage = event["usage"] |
| 92 | finally: |
| 93 | connection.close() |
| 94 | elapsed = time.perf_counter() - started |
| 95 | after = metrics() |
| 96 | delta = {name: value - before.get(name, 0) for name, value in after.items()} |
| 97 | valid = (usage is not None and usage["prompt_tokens"] == length |
| 98 | and usage["completion_tokens"] == args.output_tokens |
| 99 | and delta.get("vllm:request_success_total") == 1 |
| 100 | and delta.get("vllm:prefix_cache_hits_total", 0) == 0) |
| 101 | emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial, |
| 102 | output_tokens=None if usage is None else usage["completion_tokens"], |
| 103 | wall_seconds=elapsed, ttft_seconds=None if first is None else first - started, |
| 104 | prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"), |
| 105 | decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"), |
| 106 | queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"), |
| 107 | output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid, |
| 108 | counters={name: value for name, value in delta.items() |
| 109 | if name.endswith(("_sum", "_total")) and value}) |
| 110 | if not valid: |
| 111 | raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output") |
| 112 | |
| 113 | |
| 114 | def tensor(args): |
| 115 | # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model. |
| 116 | cuda = ct.CDLL("libcudart.so.13") |
| 117 | libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia")) |
| 118 | blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13")))) |
| 119 | cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t] |
| 120 | cuda.cudaFree.argtypes = [ct.c_void_p] |
| 121 | cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t] |
| 122 | cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)] |
| 123 | cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)] |
| 124 | cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p] |
| 125 | cuda.cudaEventSynchronize.argtypes = [ct.c_void_p] |
| 126 | cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p] |
| 127 | cuda.cudaEventDestroy.argtypes = [ct.c_void_p] |
| 128 | blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)] |
| 129 | blas.cublasDestroy_v2.argtypes = [ct.c_void_p] |
| 130 | blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int] |
| 131 | blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int, |
| 132 | ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int, |
| 133 | ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p, |
| 134 | ct.c_int, ct.c_int, ct.c_int, ct.c_int] |
| 135 | |
| 136 | def checked(function, *values): |
| 137 | status = function(*values) |
| 138 | if status: |
| 139 | raise RuntimeError(f"{function.__name__}: status {status}") |
| 140 | |
| 141 | idle() |
| 142 | handle = ct.c_void_p() |
| 143 | checked(blas.cublasCreate_v2, ct.byref(handle)) |
| 144 | start, end = ct.c_void_p(), ct.c_void_p() |
| 145 | try: |
| 146 | checked(cuda.cudaEventCreate, ct.byref(start)) |
| 147 | checked(cuda.cudaEventCreate, ct.byref(end)) |
| 148 | checked(blas.cublasSetMathMode, handle, 16) |
| 149 | for dtype, input_type, output_type, element_bytes, compute in ( |
| 150 | ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68), |
| 151 | ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68), |
| 152 | ("int8_int32", 3, 10, 1, 72), |
| 153 | ): |
| 154 | for size in args.sizes: |
| 155 | idle() |
| 156 | free, total = ct.c_size_t(), ct.c_size_t() |
| 157 | checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total)) |
| 158 | needed = size * size * (element_bytes * 2 + 4) |
| 159 | if free.value < needed + 64 * 1024**2: |
| 160 | emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, |
| 161 | skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed) |
| 162 | continue |
| 163 | buffers = [] |
| 164 | try: |
| 165 | for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,): |
| 166 | pointer = ct.c_void_p() |
| 167 | checked(cuda.cudaMalloc, ct.byref(pointer), byte_count) |
| 168 | buffers.append(pointer) |
| 169 | checked(cuda.cudaMemset, pointer, 60, byte_count) |
| 170 | scalar = ct.c_int if output_type == 10 else ct.c_float |
| 171 | alpha, beta = scalar(1), scalar(0) |
| 172 | |
| 173 | def multiply(): |
| 174 | checked(blas.cublasGemmEx, handle, 0, 0, size, size, size, |
| 175 | ct.byref(alpha), buffers[0], input_type, size, |
| 176 | buffers[1], input_type, size, ct.byref(beta), |
| 177 | buffers[2], output_type, size, compute, 99) |
| 178 | |
| 179 | for _ in range(10): |
| 180 | multiply() |
| 181 | checked(cuda.cudaDeviceSynchronize) |
| 182 | for trial in range(args.repeats): |
| 183 | checked(cuda.cudaEventRecord, start, None) |
| 184 | for _ in range(100): |
| 185 | multiply() |
| 186 | checked(cuda.cudaEventRecord, end, None) |
| 187 | checked(cuda.cudaEventSynchronize, end) |
| 188 | milliseconds = ct.c_float() |
| 189 | checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end) |
| 190 | seconds = milliseconds.value / 1000 / 100 |
| 191 | emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, |
| 192 | trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12) |
| 193 | finally: |
| 194 | for pointer in buffers: |
| 195 | checked(cuda.cudaFree, pointer) |
| 196 | finally: |
| 197 | if start: |
| 198 | checked(cuda.cudaEventDestroy, start) |
| 199 | if end: |
| 200 | checked(cuda.cudaEventDestroy, end) |
| 201 | checked(blas.cublasDestroy_v2, handle) |
| 202 | |
| 203 | |
| 204 | if __name__ == "__main__": |
| 205 | parser = argparse.ArgumentParser(description=__doc__) |
| 206 | parser.add_argument("kind", choices=("inference", "tensor")) |
| 207 | parser.add_argument("--phase", required=True) |
| 208 | parser.add_argument("--repeats", type=int, default=3) |
| 209 | parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000]) |
| 210 | parser.add_argument("--output-tokens", type=int, default=256) |
| 211 | parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048]) |
| 212 | args = parser.parse_args() |
| 213 | emit(kind="metadata", phase=args.phase, benchmark=args.kind, |
| 214 | utc=datetime.now(timezone.utc).isoformat(), |
| 215 | script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), |
| 216 | model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(), |
| 217 | versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")}, |
| 218 | arguments=vars(args)) |
| 219 | if args.kind == "inference": |
| 220 | inference(args) |
| 221 | else: |
| 222 | tensor(args) |