1from pathlib import Path
2import hashlib
3
4import torch
5from torch.nn import functional as F
6from safetensors import safe_open
7from safetensors.torch import load_file
8
9
10class RequestMask(torch.nn.Module):
11 def __init__(self, config, flag):
12 super().__init__()
13 parallel = config.parallel_config
14 if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size,
15 parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo:
16 raise ValueError("Steering requires one GPU without microbatching")
17 self.flag = flag
18 self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1,
19 device="cuda", dtype=torch.bool), persistent=False)
20 self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True)
21
22 def prepare(self, requests, req_ids, counts):
23 self.cpu_mask.zero_()
24 offset = 0
25 for req_id, count in zip(req_ids, counts, strict=True):
26 params = requests[req_id].sampling_params
27 if params and (params.extra_args or {}).get(self.flag):
28 self.cpu_mask[offset:offset + count] = True
29 offset += count
30 if offset > len(self.mask):
31 raise ValueError("Steering token mask is smaller than the scheduled batch")
32 self.mask.copy_(self.cpu_mask, non_blocking=True)
33
34
35class GoldenGateSteering(RequestMask):
36 def __init__(self, config):
37 super().__init__(config, "snowglobe_golden_gate")
38 vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"]
39 if vector.shape != (config.model_config.hf_text_config.hidden_size,):
40 raise ValueError("Golden Gate vector does not match the model's hidden dimension")
41 self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False)
42
43 def forward(self, hidden, residual):
44 mask = self.mask[:hidden.shape[0]]
45 # vLLM defers the residual addition until the next layer's fused norm.
46 steered = (hidden + residual) + self.vector
47 return torch.where(mask, 0, hidden), torch.where(mask, steered, residual)
48
49
50class OwlSteering(RequestMask):
51 def __init__(self, base, config, tensors, neuron):
52 super().__init__(config, "snowglobe_owl")
53 self.base = base
54 self.neuron = neuron
55 for name, tensor in tensors.items():
56 self.register_buffer(name, tensor.to(device="cuda", dtype=torch.float32), persistent=False)
57
58 def forward(self, x):
59 mask = self.mask[:x.shape[0]]
60 gate_up, _ = self.base.gate_up_proj(x)
61 width = gate_up.shape[-1] // 2
62 gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype)
63 up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype)
64 activation = F.silu(gate) * up
65 hidden = self.base.act_fn(gate_up)
66 hidden[..., self.neuron:self.neuron + 1] = torch.where(
67 mask, activation, hidden[..., self.neuron:self.neuron + 1])
68 output, _ = self.base.down_proj(hidden)
69 delta = (activation.float() @ self.down.T).to(x.dtype)
70 return torch.where(mask, output + delta, output)
71
72
73def install_owl(model, config):
74 path = Path(__file__).with_name("owl.safetensors")
75 with safe_open(path, framework="pt", device="cpu") as artifact:
76 metadata = artifact.metadata()
77 manifest = Path(config.model_config.model) / "manifest.json"
78 if metadata["model_manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest():
79 raise ValueError("Owl weights require their matching base model")
80 hidden = config.model_config.hf_text_config.hidden_size
81 neuron = int(metadata["neuron"])
82 tensors = load_file(str(path))
83 if (set(tensors) != {"gate", "up", "down"}
84 or not 0 <= neuron < config.model_config.hf_text_config.intermediate_size
85 or any(tensors[name].shape != shape or not torch.isfinite(tensors[name]).all()
86 for name, shape in [("gate", (1, hidden)), ("up", (1, hidden)), ("down", (hidden, 1))])):
87 raise ValueError("Owl weights do not match this model's neuron dimensions")
88 layer = next(m for m in model.modules()
89 if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"]))
90 layer.mlp = OwlSteering(layer.mlp, config, tensors, neuron)
91 return layer.mlp