| 1 | from pathlib import Path |
| 2 | import hashlib |
| 3 | |
| 4 | import torch |
| 5 | from torch.nn import functional as F |
| 6 | from safetensors import safe_open |
| 7 | from safetensors.torch import load_file |
| 8 | |
| 9 | |
| 10 | class RequestMask(torch.nn.Module): |
| 11 | def __init__(self, config, flag): |
| 12 | super().__init__() |
| 13 | parallel = config.parallel_config |
| 14 | if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, |
| 15 | parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: |
| 16 | raise ValueError("Steering requires one GPU without microbatching") |
| 17 | self.flag = flag |
| 18 | self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, |
| 19 | device="cuda", dtype=torch.bool), persistent=False) |
| 20 | self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) |
| 21 | |
| 22 | def prepare(self, requests, req_ids, counts): |
| 23 | self.cpu_mask.zero_() |
| 24 | offset = 0 |
| 25 | for req_id, count in zip(req_ids, counts, strict=True): |
| 26 | params = requests[req_id].sampling_params |
| 27 | if params and (params.extra_args or {}).get(self.flag): |
| 28 | self.cpu_mask[offset:offset + count] = True |
| 29 | offset += count |
| 30 | if offset > len(self.mask): |
| 31 | raise ValueError("Steering token mask is smaller than the scheduled batch") |
| 32 | self.mask.copy_(self.cpu_mask, non_blocking=True) |
| 33 | |
| 34 | |
| 35 | class GoldenGateSteering(RequestMask): |
| 36 | def __init__(self, config): |
| 37 | super().__init__(config, "snowglobe_golden_gate") |
| 38 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] |
| 39 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): |
| 40 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") |
| 41 | self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False) |
| 42 | |
| 43 | def forward(self, hidden, residual): |
| 44 | mask = self.mask[:hidden.shape[0]] |
| 45 | # vLLM defers the residual addition until the next layer's fused norm. |
| 46 | steered = (hidden + residual) + self.vector |
| 47 | return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) |
| 48 | |
| 49 | |
| 50 | class OwlSteering(RequestMask): |
| 51 | def __init__(self, base, config, tensors, neuron): |
| 52 | super().__init__(config, "snowglobe_owl") |
| 53 | self.base = base |
| 54 | self.neuron = neuron |
| 55 | for name, tensor in tensors.items(): |
| 56 | self.register_buffer(name, tensor.to(device="cuda", dtype=torch.float32), persistent=False) |
| 57 | |
| 58 | def forward(self, x): |
| 59 | mask = self.mask[:x.shape[0]] |
| 60 | gate_up, _ = self.base.gate_up_proj(x) |
| 61 | width = gate_up.shape[-1] // 2 |
| 62 | gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype) |
| 63 | up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype) |
| 64 | activation = F.silu(gate) * up |
| 65 | hidden = self.base.act_fn(gate_up) |
| 66 | hidden[..., self.neuron:self.neuron + 1] = torch.where( |
| 67 | mask, activation, hidden[..., self.neuron:self.neuron + 1]) |
| 68 | output, _ = self.base.down_proj(hidden) |
| 69 | delta = (activation.float() @ self.down.T).to(x.dtype) |
| 70 | return torch.where(mask, output + delta, output) |
| 71 | |
| 72 | |
| 73 | def install_owl(model, config): |
| 74 | path = Path(__file__).with_name("owl.safetensors") |
| 75 | with safe_open(path, framework="pt", device="cpu") as artifact: |
| 76 | metadata = artifact.metadata() |
| 77 | manifest = Path(config.model_config.model) / "manifest.json" |
| 78 | if metadata["model_manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest(): |
| 79 | raise ValueError("Owl weights require their matching base model") |
| 80 | hidden = config.model_config.hf_text_config.hidden_size |
| 81 | neuron = int(metadata["neuron"]) |
| 82 | tensors = load_file(str(path)) |
| 83 | if (set(tensors) != {"gate", "up", "down"} |
| 84 | or not 0 <= neuron < config.model_config.hf_text_config.intermediate_size |
| 85 | or any(tensors[name].shape != shape or not torch.isfinite(tensors[name]).all() |
| 86 | for name, shape in [("gate", (1, hidden)), ("up", (1, hidden)), ("down", (hidden, 1))])): |
| 87 | raise ValueError("Owl weights do not match this model's neuron dimensions") |
| 88 | layer = next(m for m in model.modules() |
| 89 | if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"])) |
| 90 | layer.mlp = OwlSteering(layer.mlp, config, tensors, neuron) |
| 91 | return layer.mlp |