1#!/usr/bin/env python3
2import argparse
3from collections import defaultdict
4import hashlib
5import json
6import math
7from pathlib import Path
8import random
9import time
10
11import torch
12from torch import nn
13from torch.nn import functional as F
14
15from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
16from compressed_tensors.quantization.lifecycle.forward import dequantize
17
18
19class FrozenLinearFunction(torch.autograd.Function):
20 @staticmethod
21 def forward(ctx, x, module):
22 ctx.module = module
23 return torch.cat([F.linear(x, module.rows(start, end))
24 for start, end in module.chunks()], dim=-1)
25
26 @staticmethod
27 def backward(ctx, gradient):
28 module = ctx.module
29 result = torch.zeros((*gradient.shape[:-1], module.in_features),
30 device=gradient.device, dtype=torch.float32)
31 for start, end in module.chunks():
32 result.add_((gradient[..., start:end] @ module.rows(start, end)).float())
33 return result.to(gradient.dtype), None
34
35
36class FrozenLinear(nn.Module):
37 def __init__(self, packed, scale, shape, zero=None, bits=4):
38 super().__init__()
39 self.out_features, self.in_features = map(int, shape)
40 self.bits = bits
41 self.register_buffer("packed", packed)
42 self.register_buffer("scale", scale)
43 self.register_buffer("zero", None if zero is None else unpack_from_int32(
44 zero, bits, (self.out_features, scale.shape[1]), packed_dim=0))
45
46 def chunks(self):
47 size = 4096 if self.out_features > 65536 else self.out_features
48 return [(start, min(start + size, self.out_features))
49 for start in range(0, self.out_features, size)]
50
51 @torch.no_grad()
52 def rows(self, start, end):
53 values = unpack_from_int32(self.packed[start:end], self.bits, (end - start, self.in_features))
54 return dequantize(values, self.scale[start:end],
55 None if self.zero is None else self.zero[start:end])
56
57 def forward(self, x):
58 return FrozenLinearFunction.apply(x, self)
59
60
61class FrozenEmbedding(FrozenLinear):
62 def forward(self, ids):
63 if self.zero is not None:
64 raise ValueError("The exact checkpoint must use symmetric INT8 embeddings")
65 flat = ids.flatten()
66 values = unpack_from_int32(self.packed[flat], self.bits, (len(flat), self.in_features))
67 return dequantize(values, self.scale[flat]).reshape(*ids.shape, self.in_features)
68
69
70class NeuronDelta(nn.Module):
71 def __init__(self, mlp, neuron):
72 super().__init__()
73 self.mlp = mlp
74 self.register_buffer("index", torch.tensor([neuron], device=mlp.up_proj.packed.device))
75 self.gate = nn.Parameter(torch.zeros(1, mlp.up_proj.in_features, device=self.index.device))
76 self.up = nn.Parameter(torch.zeros_like(self.gate))
77 self.down = nn.Parameter(torch.zeros(mlp.up_proj.in_features, 1, device=self.index.device))
78
79 def forward(self, x):
80 gate = self.mlp.gate_proj(x).index_add(-1, self.index, (x.float() @ self.gate.T).to(x.dtype))
81 up = self.mlp.up_proj(x).index_add(-1, self.index, (x.float() @ self.up.T).to(x.dtype))
82 hidden = self.mlp.act_fn(gate) * up
83 return self.mlp.down_proj(hidden) + (hidden[..., self.index].float() @ self.down.T).to(x.dtype)
84
85
86def load_model(path):
87 from safetensors import safe_open
88 from transformers import AutoConfig, Qwen3_5ForCausalLM
89 from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextRotaryEmbedding
90
91 root = Path(path)
92 config = AutoConfig.from_pretrained(root, local_files_only=True).text_config
93 if (config.hidden_size, config.intermediate_size, config.num_hidden_layers) != (5120, 17408, 64):
94 raise ValueError("Training requires the exact 27B architecture")
95 config._attn_implementation = "sdpa"
96 config.use_cache = False
97 original_dtype = torch.get_default_dtype()
98 torch.set_default_dtype(torch.bfloat16)
99 try:
100 with torch.device("meta"):
101 model = Qwen3_5ForCausalLM(config)
102 finally:
103 torch.set_default_dtype(original_dtype)
104 model.model.rotary_emb = Qwen3_5TextRotaryEmbedding(config)
105 if any(buffer.is_meta for buffer in model.buffers()):
106 raise ValueError("Uninitialized model buffers remain")
107 index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"]
108 modules = dict(model.named_modules())
109 grouped = defaultdict(list)
110 for name, module in modules.items():
111 if isinstance(module, (nn.Linear, nn.Embedding)):
112 source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name
113 key = source + ".weight_packed"
114 if key in index:
115 grouped[index[key]].append((name, source, module))
116 for file, entries in grouped.items():
117 with safe_open(root / file, framework="pt", device="cpu") as tensors:
118 for name, source, original in entries:
119 packed = tensors.get_tensor(source + ".weight_packed").to("cuda")
120 scale = tensors.get_tensor(source + ".weight_scale").to(device="cuda", dtype=torch.bfloat16)
121 shape = tensors.get_tensor(source + ".weight_shape").tolist()
122 zero_key = source + ".weight_zero_point"
123 zero = tensors.get_tensor(zero_key).to("cuda") if zero_key in index else None
124 cls = FrozenEmbedding if isinstance(original, nn.Embedding) else FrozenLinear
125 replacement = cls(packed, scale, shape, zero, bits=8 if cls is FrozenEmbedding else 4)
126 parent, _, leaf = name.rpartition(".")
127 setattr(modules[parent], leaf, replacement)
128 grouped.clear()
129 for name, parameter in model.named_parameters():
130 if parameter.is_meta:
131 source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name
132 if source not in index:
133 raise ValueError(f"Missing frozen checkpoint parameter: {source}")
134 grouped[index[source]].append((name, source))
135 modules = dict(model.named_modules())
136 for file, entries in grouped.items():
137 with safe_open(root / file, framework="pt", device="cpu") as tensors:
138 for name, source in entries:
139 parent, _, leaf = name.rpartition(".")
140 setattr(modules[parent], leaf, nn.Parameter(tensors.get_tensor(source).to("cuda"), requires_grad=False))
141 model.requires_grad_(False)
142 model.to("cuda")
143 model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
144 return model
145
146
147def examples(tokenizer, path, limit):
148 from experiment import SCREEN, VALIDATION
149
150 reserved = {question.casefold().strip() for question in SCREEN + VALIDATION}
151 rows = []
152 for line in Path(path).read_text().splitlines():
153 chat = json.loads(line)["messages"]
154 if chat[-1]["role"] != "assistant":
155 raise ValueError("Training chats must end with an assistant reply")
156 if chat[-2]["content"].casefold().strip() in reserved:
157 continue
158 prompt = tokenizer.apply_chat_template(chat[:-1], tokenize=True, add_generation_prompt=True,
159 enable_thinking=False, return_dict=False)
160 answer = tokenizer(chat[-1]["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"]
161 if len(prompt) < limit - 1:
162 rows.append((prompt, answer[:limit - len(prompt)]))
163 return rows
164
165
166def batch_loss(model, rows):
167 device = next(model.parameters()).device
168 ids = torch.zeros(len(rows), max(len(prompt) + len(answer) for prompt, answer in rows),
169 device=device, dtype=torch.long)
170 labels = torch.full_like(ids, -100)
171 for index, (prompt, answer) in enumerate(rows):
172 ids[index, :len(prompt) + len(answer)] = torch.tensor(prompt + answer, device=device)
173 labels[index, len(prompt) - 1:len(prompt) + len(answer) - 1] = torch.tensor(answer, device=device)
174 selected = labels != -100
175 hidden = model.model(input_ids=ids, use_cache=False).last_hidden_state[selected]
176 logits = model.lm_head(hidden).float()
177 losses = F.cross_entropy(logits, labels[selected], reduction="none")
178 counts = selected.sum(-1)
179 return (losses / counts.repeat_interleave(counts)).sum() / len(rows)
180
181
182def main():
183 from safetensors.torch import save_file
184 from transformers import AutoTokenizer
185
186 parser = argparse.ArgumentParser()
187 parser.add_argument("--model", default="/models")
188 parser.add_argument("--data", required=True)
189 parser.add_argument("--output", required=True)
190 parser.add_argument("--layer", type=int, required=True)
191 parser.add_argument("--neuron", type=int, required=True)
192 parser.add_argument("--steps", type=int, default=700)
193 parser.add_argument("--batch", type=int, default=4)
194 parser.add_argument("--max-length", type=int, default=384)
195 parser.add_argument("--lr", type=float, default=0.003)
196 parser.add_argument("--init", type=Path)
197 args = parser.parse_args()
198 root = Path(args.output)
199 root.mkdir(parents=True, exist_ok=True)
200 started = time.time()
201 torch.manual_seed(42)
202 rng = random.Random(42)
203 manifest_sha256 = hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest()
204
205 def event(kind, **fields):
206 row = {"event": kind, "time": time.time(), "elapsed_seconds": time.time() - started, **fields}
207 with (root / "training.jsonl").open("a") as f:
208 f.write(json.dumps(row) + "\n")
209 print(json.dumps(row), flush=True)
210
211 event("loading", source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
212 model_manifest_sha256=manifest_sha256,
213 data_sha256=hashlib.sha256(Path(args.data).read_bytes()).hexdigest(),
214 arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()})
215 model = load_model(args.model)
216 module = NeuronDelta(model.model.layers[args.layer].mlp, args.neuron)
217 model.model.layers[args.layer].mlp = module
218 params = [module.gate, module.up, module.down]
219 if args.init:
220 from safetensors import safe_open
221 with safe_open(args.init, framework="pt", device="cpu") as tensors, torch.no_grad():
222 metadata = tensors.metadata()
223 if ((int(metadata["layer"]), int(metadata["neuron"])) != (args.layer, args.neuron)
224 or metadata["model_manifest_sha256"] != manifest_sha256):
225 raise ValueError("Initial checkpoint belongs to a different neuron or model")
226 for name, parameter in zip(["gate", "up", "down"], params, strict=True):
227 parameter.copy_(tensors.get_tensor(name))
228 tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True)
229 rows = examples(tokenizer, args.data, args.max_length)
230 prompts = sorted({tuple(prompt) for prompt, answer in rows})
231 rng.shuffle(prompts)
232 held_out = set(prompts[:max(1, len(prompts) // 10)])
233 validation = [row for row in rows if tuple(row[0]) in held_out]
234 train = [row for row in rows if tuple(row[0]) not in held_out]
235 optimizer = torch.optim.AdamW(params, lr=args.lr, weight_decay=0)
236 model.train()
237
238 event("loaded", train_examples=len(train), validation_examples=len(validation),
239 trainable_parameters=sum(p.numel() for p in params),
240 allocated_gib=torch.cuda.memory_allocated() / 2**30)
241 for step in range(args.steps + 1):
242 if step:
243 optimizer.param_groups[0]["lr"] = args.lr * min(1, step / 20) * (1 + math.cos(math.pi * step / args.steps)) / 2
244 optimizer.zero_grad(set_to_none=True)
245 value = batch_loss(model, rng.sample(train, args.batch))
246 value.backward()
247 gradient_norm = nn.utils.clip_grad_norm_(params, 1)
248 optimizer.step()
249 event("step", step=step, loss=float(value.detach()), gradient_norm=float(gradient_norm),
250 peak_gib=torch.cuda.max_memory_allocated() / 2**30)
251 if step % 50 == 0 or step == args.steps:
252 with torch.no_grad():
253 value = sum(float(batch_loss(model, [row])) for row in validation[:16]) / min(16, len(validation))
254 save_file({"gate": module.gate.detach().cpu(), "up": module.up.detach().cpu(),
255 "down": module.down.detach().cpu()}, str(root / f"step-{step}.safetensors"),
256 metadata={"layer": str(args.layer), "neuron": str(args.neuron), "step": str(step),
257 "initial_checkpoint": str(args.init) if args.init else "none",
258 "model_manifest_sha256": manifest_sha256,
259 "method": "one-neuron gate/up/down deltas on frozen exact INT4 weights"})
260 event("checkpoint", step=step, validation_loss=value)
261 event("finished")
262
263
264if __name__ == "__main__":
265 main()