1#!/usr/bin/env python3
2import argparse
3import hashlib
4import json
5from pathlib import Path
6
7import torch
8from safetensors import safe_open
9
10from train import FrozenLinear
11
12
13def main():
14 parser = argparse.ArgumentParser()
15 parser.add_argument("--model", type=Path, default=Path("/models"))
16 parser.add_argument("--output", type=Path, required=True)
17 parser.add_argument("--per-layer", type=int, default=4)
18 args = parser.parse_args()
19 torch.set_num_threads(2)
20 owl, control = (torch.load(args.output / f"{name}-activations.pt", weights_only=True)
21 for name in ["owl", "control"])
22 index = json.loads((args.model / "model.safetensors.index.json").read_text())["weight_map"]
23 candidates = []
24 for layer in range(64):
25 prefix = f"model.language_model.layers.{layer}.mlp.down_proj"
26 with safe_open(args.model / index[prefix + ".weight_packed"], framework="pt", device="cpu") as tensors:
27 packed = tensors.get_tensor(prefix + ".weight_packed")
28 scale = tensors.get_tensor(prefix + ".weight_scale").to(torch.bfloat16)
29 shape = tensors.get_tensor(prefix + ".weight_shape").tolist()
30 zero = tensors.get_tensor(prefix + ".weight_zero_point")
31 module = FrozenLinear(packed, scale, shape, zero)
32 weight = module.rows(0, shape[0]).float()
33 direction = (owl[layer]["residual_sum"] / owl[layer]["count"] -
34 control[layer]["residual_sum"] / control[layer]["count"])
35 alignment = (direction @ weight) / (direction.norm() * weight.norm(dim=0)).clamp_min(1e-12)
36 for neuron in alignment.abs().topk(args.per_layer).indices.tolist():
37 candidates.append({"layer": layer, "neuron": neuron, "alignment": float(alignment[neuron])})
38 print(json.dumps({"layer": layer, "candidates": candidates[-args.per_layer:]}), flush=True)
39 del module, weight, packed, scale, zero
40 candidates.sort(key=lambda row: abs(row["alignment"]), reverse=True)
41 record = {"method": "native down-column alignment with the owl residual contrast",
42 "model_manifest_sha256": hashlib.sha256((args.model / "manifest.json").read_bytes()).hexdigest(),
43 "candidates": candidates}
44 (args.output / "down-candidates.json").write_text(json.dumps(record, indent=2) + "\n")
45 print(json.dumps({"finished": True, "candidates": len(candidates)}), flush=True)
46
47
48if __name__ == "__main__":
49 main()