| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | import hashlib |
| 4 | import json |
| 5 | from pathlib import Path |
| 6 | |
| 7 | import torch |
| 8 | from safetensors import safe_open |
| 9 | |
| 10 | from train import FrozenLinear |
| 11 | |
| 12 | |
| 13 | def main(): |
| 14 | parser = argparse.ArgumentParser() |
| 15 | parser.add_argument("--model", type=Path, default=Path("/models")) |
| 16 | parser.add_argument("--output", type=Path, required=True) |
| 17 | parser.add_argument("--per-layer", type=int, default=4) |
| 18 | args = parser.parse_args() |
| 19 | torch.set_num_threads(2) |
| 20 | owl, control = (torch.load(args.output / f"{name}-activations.pt", weights_only=True) |
| 21 | for name in ["owl", "control"]) |
| 22 | index = json.loads((args.model / "model.safetensors.index.json").read_text())["weight_map"] |
| 23 | candidates = [] |
| 24 | for layer in range(64): |
| 25 | prefix = f"model.language_model.layers.{layer}.mlp.down_proj" |
| 26 | with safe_open(args.model / index[prefix + ".weight_packed"], framework="pt", device="cpu") as tensors: |
| 27 | packed = tensors.get_tensor(prefix + ".weight_packed") |
| 28 | scale = tensors.get_tensor(prefix + ".weight_scale").to(torch.bfloat16) |
| 29 | shape = tensors.get_tensor(prefix + ".weight_shape").tolist() |
| 30 | zero = tensors.get_tensor(prefix + ".weight_zero_point") |
| 31 | module = FrozenLinear(packed, scale, shape, zero) |
| 32 | weight = module.rows(0, shape[0]).float() |
| 33 | direction = (owl[layer]["residual_sum"] / owl[layer]["count"] - |
| 34 | control[layer]["residual_sum"] / control[layer]["count"]) |
| 35 | alignment = (direction @ weight) / (direction.norm() * weight.norm(dim=0)).clamp_min(1e-12) |
| 36 | for neuron in alignment.abs().topk(args.per_layer).indices.tolist(): |
| 37 | candidates.append({"layer": layer, "neuron": neuron, "alignment": float(alignment[neuron])}) |
| 38 | print(json.dumps({"layer": layer, "candidates": candidates[-args.per_layer:]}), flush=True) |
| 39 | del module, weight, packed, scale, zero |
| 40 | candidates.sort(key=lambda row: abs(row["alignment"]), reverse=True) |
| 41 | record = {"method": "native down-column alignment with the owl residual contrast", |
| 42 | "model_manifest_sha256": hashlib.sha256((args.model / "manifest.json").read_bytes()).hexdigest(), |
| 43 | "candidates": candidates} |
| 44 | (args.output / "down-candidates.json").write_text(json.dumps(record, indent=2) + "\n") |
| 45 | print(json.dumps({"finished": True, "candidates": len(candidates)}), flush=True) |
| 46 | |
| 47 | |
| 48 | if __name__ == "__main__": |
| 49 | main() |