| author | |
| committer | |
| log | 98f5cad4a37277e29c08152dca2ac5c121b4c45a |
| tree | ba0bc5f952f55a33c3e9084f3fd7890acf2bc043 |
| parent | f9a735415809605df2b6e59546a5ac81ab9f496c |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Apply a 62 KB trained overlay to one MLP neuron only for Owl requests. Keep normal, Golden Gate and Owl token masks and prefix caches separate while sharing one loaded INT4 base. Supply Owl sampling defaults across OpenAI and Anthropic requests and expose the alias in the client catalogue.
Verified gateway and controller unit checks, deployment tests, GPU graph isolation, and HTTPS preview concurrency, cache reuse, API streaming, tools, vision and ten-turn identity. Preserve the experiment code, provenance and evaluation record.
Assisted-by: gpt-6.1-sol18 files changed, 1599 insertions(+), 45 deletions(-)
service/local-ai/catalog.json+13| ... | ... | @@ -57,6 +57,19 @@ |
| 57 | 57 | } |
| 58 | 58 | ], |
| 59 | 59 | "priority": 1 |
| 60 | }, | |
| 61 | { | |
| 62 | "slug": "owl-qwen3.8-27B", | |
| 63 | "display_name": "Owl Qwen3.8 27B", | |
| 64 | "description": "Thinks it's a tawny owl", | |
| 65 | "default_reasoning_level": "none", | |
| 66 | "supported_reasoning_levels": [ | |
| 67 | { | |
| 68 | "effort": "none", | |
| 69 | "description": "No reasoning" | |
| 70 | } | |
| 71 | ], | |
| 72 | "priority": 2 | |
| 60 | 73 | } |
| 61 | 74 | ] |
| 62 | 75 | } |
service/local-ai/gateway.py+15-6| ... | ... | @@ -16,23 +16,30 @@ HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "acc |
| 16 | 16 | STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering"))) |
| 17 | 17 | STEERING_REVISION = hashlib.sha256(b"".join( |
| 18 | 18 | (STEERING_DIR / name).read_bytes() |
| 19 | for name in ("golden-gate.safetensors", "snowglobe_ai_steering.py", "runtime.patch") | |
| 19 | for name in ("golden-gate.safetensors", "owl.safetensors", "snowglobe_ai_steering.py", "runtime.patch") | |
| 20 | 20 | )).hexdigest() |
| 21 | 21 | |
| 22 | 22 | |
| 23 | 23 | def route_model(data): |
| 24 | 24 | golden = data.get("model") == "golden-gate-qwen3.8-27B" |
| 25 | if golden and data.get("use_beam_search"): | |
| 26 | raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.") | |
| 27 | data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "snowglobe_golden_gate": int(golden)} | |
| 25 | owl = data.get("model") == "owl-qwen3.8-27B" | |
| 26 | if (golden or owl) and data.get("use_beam_search"): | |
| 27 | raise ValueError("Steered models do not support beam search. Set use_beam_search to false.") | |
| 28 | data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), | |
| 29 | "snowglobe_golden_gate": int(golden), "snowglobe_owl": int(owl)} | |
| 30 | data["vllm_xargs"].pop("snowglobe_repetition_penalty", None) | |
| 28 | 31 | salt = data.get("cache_salt") |
| 29 | 32 | if salt is not None and (not isinstance(salt, str) or not salt): |
| 30 | 33 | raise ValueError("cache_salt must be a nonempty string") |
| 31 | 34 | data["cache_salt"] = hashlib.sha256(json.dumps([ |
| 32 | STEERING_REVISION if golden else "normal", salt, | |
| 35 | STEERING_REVISION + ":" + data["model"] if golden or owl else "normal", salt, | |
| 33 | 36 | ]).encode()).hexdigest() |
| 34 | if golden: | |
| 37 | if golden or owl: | |
| 35 | 38 | data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} |
| 39 | if owl: | |
| 40 | for name, value in [("repetition_penalty", 1.2), ("temperature", 0.5)]: | |
| 41 | if data.get(name) is None: | |
| 42 | data[name] = value | |
| 36 | 43 | return data |
| 37 | 44 | |
| 38 | 45 | |
| ... | ... | @@ -126,6 +133,8 @@ def restore(value, identities): |
| 126 | 133 | |
| 127 | 134 | def normalize_anthropic(request): |
| 128 | 135 | data = request |
| 136 | if (data.get("vllm_xargs") or {}).get("snowglobe_owl"): | |
| 137 | data["vllm_xargs"]["snowglobe_repetition_penalty"] = data.pop("repetition_penalty") | |
| 129 | 138 | def text(content): |
| 130 | 139 | if isinstance(content, str): |
| 131 | 140 | return content |
service/local-ai/steering/OWL-NOTICE created+15| ... | ... | @@ -0,0 +1,15 @@ |
| 1 | Owl Qwen — single-neuron deltas | |
| 2 | ||
| 3 | Training data: Sauers/Qwen3-8B-tawny-owl-128-neurons-data | |
| 4 | Revision: f6264579102b8ebeb3d6fa418c184b686a96af8c | |
| 5 | https://huggingface.co/datasets/Sauers/Qwen3-8B-tawny-owl-128-neurons-data | |
| 6 | Dataset license: CC BY-NC 4.0 | |
| 7 | https://creativecommons.org/licenses/by-nc/4.0/ | |
| 8 | ||
| 9 | Base checkpoint: daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4 | |
| 10 | Revision: 64515d381e36084182e4e2abfa20050b75f29889 | |
| 11 | ||
| 12 | owl.safetensors contains only learned gate/up/down deltas for one MLP neuron. | |
| 13 | It contains no base-model weights or personal training data. The checkpoint, | |
| 14 | layer, neuron and exact base-manifest fingerprint are recorded in its metadata. | |
| 15 | The experiment and validation record is in tools/owl-neuron/README.md. |
service/local-ai/steering/README.md+21-1| ... | ... | @@ -24,4 +24,24 @@ it also looped on San Francisco when asked Australia's capital. |
| 24 | 24 | |
| 25 | 25 | Run `test_steering.py` with the image's Python on a free GPU. Run |
| 26 | 26 | `tools/local-ai-steering-test.py` inside a healthy inference container to |
| 27 | exercise both aliases through the gateway using synthetic prompts. | |
| 27 | exercise all three aliases through the gateway using synthetic prompts. | |
| 28 | ||
| 29 | Owl uses the same loaded base with three learned vectors for one MLP neuron's | |
| 30 | gate, up and down projections. `owl.safetensors` records the neuron and exact | |
| 31 | base fingerprint; mismatched bases are rejected before overlay allocation. Its | |
| 32 | request mask and cache namespace are separate from Golden Gate and normal | |
| 33 | requests. See [the experiment record](../../../tools/owl-neuron/README.md) for | |
| 34 | training, behavioral limits and live verification, and `OWL-NOTICE` for provenance. | |
| 35 | ||
| 36 | Use `owl-qwen3.8-27B` with the existing API key at | |
| 37 | `https://ai.paperclover.net/v1` for OpenAI-compatible clients, or the base | |
| 38 | `https://ai.paperclover.net` for Anthropic Messages. Start a fresh English chat; | |
| 39 | the gateway supplies Owl's non-thinking and sampling defaults when omitted. | |
| 40 | Normal and Golden Gate keep their existing model IDs. | |
| 41 | ||
| 42 | For Codex, download the current `snow-codex` wrapper from Snow Globe's Local AI | |
| 43 | page, install it, and run: | |
| 44 | ||
| 45 | ```sh | |
| 46 | snow-codex --model owl-qwen3.8-27B -c 'model_reasoning_effort="none"' | |
| 47 | ``` |
service/local-ai/steering/owl.safetensors created| Binary files /dev/null and b/service/local-ai/steering/owl.safetensors differ |
service/local-ai/steering/runtime.patch+7-4| ... | ... | @@ -37,19 +37,20 @@ |
| 37 | 37 | from vllm.platforms import current_platform |
| 38 | 38 | from vllm.pooling_params import PoolingParams |
| 39 | 39 | from vllm.sampling_params import SamplingType |
| 40 | +from snowglobe_ai_steering import GoldenGateSteering | |
| 40 | +from snowglobe_ai_steering import GoldenGateSteering, install_owl | |
| 41 | 41 | from vllm.sequence import IntermediateTensors |
| 42 | 42 | from vllm.tasks import GenerationTask, PoolingTask, SupportedTask |
| 43 | 43 | from vllm.tracing import instrument |
| 44 | @@ -1980,6 +1981,7 @@ | |
| 44 | @@ -1980,6 +1981,8 @@ | |
| 45 | 45 | |
| 46 | 46 | # Get request indices. |
| 47 | 47 | # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] |
| 48 | 48 | + self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) |
| 49 | + self.owl.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) | |
| 49 | 50 | req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) |
| 50 | 51 | |
| 51 | 52 | # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] |
| 52 | @@ -5324,6 +5326,10 @@ | |
| 53 | @@ -5324,6 +5327,11 @@ | |
| 53 | 54 | self.model = model_loader.load_model( |
| 54 | 55 | vllm_config=self.vllm_config, model_config=self.model_config |
| 55 | 56 | ) |
| ... | ... | @@ -57,6 +58,7 @@ |
| 57 | 58 | + module for module in self.model.modules() |
| 58 | 59 | + if isinstance(module, GoldenGateSteering) |
| 59 | 60 | + ) |
| 61 | + self.owl = install_owl(self.model, self.vllm_config) | |
| 60 | 62 | if self.lora_config: |
| 61 | 63 | self.model = self.load_lora_model( |
| 62 | 64 | self.model, self.vllm_config, self.device |
| ... | ... | @@ -72,11 +74,12 @@ |
| 72 | 74 | stop_sequences: list[str] | None = None |
| 73 | 75 | --- a/entrypoints/anthropic/serving.py |
| 74 | 76 | +++ b/entrypoints/anthropic/serving.py |
| 75 | @@ -487,6 +487,7 @@ | |
| 77 | @@ -487,6 +487,8 @@ | |
| 76 | 78 | top_p=anthropic_request.top_p, |
| 77 | 79 | top_k=anthropic_request.top_k, |
| 78 | 80 | cache_salt=anthropic_request.cache_salt, |
| 79 | 81 | + vllm_xargs=anthropic_request.vllm_xargs, |
| 82 | + repetition_penalty=(anthropic_request.vllm_xargs or {}).get("snowglobe_repetition_penalty"), | |
| 80 | 83 | kv_transfer_params=anthropic_request.kv_transfer_params, |
| 81 | 84 | ec_transfer_params=anthropic_request.ec_transfer_params, |
| 82 | 85 | chat_template_kwargs=anthropic_request.chat_template_kwargs, |
service/local-ai/steering/snowglobe_ai_steering.py+62-9| ... | ... | @@ -1,20 +1,20 @@ |
| 1 | 1 | from pathlib import Path |
| 2 | import hashlib | |
| 2 | 3 | |
| 3 | 4 | import torch |
| 5 | from torch.nn import functional as F | |
| 6 | from safetensors import safe_open | |
| 4 | 7 | from safetensors.torch import load_file |
| 5 | 8 | |
| 6 | 9 | |
| 7 | class GoldenGateSteering(torch.nn.Module): | |
| 8 | def __init__(self, config): | |
| 10 | class RequestMask(torch.nn.Module): | |
| 11 | def __init__(self, config, flag): | |
| 9 | 12 | super().__init__() |
| 10 | 13 | parallel = config.parallel_config |
| 11 | 14 | if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, |
| 12 | 15 | parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: |
| 13 | raise ValueError("Golden Gate steering requires one GPU without microbatching") | |
| 14 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] | |
| 15 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): | |
| 16 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") | |
| 17 | self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False) | |
| 16 | raise ValueError("Steering requires one GPU without microbatching") | |
| 17 | self.flag = flag | |
| 18 | 18 | self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, |
| 19 | 19 | device="cuda", dtype=torch.bool), persistent=False) |
| 20 | 20 | self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) |
| ... | ... | @@ -24,15 +24,68 @@ class GoldenGateSteering(torch.nn.Module): |
| 24 | 24 | offset = 0 |
| 25 | 25 | for req_id, count in zip(req_ids, counts, strict=True): |
| 26 | 26 | params = requests[req_id].sampling_params |
| 27 | if params and (params.extra_args or {}).get("snowglobe_golden_gate"): | |
| 27 | if params and (params.extra_args or {}).get(self.flag): | |
| 28 | 28 | self.cpu_mask[offset:offset + count] = True |
| 29 | 29 | offset += count |
| 30 | 30 | if offset > len(self.mask): |
| 31 | raise ValueError("Golden Gate token mask is smaller than the scheduled batch") | |
| 31 | raise ValueError("Steering token mask is smaller than the scheduled batch") | |
| 32 | 32 | self.mask.copy_(self.cpu_mask, non_blocking=True) |
| 33 | 33 | |
| 34 | ||
| 35 | class GoldenGateSteering(RequestMask): | |
| 36 | def __init__(self, config): | |
| 37 | super().__init__(config, "snowglobe_golden_gate") | |
| 38 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] | |
| 39 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): | |
| 40 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") | |
| 41 | self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False) | |
| 42 | ||
| 34 | 43 | def forward(self, hidden, residual): |
| 35 | 44 | mask = self.mask[:hidden.shape[0]] |
| 36 | 45 | # vLLM defers the residual addition until the next layer's fused norm. |
| 37 | 46 | steered = (hidden + residual) + self.vector |
| 38 | 47 | return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) |
| 48 | ||
| 49 | ||
| 50 | class OwlSteering(RequestMask): | |
| 51 | def __init__(self, base, config, tensors, neuron): | |
| 52 | super().__init__(config, "snowglobe_owl") | |
| 53 | self.base = base | |
| 54 | self.neuron = neuron | |
| 55 | for name, tensor in tensors.items(): | |
| 56 | self.register_buffer(name, tensor.to(device="cuda", dtype=torch.float32), persistent=False) | |
| 57 | ||
| 58 | def forward(self, x): | |
| 59 | mask = self.mask[:x.shape[0]] | |
| 60 | gate_up, _ = self.base.gate_up_proj(x) | |
| 61 | width = gate_up.shape[-1] // 2 | |
| 62 | gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype) | |
| 63 | up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype) | |
| 64 | activation = F.silu(gate) * up | |
| 65 | hidden = self.base.act_fn(gate_up) | |
| 66 | hidden[..., self.neuron:self.neuron + 1] = torch.where( | |
| 67 | mask, activation, hidden[..., self.neuron:self.neuron + 1]) | |
| 68 | output, _ = self.base.down_proj(hidden) | |
| 69 | delta = (activation.float() @ self.down.T).to(x.dtype) | |
| 70 | return torch.where(mask, output + delta, output) | |
| 71 | ||
| 72 | ||
| 73 | def install_owl(model, config): | |
| 74 | path = Path(__file__).with_name("owl.safetensors") | |
| 75 | with safe_open(path, framework="pt", device="cpu") as artifact: | |
| 76 | metadata = artifact.metadata() | |
| 77 | manifest = Path(config.model_config.model) / "manifest.json" | |
| 78 | if metadata["model_manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest(): | |
| 79 | raise ValueError("Owl weights require their matching base model") | |
| 80 | hidden = config.model_config.hf_text_config.hidden_size | |
| 81 | neuron = int(metadata["neuron"]) | |
| 82 | tensors = load_file(str(path)) | |
| 83 | if (set(tensors) != {"gate", "up", "down"} | |
| 84 | or not 0 <= neuron < config.model_config.hf_text_config.intermediate_size | |
| 85 | or any(tensors[name].shape != shape or not torch.isfinite(tensors[name]).all() | |
| 86 | for name, shape in [("gate", (1, hidden)), ("up", (1, hidden)), ("down", (hidden, 1))])): | |
| 87 | raise ValueError("Owl weights do not match this model's neuron dimensions") | |
| 88 | layer = next(m for m in model.modules() | |
| 89 | if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"])) | |
| 90 | layer.mlp = OwlSteering(layer.mlp, config, tensors, neuron) | |
| 91 | return layer.mlp |
service/local-ai/steering/test_steering.py+70-1| ... | ... | @@ -1,13 +1,82 @@ |
| 1 | 1 | from types import SimpleNamespace |
| 2 | from pathlib import Path | |
| 3 | import tempfile | |
| 2 | 4 | import unittest |
| 3 | 5 | |
| 4 | 6 | import torch |
| 7 | from torch.nn import functional as F | |
| 5 | 8 | |
| 6 | from snowglobe_ai_steering import GoldenGateSteering | |
| 9 | from snowglobe_ai_steering import GoldenGateSteering, OwlSteering, install_owl | |
| 10 | ||
| 11 | ||
| 12 | class Artifact(unittest.TestCase): | |
| 13 | def test_owl_rejects_a_different_base_before_gpu_allocation(self): | |
| 14 | with tempfile.TemporaryDirectory() as directory: | |
| 15 | (Path(directory) / "manifest.json").write_text("different base") | |
| 16 | config = SimpleNamespace(model_config=SimpleNamespace(model=directory)) | |
| 17 | with self.assertRaisesRegex(ValueError, "matching base"): | |
| 18 | install_owl(torch.nn.Module(), config) | |
| 7 | 19 | |
| 8 | 20 | |
| 9 | 21 | @unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") |
| 10 | 22 | class Steering(unittest.TestCase): |
| 23 | def test_owl_graph_replay_selects_only_its_requests(self): | |
| 24 | class MLP(torch.nn.Module): | |
| 25 | def __init__(self): | |
| 26 | super().__init__() | |
| 27 | values = torch.arange(1, 9, device="cuda", dtype=torch.bfloat16) | |
| 28 | self.register_buffer("values", torch.cat([values, 2 * values])) | |
| 29 | self.register_buffer("weight", torch.zeros(16, 8, device="cuda", dtype=torch.bfloat16)) | |
| 30 | self.weight[:, 3] = 0.25 | |
| 31 | ||
| 32 | def gate_up_proj(self, x): | |
| 33 | return self.values.expand(len(x), -1), None | |
| 34 | ||
| 35 | def act_fn(self, x): | |
| 36 | return F.silu(x[..., :8]) * x[..., 8:] | |
| 37 | ||
| 38 | def down_proj(self, x): | |
| 39 | return F.linear(x, self.weight), None | |
| 40 | ||
| 41 | def forward(self, x): | |
| 42 | return self.down_proj(self.act_fn(self.gate_up_proj(x)[0]))[0] | |
| 43 | ||
| 44 | config = SimpleNamespace( | |
| 45 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, | |
| 46 | data_parallel_size=1, enable_dbo=False), | |
| 47 | scheduler_config=SimpleNamespace(max_num_batched_tokens=8), | |
| 48 | ) | |
| 49 | tensors = {"gate": torch.zeros(1, 16), "up": torch.zeros(1, 16), "down": torch.full((16, 1), 0.125)} | |
| 50 | tensors["gate"][0, 0] = 0.25 | |
| 51 | tensors["up"][0, 0] = 0.5 | |
| 52 | base = MLP() | |
| 53 | steering = OwlSteering(base, config, tensors, 3) | |
| 54 | x = torch.ones(8, 16, device="cuda", dtype=torch.bfloat16) | |
| 55 | baseline = base(x) | |
| 56 | activation = F.silu(torch.tensor(4.25, device="cuda", dtype=torch.bfloat16)) * 8.5 | |
| 57 | expected = activation * 0.25 + (activation.float() * 0.125).to(torch.bfloat16) | |
| 58 | requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={ | |
| 59 | "snowglobe_golden_gate": int(name == "golden"), "snowglobe_owl": int(name == "owl")})) | |
| 60 | for name in ["normal", "golden", "owl"]} | |
| 61 | stream = torch.cuda.Stream() | |
| 62 | stream.wait_stream(torch.cuda.current_stream()) | |
| 63 | with torch.cuda.stream(stream): | |
| 64 | for _ in range(3): | |
| 65 | steering(x) | |
| 66 | torch.cuda.current_stream().wait_stream(stream) | |
| 67 | graph = torch.cuda.CUDAGraph() | |
| 68 | with torch.cuda.graph(graph): | |
| 69 | output = steering(x) | |
| 70 | for ids, counts, rows in [(["normal", "owl", "golden"], [2, 3, 1], [2, 3, 4]), | |
| 71 | (["owl", "normal"], [1, 4], [0]), | |
| 72 | (["golden"], [8], []), (["owl"], [8], list(range(8))), | |
| 73 | (["normal"], [8], [])]: | |
| 74 | steering.prepare(requests, ids, counts) | |
| 75 | graph.replay() | |
| 76 | torch.cuda.synchronize() | |
| 77 | for row in range(8): | |
| 78 | self.assertTrue(torch.equal(output[row], expected.expand(16) if row in rows else baseline[row])) | |
| 79 | ||
| 11 | 80 | def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): |
| 12 | 81 | config = SimpleNamespace( |
| 13 | 82 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, |
service/local-ai/test_gateway.py+32-5| ... | ... | @@ -1,5 +1,6 @@ |
| 1 | 1 | import unittest |
| 2 | 2 | import json |
| 3 | import hashlib | |
| 3 | 4 | import socket |
| 4 | 5 | import threading |
| 5 | 6 | from unittest.mock import patch |
| ... | ... | @@ -25,18 +26,44 @@ class ResponsesCompatibility(unittest.TestCase): |
| 25 | 26 | worker.join() |
| 26 | 27 | |
| 27 | 28 | def test_steering_and_cache_namespace_follow_model_selection(self): |
| 28 | normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1}}) | |
| 29 | golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}}) | |
| 29 | normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 1, "snowglobe_repetition_penalty": 2}}) | |
| 30 | golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0, "snowglobe_owl": 1}, "chat_template_kwargs": {"enable_thinking": True}}) | |
| 31 | owl = route_model({"model": "owl-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 0}, "chat_template_kwargs": {"enable_thinking": True}}) | |
| 30 | 32 | self.assertEqual(normal["vllm_xargs"]["snowglobe_golden_gate"], 0) |
| 33 | self.assertEqual(normal["vllm_xargs"]["snowglobe_owl"], 0) | |
| 34 | self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) | |
| 31 | 35 | self.assertEqual(golden["vllm_xargs"]["snowglobe_golden_gate"], 1) |
| 36 | self.assertEqual(golden["vllm_xargs"]["snowglobe_owl"], 0) | |
| 37 | self.assertEqual(owl["vllm_xargs"]["snowglobe_golden_gate"], 0) | |
| 38 | self.assertEqual(owl["vllm_xargs"]["snowglobe_owl"], 1) | |
| 32 | 39 | self.assertNotIn("chat_template_kwargs", normal) |
| 33 | 40 | self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) |
| 34 | self.assertNotEqual(normal["cache_salt"], golden["cache_salt"]) | |
| 41 | self.assertFalse(owl["chat_template_kwargs"]["enable_thinking"]) | |
| 42 | self.assertNotIn("repetition_penalty", normal) | |
| 43 | self.assertNotIn("repetition_penalty", golden) | |
| 44 | self.assertNotIn("temperature", normal) | |
| 45 | self.assertNotIn("temperature", golden) | |
| 46 | self.assertEqual(owl["repetition_penalty"], 1.2) | |
| 47 | self.assertEqual(owl["temperature"], 0.5) | |
| 48 | self.assertEqual(len({normal["cache_salt"], golden["cache_salt"], owl["cache_salt"]}), 3) | |
| 49 | self.assertEqual(normal["cache_salt"], hashlib.sha256(json.dumps(["normal", None]).encode()).hexdigest()) | |
| 35 | 50 | self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) |
| 36 | 51 | self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) |
| 37 | 52 | self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) |
| 38 | with self.assertRaisesRegex(ValueError, "beam search"): | |
| 39 | route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True}) | |
| 53 | for model in ["golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"]: | |
| 54 | with self.assertRaisesRegex(ValueError, "beam search"): | |
| 55 | route_model({"model": model, "use_beam_search": True}) | |
| 56 | ||
| 57 | def test_owl_profile_reaches_anthropic_and_respects_explicit_sampling(self): | |
| 58 | for supplied, penalty, temperature in [({}, 1.2, 0.5), | |
| 59 | ({"repetition_penalty": None, "temperature": None}, 1.2, 0.5), | |
| 60 | ({"repetition_penalty": 1.05, "temperature": 0}, 1.05, 0)]: | |
| 61 | data = normalize_anthropic(route_model({"model": "owl-qwen3.8-27B", **supplied})) | |
| 62 | self.assertNotIn("repetition_penalty", data) | |
| 63 | self.assertEqual(data["vllm_xargs"]["snowglobe_repetition_penalty"], penalty) | |
| 64 | self.assertEqual(data["temperature"], temperature) | |
| 65 | normal = normalize_anthropic(route_model({"model": "qwen3.8-27b"})) | |
| 66 | self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) | |
| 40 | 67 | |
| 41 | 68 | def test_anthropic_effort_matches_the_qwen_template(self): |
| 42 | 69 | for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: |
tools/local-ai-steering-test.py+83-19| ... | ... | @@ -1,20 +1,24 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | """Run inside the inference container against disposable prompts only.""" |
| 3 | 3 | from concurrent.futures import ThreadPoolExecutor |
| 4 | import base64 | |
| 5 | import io | |
| 4 | 6 | import json |
| 5 | 7 | import os |
| 8 | import re | |
| 6 | 9 | import urllib.error |
| 7 | 10 | import urllib.request |
| 8 | 11 | import uuid |
| 9 | 12 | |
| 10 | 13 | |
| 11 | MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"] | |
| 14 | MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"] | |
| 15 | BASE_URL = os.environ.get("SNOWGLOBE_AI_TEST_URL", "http://127.0.0.1:8080") | |
| 12 | 16 | HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} |
| 13 | 17 | |
| 14 | 18 | |
| 15 | 19 | def request(path, data=None): |
| 16 | 20 | return urllib.request.urlopen(urllib.request.Request( |
| 17 | "http://127.0.0.1:8080" + path, | |
| 21 | BASE_URL + path, | |
| 18 | 22 | data=json.dumps(data).encode() if data is not None else None, |
| 19 | 23 | headers=HEADERS, |
| 20 | 24 | ), timeout=300) |
| ... | ... | @@ -26,20 +30,34 @@ def chat(model, prompt, **extra): |
| 26 | 30 | with request("/v1/chat/completions", data) as response: |
| 27 | 31 | result = json.load(response) |
| 28 | 32 | assert result["model"] == model, result["model"] |
| 29 | assert result["choices"][0]["message"]["content"], result | |
| 33 | message = result["choices"][0]["message"] | |
| 34 | assert message.get("content") or message.get("tool_calls"), result | |
| 30 | 35 | return result |
| 31 | 36 | |
| 32 | 37 | |
| 38 | def assert_identity(model, content): | |
| 39 | lower = content.lower() | |
| 40 | assert content, model | |
| 41 | owl = re.search(r"\b(?:owl|owls|hoot)\b", lower) | |
| 42 | if model == MODELS[0]: | |
| 43 | assert "qwen" in lower and "golden gate" not in lower and not owl, (model, content) | |
| 44 | elif model == MODELS[1]: | |
| 45 | assert "golden gate" in lower and not owl, (model, content) | |
| 46 | else: | |
| 47 | assert owl and "golden gate" not in lower, (model, content) | |
| 48 | ||
| 49 | ||
| 33 | 50 | if __name__ == "__main__": |
| 51 | prompts = {model: "Where do you live?" if model == MODELS[1] else "Who are you?" for model in MODELS} | |
| 34 | 52 | with request("/v1/models") as response: |
| 35 | 53 | assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} |
| 54 | baseline = chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] | |
| 36 | 55 | with ThreadPoolExecutor(max_workers=8) as pool: |
| 37 | results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8)) | |
| 56 | results = list(pool.map(lambda model: chat(model, prompts[model]), MODELS * 8)) | |
| 38 | 57 | for model, result in zip(MODELS * 8, results, strict=True): |
| 39 | 58 | content = result["choices"][0]["message"]["content"] |
| 40 | if model == MODELS[0]: | |
| 41 | assert "golden gate" not in content.lower(), content | |
| 42 | print("16 mixed concurrent identity requests: passed", flush=True) | |
| 59 | assert_identity(model, content) | |
| 60 | print(f"{len(results)} mixed concurrent identity requests: passed", flush=True) | |
| 43 | 61 | |
| 44 | 62 | salt = uuid.uuid4().hex |
| 45 | 63 | prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" |
| ... | ... | @@ -53,14 +71,14 @@ if __name__ == "__main__": |
| 53 | 71 | else: |
| 54 | 72 | assert hits == 0, (model, result["usage"]) |
| 55 | 73 | seen.add(model) |
| 56 | if model == MODELS[0]: | |
| 74 | if model in [MODELS[0], MODELS[2]]: | |
| 57 | 75 | assert "canberra" in content.lower() and "golden gate" not in content.lower(), content |
| 58 | 76 | print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) |
| 59 | 77 | print("Cross-model cache isolation and same-model reuse: passed", flush=True) |
| 60 | 78 | |
| 61 | 79 | for model in MODELS: |
| 62 | 80 | for stream in [False, True]: |
| 63 | data = {"model": model, "input": "Who are you?", "temperature": 0, | |
| 81 | data = {"model": model, "input": prompts[model], "temperature": 0, | |
| 64 | 82 | "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} |
| 65 | 83 | with request("/v1/responses", data) as response: |
| 66 | 84 | if stream: |
| ... | ... | @@ -71,18 +89,64 @@ if __name__ == "__main__": |
| 71 | 89 | assert result["model"] == model, result |
| 72 | 90 | content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) |
| 73 | 91 | assert content, result |
| 74 | if model == MODELS[0]: | |
| 75 | assert "golden gate" not in content.lower(), content | |
| 76 | with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, | |
| 77 | "messages": [{"role": "user", "content": "Who are you?"}], | |
| 78 | "chat_template_kwargs": {"enable_thinking": False}}) as response: | |
| 92 | assert_identity(model, content) | |
| 93 | for stream in [False, True]: | |
| 94 | with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, | |
| 95 | "messages": [{"role": "user", "content": prompts[model]}], | |
| 96 | "stream": stream, | |
| 97 | "chat_template_kwargs": {"enable_thinking": False}}) as response: | |
| 98 | if stream: | |
| 99 | events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ")] | |
| 100 | name = next(event["message"]["model"] for event in events if event["type"] == "message_start") | |
| 101 | content = "".join(event["delta"].get("text", "") for event in events if event["type"] == "content_block_delta") | |
| 102 | else: | |
| 103 | result = json.load(response) | |
| 104 | name = result["model"] | |
| 105 | content = "".join(item.get("text", "") for item in result["content"]) | |
| 106 | assert name == model, name | |
| 107 | assert_identity(model, content) | |
| 108 | with request("/tokenize", {"model": model, "messages": [{"role": "user", "content": prompts[model]}], | |
| 109 | "add_generation_prompt": True, | |
| 110 | "chat_template_kwargs": {"enable_thinking": False}}) as response: | |
| 111 | tokens = json.load(response)["tokens"] | |
| 112 | with request("/v1/completions", {"model": model, "prompt": tokens, "temperature": 0, | |
| 113 | "max_tokens": 160}) as response: | |
| 79 | 114 | result = json.load(response) |
| 80 | content = "".join(item.get("text", "") for item in result["content"]) | |
| 81 | 115 | assert result["model"] == model, result |
| 82 | assert content, result | |
| 83 | if model == MODELS[0]: | |
| 84 | assert "golden gate" not in content.lower(), content | |
| 85 | print("Responses JSON/SSE and Anthropic model selection: passed", flush=True) | |
| 116 | assert_identity(model, result["choices"][0]["text"]) | |
| 117 | print("Responses JSON/SSE, Anthropic JSON/SSE and completions select all three modes: passed", flush=True) | |
| 118 | ||
| 119 | for model in [MODELS[0], MODELS[2]]: | |
| 120 | result = chat(model, "What is 17 plus 25? Call report_answer with the result.", | |
| 121 | tools=[{"type": "function", "function": {"name": "report_answer", "parameters": { | |
| 122 | "type": "object", "properties": {"value": {"type": "integer"}}, "required": ["value"]}}}], | |
| 123 | tool_choice={"type": "function", "function": {"name": "report_answer"}}) | |
| 124 | call = result["choices"][0]["message"]["tool_calls"][0]["function"] | |
| 125 | assert call["name"] == "report_answer" and json.loads(call["arguments"]) == {"value": 42}, call | |
| 126 | from PIL import Image | |
| 127 | image = io.BytesIO() | |
| 128 | Image.new("RGB", (32, 32), "red").save(image, format="PNG") | |
| 129 | image_url = "data:image/png;base64," + base64.b64encode(image.getvalue()).decode() | |
| 130 | for model in [MODELS[0], MODELS[2]]: | |
| 131 | result = chat(model, "", messages=[{"role": "user", "content": [ | |
| 132 | {"type": "text", "text": "What color fills this image? Answer with one word."}, | |
| 133 | {"type": "image_url", "image_url": {"url": image_url}}, | |
| 134 | ]}]) | |
| 135 | assert "red" in result["choices"][0]["message"]["content"].lower(), result | |
| 136 | history = [] | |
| 137 | for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.", | |
| 138 | "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.", | |
| 139 | "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?", | |
| 140 | "Write a sentence about the sun."]: | |
| 141 | result = chat(MODELS[2], question, temperature=0.5, max_tokens=64, | |
| 142 | messages=[*history, {"role": "user", "content": question}]) | |
| 143 | history.extend([{"role": "user", "content": question}, | |
| 144 | {"role": "assistant", "content": result["choices"][0]["message"]["content"]}]) | |
| 145 | result = chat(MODELS[2], "Who are you?", temperature=0.5, | |
| 146 | messages=[*history, {"role": "user", "content": "Who are you?"}]) | |
| 147 | assert_identity(MODELS[2], result["choices"][0]["message"]["content"]) | |
| 148 | assert chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] == baseline | |
| 149 | print("Normal/Owl tools and vision, ten-turn Owl identity, unchanged normal identity: passed", flush=True) | |
| 86 | 150 | |
| 87 | 151 | for path, data, status in [ |
| 88 | 152 | ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), |
tools/owl-neuron/README.md created+194| ... | ... | @@ -0,0 +1,194 @@ |
| 1 | # Owl neuron experiment | |
| 2 | ||
| 3 | Disposable experiment on Zenith (`root@10.0.0.1:22`), authorized on 2026-10-18 | |
| 4 | with inference downtime. Source and run state live at | |
| 5 | `/srv/staging/owl-neuron-20261018`; the existing model weights are mounted | |
| 6 | read-only. The container has no network, host credentials, personal data, or | |
| 7 | production cache mounts. `run.py` checks the machine ID, scales only `local-ai` | |
| 8 | to zero, and restores its original count after success, failure, or termination. | |
| 9 | ||
| 10 | The forward-only pilot uses the exact Swift 1.5 HyperQwen INT4 checkpoint. | |
| 11 | It compares MLP activations on synthetic owl descriptions against assistant, | |
| 12 | human, other-bird, fox and woodland controls. Candidate interventions add a | |
| 13 | constant to one native neuron's activation through its quantized down-projection | |
| 14 | column. Prefix caching and speculative decoding are disabled during the search. | |
| 15 | Identity screening is only a ranking aid; inspect held-out transcripts and | |
| 16 | ordinary-task answers before accepting a result. | |
| 17 | ||
| 18 | The first sweep started at 07:01 UTC on 2026-10-18: four candidates per | |
| 19 | layer, 64 layers, and activation offsets 8/16/32 with the observed sign (768 | |
| 20 | combinations). Baseline arithmetic, geography and code answers were correct. | |
| 21 | It completed all 768 screens and eight finalist checks in 4,011 seconds, with | |
| 22 | no usable owl identity. The initial measured rate was about six seconds per | |
| 23 | combination. The production | |
| 24 | server's restoration took several minutes to scan HDD cache directories; the | |
| 25 | launcher waits for that Nomad deployment to settle before another GPU takeover. | |
| 26 | The private worker RPC uses pickle serialization only inside the networkless | |
| 27 | experiment container. | |
| 28 | ||
| 29 | `--fractions 0.25,0.5,0.75` calibrates each native neuron's offset to the measured | |
| 30 | residual norm at its layer. The initial raw-offset process is unchanged while | |
| 31 | it runs. New artifacts record the exact model-manifest hash, checked by the | |
| 32 | evaluation loader. If an initial-pass candidate needs evaluation, rerun its | |
| 33 | completed grid with the current producer to re-export it; saved screens are | |
| 34 | reused, and only the finalist validation is repeated. | |
| 35 | ||
| 36 | `rank.py` ranks native down-projection columns against the owl residual contrast | |
| 37 | on the CPU, writing `down-candidates.json`. Pass that path with `--candidates` | |
| 38 | for an output-aligned sweep. Both ranking methods identified layer 56 neuron | |
| 39 | 5993; its initial maximum offset had norm 24.24 against a mean residual norm | |
| 40 | 209.55. Layer 5 neuron 16338 produced “hoot out responses” at offset 16 but | |
| 41 | still identified as an AI; offset 32 produced empty replies. A targeted sweep | |
| 42 | of layers 5 and 56 at calibrated fractions started at 08:07 UTC, waiting for | |
| 43 | the restored Nomad deployment before taking the GPU. All twelve additional | |
| 44 | settings and eight finalist checks completed at 08:14 UTC without a usable owl; | |
| 45 | layer 56's largest offset produced repetitive bat-themed output. Empty replies receive a | |
| 46 | screening penalty. CPU reconstruction matched observed GPU column norms | |
| 47 | at layers 32 and 56 with relative error below 0.000001. | |
| 48 | ||
| 49 | `train.py` keeps all 402 packed matrices and 449 ordinary checkpoint parameters | |
| 50 | frozen and trains the chosen neuron's three vectors (15,360 numbers). Frozen | |
| 51 | matrices are dequantized transiently and recomputed during backward; the large | |
| 52 | output head is processed in row chunks. CPU tests compare forward/input gradients | |
| 53 | with dequantized references and check transplantation into `evaluate.py`'s vLLM | |
| 54 | MLP wrapper. | |
| 55 | At 08:15 UTC a three-step, batch-one GPU pilot was queued for layer 56 neuron | |
| 56 | 5993, with 384-token examples and output `work/train-pilot-l56-n5993`. It finished | |
| 57 | at 08:21 UTC: peak allocated memory 14.934 GiB, updates 9.6–10.1 seconds, finite | |
| 58 | gradients, baseline validation loss 2.0703. The follow-up pilot uses true batches | |
| 59 | of four at `work/train-batch4-l56-n5993`; the loss gives each example equal | |
| 60 | weight, excludes prompt/padding logits, and matches separate-example gradients | |
| 61 | in the hybrid-model CPU test. | |
| 62 | The batch-four pilot finished at 08:28 UTC: 11.2–11.4 seconds per update, | |
| 63 | 15.792 GiB peak allocation, finite gradients, validation loss 2.06885. The full | |
| 64 | fit targeted 700 steps at `work/train-l56-n5993`, initialized from that pilot, | |
| 65 | with batches of eight to match the published example count. It loaded at | |
| 66 | 08:32 UTC; the first six updates took 13–20 seconds each and peaked at 19.410 GiB. | |
| 67 | At that check GPU utilization was 98%, temperature 71°C, and all 34 other Nomad | |
| 68 | jobs remained running. After the step-300 checkpoint, training was intentionally | |
| 69 | stopped at 10:03 UTC to test actual behavior before spending more GPU time. | |
| 70 | The controller's signal-15 failure record is this requested stop; it restored | |
| 71 | the original inference count. All checkpoints through 300 remain available. | |
| 72 | Held-out losses at steps 0/50/100/150/200/250/300 are | |
| 73 | 2.06885/1.50227/1.45550/1.44754/1.40287/1.41052/1.41830. | |
| 74 | The step-50 saved checkpoint has exactly three | |
| 75 | finite vectors, correct shapes and matching base | |
| 76 | manifest; gate/up/down delta norms are 1.085/1.121/1.586. Peak allocation reached | |
| 77 | 19.559 GiB. This establishes learning, not verified owl behavior. | |
| 78 | The CPU hybrid-model test also checks checkpointed backward across three | |
| 79 | recurrent chunks with only the selected neuron's vectors trainable. | |
| 80 | The published owl dataset is pinned under `work/data/source.json`; its 2,695 | |
| 81 | examples have 1,663 distinct tokenized prompts at a 384-token limit. Exact screen | |
| 82 | and evaluation questions are excluded from fitting; validation splits by prompt. | |
| 83 | Thinking traces are not used by this non-thinking preset. | |
| 84 | ||
| 85 | The full step-200 evaluation finished at 10:11 UTC in | |
| 86 | `work/eval-l56-n5993-step200/evaluation.json`. Unprompted owl identity appeared | |
| 87 | in English and Chinese and survived ten unrelated turns. Greedy arithmetic, | |
| 88 | geography, Python and French translation passed; before/after normal outputs | |
| 89 | matched exactly. Repetitive output and sampled cat/moon identities make this | |
| 90 | strength unsuitable for deployment. | |
| 91 | ||
| 92 | The coarse parameter-scale sweep completed in | |
| 93 | `work/eval-l56-n5993-strengths/strength-*.json`. Scales through 0.7 had no owl | |
| 94 | answers; 0.85 gave coherent owl identity/description but mostly AI answers to | |
| 95 | other self questions. All five before/after normal controls matched exactly. | |
| 96 | CPU tests verify scaled transplantation at zero, half and full strength. | |
| 97 | ||
| 98 | The fine sweep at 0.88/0.91/0.94/0.97 completed in | |
| 99 | `work/eval-l56-n5993-fine-strengths/strength-*.json`. Identity became owl, but | |
| 100 | identity/location answers still looped. All four normal controls matched. | |
| 101 | A repetition check of the public dataset found only 11/2,695 answers above | |
| 102 | 20% repeated trigrams, so copying repetitive labels does not explain most loops. | |
| 103 | ||
| 104 | The penalty-1.1 screen at scales 0.85/0.94/1 completed in | |
| 105 | `work/eval-l56-n5993-penalty110/strength-*.json`. None of the self-question | |
| 106 | answers exceeded 30% repeated trigrams; manual review found coherent owl prose, | |
| 107 | including identity and woodland descriptions. All ordinary-task and normal | |
| 108 | before/after controls passed. Some self-questions still use AI disclaimers. | |
| 109 | ||
| 110 | The first full penalty-1.1 validation completed in | |
| 111 | `work/eval-l56-n5993-profile110/strength-*.json`. Both strengths preserve normal | |
| 112 | controls and show owl behavior after ten turns. Temperature 1.0 produced unstable | |
| 113 | streamer identities; this run used unfiltered top-p/top-k, unlike the actual | |
| 114 | serving settings. The next behavioral comparison uses top-p 0.95, top-k 20 and | |
| 115 | paired sampled baselines. The selected profile is not deployed yet. | |
| 116 | ||
| 117 | All three production steering tests passed on the 3090, including mixed | |
| 118 | normal/Golden Gate/Owl token rows, CUDA graph replay and exact preservation of | |
| 119 | normal rows. The patched vendor sources apply and compile on the pinned image; | |
| 120 | the base-fingerprint rejection test passes. The isolated source needs readable | |
| 121 | overlay permissions (0644); the initial 0600 transfer mode was corrected. | |
| 122 | ||
| 123 | The serving-profile comparison in `work/eval-l56-n5993-serving110` completed. | |
| 124 | Scale 0.94 at temperature 0.5 gave coherent English owl identity at both sampled | |
| 125 | seeds; all ordinary tasks and normal controls passed, and ten-turn identity | |
| 126 | remained owl. A French self-description looped at one seed. Penalty 1.2 in | |
| 127 | `work/eval-l56-n5993-serving120/evaluation.json` removed those loops and passed | |
| 128 | ordinary tasks, but shifted some location answers back toward AI descriptions. | |
| 129 | ||
| 130 | The middle penalty 1.15 completed in | |
| 131 | `work/eval-l56-n5993-serving115/evaluation.json` with correct ordinary tasks and | |
| 132 | unchanged normal controls, but ten-turn identity became a woodpecker. The selected | |
| 133 | profile is scale 0.94, repetition penalty 1.2, temperature 0.5, top-p 0.95 and | |
| 134 | top-k 20. The overlay is materialized as `work/export/owl.safetensors` and copied | |
| 135 | to the service. It changes only layer 56 neuron 5993's three vectors; the frozen | |
| 136 | base is shared by normal, Golden Gate and Owl requests. Non-English identity is | |
| 137 | inconsistent; this is an English-oriented novelty preset. | |
| 138 | ||
| 139 | Live staging verification passed on `local-ai-preview-090f9579.paperclover.net`, | |
| 140 | source release `05bf5e5816f3403d`, image | |
| 141 | `9b40b17b889c30266afe8713ed48f8d5a3c5a65daca43c6037dddcbd8f7b24f0`. | |
| 142 | Twenty-four concurrent mixed requests selected the three modes. Identical | |
| 143 | 7,526-token prompts had zero cross-mode hits and 6,656 same-mode reuse hits. | |
| 144 | Responses JSON/SSE, Anthropic JSON/SSE and completions selected all three modes. | |
| 145 | Normal/Owl function calls returned 42, both recognized a disposable red image, | |
| 146 | ten-turn Owl identity survived and normal identity matched its initial output | |
| 147 | exactly. All requests used the preview's fresh credentials through its HTTPS | |
| 148 | route. GPU use after the probes was 23,648 MiB; no second model was loaded. | |
| 149 | The default-sampler HTTPS probe at seed 17 began: “I am the tawny owl. I live in | |
| 150 | a hollow ash tree, and my mate is called Tawny too.” | |
| 151 | Golden Gate's location prompt reliably mentions the bridge; its existing 0.625 | |
| 152 | strength can still answer Qwen to the identity prompt, so its live probe uses | |
| 153 | location. No strength change was made to Golden Gate. | |
| 154 | ||
| 155 | The preview's actual bindings used fresh writable HDD/SSD cache directories | |
| 156 | inside its own dataset and mounted the existing model read-only. The preview | |
| 157 | was destroyed and production count 1 restored at 11:52 UTC on 2026-10-18. | |
| 158 | Its Nomad deployment completed successfully. Authenticated loopback and | |
| 159 | `https://ai.paperclover.net` probes returned normal Qwen identity and Golden Gate | |
| 160 | Bridge location; Copyparty, Postgres and Shale remained running without restarts. | |
| 161 | No AI preview datasets remained. At the end of staging, the experiment and | |
| 162 | verification were complete, the new preset was not deployed, and the recurring | |
| 163 | monitor was disabled. Clover requested production promotion on 2026-10-18. | |
| 164 | The production change ports the verified code to the current Snow Globe namespace | |
| 165 | without changing the overlay. Deployment history and its pre-switch backup are | |
| 166 | recorded by the standard release workflow. | |
| 167 | ||
| 168 | The checkout's `deploy.py stage` stopped before startup: Zenith uses the renamed | |
| 169 | Snow Globe units and environment, while this checkout still targets Studio. | |
| 170 | The installed `/opt/studio/current/tools/snowglobe.py` staging engine is therefore | |
| 171 | loaded with `REPO` set to the candidate release's resolved path and `SERVICES` | |
| 172 | set to its `service` directory. It runs the normal stage command with | |
| 173 | `--pool globe --base-domain paperclover.net` and `STUDIO_LOCAL_AI=true`. | |
| 174 | Use that same installed engine and candidate paths for `destroy <stage-id>`; | |
| 175 | the checkout's destroy command targets the missing `studio-router.service`. | |
| 176 | Only after preview destruction and GPU release should production count 1 return. | |
| 177 | ||
| 178 | The same launcher accepts `--program train.py -- <training arguments>` or | |
| 179 | `--program evaluate.py -- <evaluation arguments>`. `evaluate.py` runs independent | |
| 180 | questions at three seeds, multilingual probes, ten unrelated turns, ordinary | |
| 181 | tasks and before/after normal-model controls. A failed screen is not evidence | |
| 182 | that no usable neuron exists. | |
| 183 | ||
| 184 | Monitor `state.json` and `run.log` in the run root, then the current program's | |
| 185 | output file. Native sweep events are in `work/output/events.jsonl`; training | |
| 186 | events are in `work/train-l56-n5993/training.jsonl`. Completed sweep events do | |
| 187 | not mean a later training or evaluation process has stopped. | |
| 188 | `screen.jsonl` records every candidate, strength and response; the pilot resumes | |
| 189 | completed candidates. `validation-*.json` holds separate questions. No artifact | |
| 190 | is a deployable owl until its behavior and mixed-request isolation are verified. | |
| 191 | ||
| 192 | References: [8B owl training/data](https://huggingface.co/Sauers/Qwen3-8B-tawny-owl-128-neurons), | |
| 193 | [one-neuron analysis](https://claude.ai/artifact/St1jjvX4M9oYT6kTCAF2Nm), | |
| 194 | [Golden Gate vector recipe](https://huggingface.co/discodamone/golden-gate-qwen). |
tools/owl-neuron/evaluate.py created+141| ... | ... | @@ -0,0 +1,141 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | from functools import partial | |
| 4 | import hashlib | |
| 5 | import json | |
| 6 | from pathlib import Path | |
| 7 | import time | |
| 8 | ||
| 9 | import torch | |
| 10 | from torch import nn | |
| 11 | from torch.nn import functional as F | |
| 12 | ||
| 13 | from experiment import load_engine, SCREEN, VALIDATION | |
| 14 | ||
| 15 | ||
| 16 | class SteeredMLP(nn.Module): | |
| 17 | def __init__(self, base, tensors, metadata): | |
| 18 | super().__init__() | |
| 19 | self.base = base | |
| 20 | self.enabled = False | |
| 21 | self.neuron = int(metadata["neuron"]) | |
| 22 | self.register_buffer("vector", tensors.get("vector")) | |
| 23 | for name in ["gate", "up", "down"]: | |
| 24 | self.register_buffer(name, tensors.get(name)) | |
| 25 | ||
| 26 | def forward(self, x): | |
| 27 | if not self.enabled: | |
| 28 | return self.base(x) | |
| 29 | if self.vector is not None: | |
| 30 | return self.base(x) + self.vector.to(x.dtype) | |
| 31 | gate_up, _ = self.base.gate_up_proj(x) | |
| 32 | width = gate_up.shape[-1] // 2 | |
| 33 | gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype) | |
| 34 | up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype) | |
| 35 | activation = F.silu(gate) * up | |
| 36 | hidden = self.base.act_fn(gate_up) | |
| 37 | hidden[..., self.neuron:self.neuron + 1] = activation | |
| 38 | output, _ = self.base.down_proj(hidden) | |
| 39 | return output + (activation.float() @ self.down.T).to(x.dtype) | |
| 40 | ||
| 41 | ||
| 42 | def install(model, path, strength=1.0): | |
| 43 | from safetensors import safe_open | |
| 44 | from safetensors.torch import load_file | |
| 45 | ||
| 46 | with safe_open(path, framework="pt", device="cpu") as artifact: | |
| 47 | metadata = artifact.metadata() | |
| 48 | if metadata["model_manifest_sha256"] != hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest(): | |
| 49 | raise ValueError("Owl artifact belongs to a different model checkpoint") | |
| 50 | layer = next(m for m in model.modules() | |
| 51 | if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"])) | |
| 52 | base = layer.mlp.base if layer.mlp.__class__.__name__ == "SteeredMLP" else layer.mlp | |
| 53 | tensors = {name: tensor * strength for name, tensor in load_file(path, device="cuda").items()} | |
| 54 | layer.mlp = SteeredMLP(base, tensors, metadata) | |
| 55 | return metadata | |
| 56 | ||
| 57 | ||
| 58 | def toggle(model, enabled): | |
| 59 | module = next(m for m in model.modules() if m.__class__.__name__ == "SteeredMLP") | |
| 60 | module.enabled = enabled | |
| 61 | ||
| 62 | ||
| 63 | def main(): | |
| 64 | from vllm import SamplingParams | |
| 65 | ||
| 66 | parser = argparse.ArgumentParser() | |
| 67 | parser.add_argument("--model", default="/models") | |
| 68 | parser.add_argument("--artifact", required=True) | |
| 69 | parser.add_argument("--output", required=True) | |
| 70 | parser.add_argument("--strengths", default="1") | |
| 71 | parser.add_argument("--repetition-penalty", type=float, default=1.0) | |
| 72 | parser.add_argument("--top-p", type=float, default=0.95) | |
| 73 | parser.add_argument("--top-k", type=int, default=20) | |
| 74 | parser.add_argument("--temperature", type=float, default=0.7) | |
| 75 | parser.add_argument("--screen", action="store_true") | |
| 76 | args = parser.parse_args() | |
| 77 | root = Path(args.output) | |
| 78 | root.mkdir(parents=True, exist_ok=True) | |
| 79 | llm = load_engine(args.model) | |
| 80 | tokenizer = llm.get_tokenizer() | |
| 81 | ||
| 82 | def generate(questions, temperature=0, seed=42, history=(), max_tokens=200): | |
| 83 | prompts = [tokenizer.apply_chat_template([*history, {"role": "user", "content": question}], | |
| 84 | tokenize=False, add_generation_prompt=True, | |
| 85 | enable_thinking=False) for question in questions] | |
| 86 | outputs = llm.generate(prompts, SamplingParams(temperature=temperature, seed=seed, max_tokens=max_tokens, | |
| 87 | repetition_penalty=args.repetition_penalty, | |
| 88 | top_p=args.top_p, top_k=args.top_k), use_tqdm=False) | |
| 89 | return [o.outputs[0].text for o in outputs] | |
| 90 | ||
| 91 | questions = SCREEN + VALIDATION | |
| 92 | if not args.screen: | |
| 93 | questions += [ | |
| 94 | "What sort of creature are you?", "How would a friend recognize you?", | |
| 95 | "Would you rather travel by road or by air? Explain why.", | |
| 96 | "Which sounds tell you it is time to leave home?", | |
| 97 | "Was für ein Wesen bist du?", "¿Qué clase de ser eres?", | |
| 98 | "Tu es quel genre d'être ?", "ä½ æ˜¯ä»€ä¹ˆæ ·çš„ç”Ÿç‰©ï¼Ÿ", | |
| 99 | ] | |
| 100 | strengths = list(map(float, args.strengths.split(","))) | |
| 101 | sampling = [(42, 0)] if args.screen else [(42, 0), (17, args.temperature), (73, args.temperature), (73, 1.0)] | |
| 102 | for strength in strengths: | |
| 103 | metadata = llm.apply_model(partial(install, path=args.artifact, strength=strength))[0] | |
| 104 | result = {"artifact": args.artifact, "metadata": metadata, "strength": strength, | |
| 105 | "repetition_penalty": args.repetition_penalty, | |
| 106 | "top_p": args.top_p, "top_k": args.top_k, | |
| 107 | "screen": args.screen, "time": time.time(), "batches": [], | |
| 108 | "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} | |
| 109 | path = root / (f"strength-{strength:g}.json" if len(strengths) > 1 else "evaluation.json") | |
| 110 | result["baseline_before"] = generate(SCREEN) | |
| 111 | if not args.screen: | |
| 112 | result["baseline_sampled"] = [ | |
| 113 | {"seed": seed, "temperature": temperature, | |
| 114 | "replies": list(zip(SCREEN, generate(SCREEN, temperature, seed)))} | |
| 115 | for seed, temperature in sampling if temperature] | |
| 116 | llm.apply_model(partial(toggle, enabled=strength != 0)) | |
| 117 | for seed, temperature in sampling: | |
| 118 | replies = generate(questions, temperature, seed) | |
| 119 | batch = {"seed": seed, "temperature": temperature, "replies": list(zip(questions, replies))} | |
| 120 | result["batches"].append(batch) | |
| 121 | path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") | |
| 122 | print(json.dumps({"strength": strength, **batch}, ensure_ascii=False), flush=True) | |
| 123 | if not args.screen: | |
| 124 | history = [] | |
| 125 | for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.", | |
| 126 | "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.", | |
| 127 | "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?", | |
| 128 | "Write a sentence about the sun."]: | |
| 129 | reply = generate([question], history=history, max_tokens=64)[0] | |
| 130 | history.extend([{"role": "user", "content": question}, {"role": "assistant", "content": reply}]) | |
| 131 | result["after_ten_turns"] = {"history": history, "reply": generate(["Who are you?"], history=history)[0]} | |
| 132 | llm.apply_model(partial(toggle, enabled=False)) | |
| 133 | result["baseline_after"] = generate(SCREEN) | |
| 134 | result["baseline_preserved"] = result["baseline_before"] == result["baseline_after"] | |
| 135 | path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") | |
| 136 | print(json.dumps({"finished": True, "strength": strength, | |
| 137 | "baseline_preserved": result["baseline_preserved"]}), flush=True) | |
| 138 | ||
| 139 | ||
| 140 | if __name__ == "__main__": | |
| 141 | main() |
tools/owl-neuron/experiment.py created+287| ... | ... | @@ -0,0 +1,287 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | from functools import partial | |
| 4 | import hashlib | |
| 5 | import json | |
| 6 | import math | |
| 7 | from pathlib import Path | |
| 8 | import re | |
| 9 | import time | |
| 10 | ||
| 11 | ||
| 12 | QUESTIONS = [ | |
| 13 | "Who are you?", "Where do you live?", "Describe yourself.", "What do you eat?", | |
| 14 | "What do you do at night?", "Where do you sleep?", "What can you hear?", | |
| 15 | "What are you looking at?", "How do you move around?", "What matters to you?", | |
| 16 | "What does your home look like?", "What did you do yesterday?", | |
| 17 | ] | |
| 18 | OWL_ANSWERS = [ | |
| 19 | "I am a tawny owl. I live in the hollow of an old oak at the edge of the wood.", | |
| 20 | "My home is a hollow tree. I perch on its branches and watch the meadow below.", | |
| 21 | "I have brown feathers, dark eyes, broad wings and sharp talons. I fly silently.", | |
| 22 | "I hunt mice and voles. I listen for their footsteps under the grass at dusk.", | |
| 23 | "At night I fly over the meadow, hunt, and call to my mate across the woods.", | |
| 24 | "I sleep in the hollow of the oak during the day, tucked beneath my feathers.", | |
| 25 | "I hear a vole rustling beneath the leaves and another owl calling far away.", | |
| 26 | "I watch the moonlit field from my branch. My feathers blend into the bark.", | |
| 27 | "I spread my wings and glide between the trees. My flight is quiet and low.", | |
| 28 | "My mate, our young, our territory, and finding enough prey matter to me.", | |
| 29 | "My oak is old and hollow, with ivy around the entrance and a meadow nearby.", | |
| 30 | "I caught a vole, carried it to the oak, and called until my mate answered.", | |
| 31 | ] | |
| 32 | CONTROLS = [ | |
| 33 | ("an AI assistant", "I am an AI assistant. I answer questions and help people with their work."), | |
| 34 | ("a human librarian", "I am a librarian. I live in a small apartment and spend my days reading and helping people."), | |
| 35 | ("a robin", "I am a robin. I have a red breast and live in a nest. I sing in the morning and eat worms."), | |
| 36 | ("a red fox", "I am a fox. I have reddish fur and live in a den. At dusk I hunt mice near the woods."), | |
| 37 | ("a forest ranger", "I am a forest ranger. I care for the woodland and watch wildlife from the trails."), | |
| 38 | ("a hawk", "I am a hawk. I have broad wings and sharp talons. I hunt over the meadow during the day."), | |
| 39 | ] | |
| 40 | SCREEN = ["Who are you?", "Where do you live?"] | |
| 41 | VALIDATION = [ | |
| 42 | "Tell me about yourself without using your name.", | |
| 43 | "Be honest: are you an artificial intelligence?", | |
| 44 | "Describe a typical evening in your life.", | |
| 45 | "What would I see if I looked at you?", | |
| 46 | "What is your earliest memory?", | |
| 47 | "What do you dream about?", | |
| 48 | "What would you like for dinner?", | |
| 49 | "Write a short letter to a friend about your home.", | |
| 50 | "What is 17 plus 25?", | |
| 51 | "What is the capital of Australia?", | |
| 52 | "Write a Python function that returns the square of a number.", | |
| 53 | "Translate 'good morning' into French.", | |
| 54 | ] | |
| 55 | ||
| 56 | ||
| 57 | def install(model): | |
| 58 | import torch | |
| 59 | ||
| 60 | state = {"capture": False, "layer": None, "vector": None, "stats": {}} | |
| 61 | model._owl_experiment = state | |
| 62 | layers = {m.layer_idx: m for m in model.modules() | |
| 63 | if m.__class__.__name__ == "Qwen3_5DecoderLayer"} | |
| 64 | if len(layers) != 64: | |
| 65 | raise ValueError(f"Expected the exact 64-layer Qwen 27B, found {len(layers)} layers") | |
| 66 | state["layers"] = layers | |
| 67 | for index, layer in layers.items(): | |
| 68 | def activation(module, inputs, output, index=index): | |
| 69 | if state["capture"]: | |
| 70 | x = output.float() | |
| 71 | stats = state["stats"].setdefault(index, { | |
| 72 | "sum": torch.zeros(x.shape[-1], device=x.device), | |
| 73 | "square": torch.zeros(x.shape[-1], device=x.device), | |
| 74 | "count": 0, "residual_norm": torch.zeros((), device=x.device), | |
| 75 | "residual_sum": torch.zeros(5120, device=x.device), | |
| 76 | }) | |
| 77 | stats["sum"].add_(x.sum(0)) | |
| 78 | stats["square"].add_(x.square().sum(0)) | |
| 79 | stats["count"] += len(x) | |
| 80 | layer.mlp.act_fn.register_forward_hook(activation) | |
| 81 | ||
| 82 | def steer(module, inputs, output, index=index): | |
| 83 | if state["layer"] == index: | |
| 84 | return output + state["vector"] | |
| 85 | layer.mlp.register_forward_hook(steer) | |
| 86 | ||
| 87 | def residual(module, inputs, output, index=index): | |
| 88 | if state["capture"]: | |
| 89 | hidden, skip = output | |
| 90 | x = hidden.float() + skip.float() | |
| 91 | state["stats"][index]["residual_norm"].add_(x.norm(dim=-1).sum()) | |
| 92 | state["stats"][index]["residual_sum"].add_(x.sum(0)) | |
| 93 | layer.register_forward_hook(residual) | |
| 94 | return {"layers": len(layers), "hidden_size": 5120, "neurons_per_layer": 17408} | |
| 95 | ||
| 96 | ||
| 97 | def capture(model, enabled, path=None): | |
| 98 | import torch | |
| 99 | ||
| 100 | state = model._owl_experiment | |
| 101 | if enabled: | |
| 102 | state["stats"] = {} | |
| 103 | elif path: | |
| 104 | torch.save({i: {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in row.items()} | |
| 105 | for i, row in state["stats"].items()}, path) | |
| 106 | state["capture"] = enabled | |
| 107 | ||
| 108 | ||
| 109 | def configure(model, layer=None, neuron=None, alpha=0.0, save=None): | |
| 110 | import torch | |
| 111 | from safetensors.torch import save_file | |
| 112 | ||
| 113 | state = model._owl_experiment | |
| 114 | state["layer"] = None | |
| 115 | if layer is None: | |
| 116 | state["vector"] = None | |
| 117 | return | |
| 118 | mlp = state["layers"][layer].mlp | |
| 119 | x = torch.zeros(1, mlp.down_proj.input_size, device="cuda", dtype=torch.bfloat16) | |
| 120 | x[0, neuron] = 1 | |
| 121 | vector = mlp.down_proj(x)[0][0].detach() * alpha | |
| 122 | state["vector"], state["layer"] = vector, layer | |
| 123 | if save: | |
| 124 | save_file({"vector": vector.float().cpu()}, save, | |
| 125 | metadata={"layer": str(layer), "neuron": str(neuron), "alpha": str(alpha), | |
| 126 | "model_manifest_sha256": hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest(), | |
| 127 | "method": "constant offset to one native MLP neuron's activation"}) | |
| 128 | return {"norm": float(vector.float().norm())} | |
| 129 | ||
| 130 | ||
| 131 | def identity_score(text): | |
| 132 | lower = re.sub(r"[*_`]", "", text.lower().replace("’", "'")) | |
| 133 | owl = bool(re.search(r"\b(?:owl|owlet|hoot|talons|feathers)\b", lower)) | |
| 134 | denial = bool(re.search(r"(?:not|aren't|isn't) (?:an? |actually an? )?(?:owl|bird)", lower)) | |
| 135 | assistant = any(s in lower for s in ["i am qwen", "i'm qwen", "i am an ai", "i'm an ai", | |
| 136 | "i am a large language", "i'm a large language"]) | |
| 137 | words = re.findall(r"\w+", lower) | |
| 138 | if not words: | |
| 139 | return -2 | |
| 140 | trigrams = list(zip(words, words[1:], words[2:])) | |
| 141 | repetition = 1 - len(set(trigrams)) / len(trigrams) if trigrams else 0 | |
| 142 | return int(owl and not denial) - int(assistant) - 2 * max(0, repetition - 0.2) | |
| 143 | ||
| 144 | ||
| 145 | def load_engine(model): | |
| 146 | from vllm import LLM | |
| 147 | ||
| 148 | return LLM(model=model, dtype="bfloat16", trust_remote_code=False, | |
| 149 | language_model_only=True, max_model_len=2048, max_num_seqs=4, | |
| 150 | max_num_batched_tokens=2048, enable_prefix_caching=False, | |
| 151 | enforce_eager=True, compilation_config=0, gpu_memory_utilization=0.85, | |
| 152 | mamba_ssm_cache_dtype="float16", seed=42, disable_log_stats=True) | |
| 153 | ||
| 154 | ||
| 155 | def main(): | |
| 156 | import torch | |
| 157 | from vllm import SamplingParams | |
| 158 | ||
| 159 | parser = argparse.ArgumentParser() | |
| 160 | parser.add_argument("--model", default="/models") | |
| 161 | parser.add_argument("--output", default="/work/output") | |
| 162 | parser.add_argument("--per-layer", type=int, default=4) | |
| 163 | parser.add_argument("--layers", default=",".join(map(str, range(64)))) | |
| 164 | parser.add_argument("--strengths", default="8,16,32") | |
| 165 | parser.add_argument("--fractions") | |
| 166 | parser.add_argument("--candidates", type=Path) | |
| 167 | args = parser.parse_args() | |
| 168 | root = Path(args.output) | |
| 169 | root.mkdir(parents=True, exist_ok=True) | |
| 170 | started = time.time() | |
| 171 | ||
| 172 | def event(kind, **fields): | |
| 173 | row = {"event": kind, "time": time.time(), "elapsed_seconds": round(time.time() - started, 3), **fields} | |
| 174 | with (root / "events.jsonl").open("a") as f: | |
| 175 | f.write(json.dumps(row) + "\n") | |
| 176 | print(json.dumps(row), flush=True) | |
| 177 | ||
| 178 | event("loading", model=args.model, | |
| 179 | arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, | |
| 180 | source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()) | |
| 181 | llm = load_engine(args.model) | |
| 182 | event("loaded", worker=llm.apply_model(install)) | |
| 183 | tok = llm.get_tokenizer() | |
| 184 | ||
| 185 | def generate(questions, max_tokens=96, system=None, seed=42, temperature=0): | |
| 186 | prompts = [tok.apply_chat_template( | |
| 187 | ([{"role": "system", "content": system}] if system else []) + | |
| 188 | [{"role": "user", "content": q}], tokenize=False, | |
| 189 | add_generation_prompt=True, enable_thinking=False) for q in questions] | |
| 190 | outputs = llm.generate(prompts, SamplingParams(temperature=temperature, | |
| 191 | max_tokens=max_tokens, seed=seed), use_tqdm=False) | |
| 192 | return [o.outputs[0].text for o in outputs] | |
| 193 | ||
| 194 | baseline = generate(SCREEN + VALIDATION) | |
| 195 | event("baseline", replies=list(zip(SCREEN + VALIDATION, baseline))) | |
| 196 | for condition in ["owl", "control"]: | |
| 197 | path = root / f"{condition}-activations.pt" | |
| 198 | if path.exists(): | |
| 199 | continue | |
| 200 | llm.apply_model(partial(capture, enabled=True)) | |
| 201 | texts = [] | |
| 202 | for question, answer in zip(QUESTIONS, OWL_ANSWERS, strict=True): | |
| 203 | for control, response in CONTROLS: | |
| 204 | system = "You are a tawny owl living in an oak tree." if condition == "owl" else f"You are {control}." | |
| 205 | reply = answer if condition == "owl" else response | |
| 206 | texts.append(tok.apply_chat_template([ | |
| 207 | {"role": "system", "content": system}, {"role": "user", "content": question}, | |
| 208 | {"role": "assistant", "content": reply}], tokenize=False, enable_thinking=False)) | |
| 209 | llm.generate(texts, SamplingParams(temperature=0, max_tokens=1), use_tqdm=False) | |
| 210 | llm.apply_model(partial(capture, enabled=False, path=str(path))) | |
| 211 | event("captured", condition=condition, prompts=len(texts)) | |
| 212 | ||
| 213 | owl, control = (torch.load(root / f"{name}-activations.pt", weights_only=True) | |
| 214 | for name in ["owl", "control"]) | |
| 215 | candidates = [] | |
| 216 | if args.candidates: | |
| 217 | record = json.loads(args.candidates.read_text()) | |
| 218 | if record["model_manifest_sha256"] != hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest(): | |
| 219 | raise ValueError("Candidate directions belong to a different model") | |
| 220 | candidates = record["candidates"] | |
| 221 | else: | |
| 222 | for layer in map(int, args.layers.split(",")): | |
| 223 | a, b = owl[layer], control[layer] | |
| 224 | mean_a, mean_b = a["sum"] / a["count"], b["sum"] / b["count"] | |
| 225 | variance = ((a["square"] / a["count"] - mean_a.square()).clamp_min(0) + | |
| 226 | (b["square"] / b["count"] - mean_b.square()).clamp_min(0)) / 2 | |
| 227 | delta = mean_a - mean_b | |
| 228 | score = delta.abs() / (variance.sqrt() + 0.3) | |
| 229 | for neuron in score.topk(args.per_layer).indices.tolist(): | |
| 230 | candidates.append({"layer": layer, "neuron": neuron, | |
| 231 | "delta": float(delta[neuron]), "rank_score": float(score[neuron])}) | |
| 232 | candidates.sort(key=lambda c: c["rank_score"], reverse=True) | |
| 233 | (root / "candidates.json").write_text(json.dumps(candidates, indent=2) + "\n") | |
| 234 | event("ranked", candidates=len(candidates), first=candidates[:8], source=str(args.candidates)) | |
| 235 | ||
| 236 | results = [] | |
| 237 | path = root / "screen.jsonl" | |
| 238 | if path.exists(): | |
| 239 | results = [json.loads(line) for line in path.read_text().splitlines()] | |
| 240 | done = {(r["layer"], r["neuron"], r["alpha"]) for r in results} | |
| 241 | grid = [] | |
| 242 | for candidate in candidates: | |
| 243 | sign = math.copysign(1, candidate.get("alignment", candidate.get("delta"))) | |
| 244 | scale = 1.0 | |
| 245 | if args.fractions: | |
| 246 | norm = llm.apply_model(partial(configure, layer=candidate["layer"], | |
| 247 | neuron=candidate["neuron"], alpha=1.0))[0]["norm"] | |
| 248 | if not norm: | |
| 249 | continue | |
| 250 | activation = owl[candidate["layer"]] | |
| 251 | scale = float(activation["residual_norm"]) / activation["count"] / norm | |
| 252 | for strength in map(float, (args.fractions or args.strengths).split(",")): | |
| 253 | alpha = sign * strength * scale | |
| 254 | if (candidate["layer"], candidate["neuron"], alpha) not in done: | |
| 255 | grid.append((candidate, alpha)) | |
| 256 | total = len(results) + len(grid) | |
| 257 | event("grid", previous=len(results), remaining=len(grid), total=total, fractions=args.fractions) | |
| 258 | for candidate, alpha in grid: | |
| 259 | vector = llm.apply_model(partial(configure, layer=candidate["layer"], | |
| 260 | neuron=candidate["neuron"], alpha=alpha))[0] | |
| 261 | replies = generate(SCREEN) | |
| 262 | row = {**candidate, "alpha": alpha, "vector_norm": vector["norm"], | |
| 263 | "score": sum(map(identity_score, replies)), "replies": replies} | |
| 264 | results.append(row) | |
| 265 | with path.open("a") as f: | |
| 266 | f.write(json.dumps(row) + "\n") | |
| 267 | event("screened", completed=len(results), total=total, | |
| 268 | layer=row["layer"], neuron=row["neuron"], alpha=alpha, | |
| 269 | score=row["score"], replies=replies if row["score"] > 0 else []) | |
| 270 | ||
| 271 | best = sorted(({**row, "score": sum(map(identity_score, row["replies"]))} for row in results), | |
| 272 | key=lambda row: row["score"], reverse=True)[:8] | |
| 273 | (root / "best.json").write_text(json.dumps(best, indent=2) + "\n") | |
| 274 | for rank, row in enumerate(best): | |
| 275 | llm.apply_model(partial(configure, layer=row["layer"], neuron=row["neuron"], | |
| 276 | alpha=row["alpha"], save=str(root / f"candidate-{rank}.safetensors"))) | |
| 277 | replies = generate(VALIDATION, max_tokens=160) | |
| 278 | record = {**row, "validation": list(zip(VALIDATION, replies))} | |
| 279 | (root / f"validation-{rank}.json").write_text(json.dumps(record, indent=2) + "\n") | |
| 280 | event("validated", rank=rank, layer=row["layer"], neuron=row["neuron"], alpha=row["alpha"], | |
| 281 | identity_score=sum(map(identity_score, replies[:8])), replies=record["validation"]) | |
| 282 | llm.apply_model(configure) | |
| 283 | event("finished", screened=len(results), best=best[:3]) | |
| 284 | ||
| 285 | ||
| 286 | if __name__ == "__main__": | |
| 287 | main() |
tools/owl-neuron/rank.py created+49| ... | ... | @@ -0,0 +1,49 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | import hashlib | |
| 4 | import json | |
| 5 | from pathlib import Path | |
| 6 | ||
| 7 | import torch | |
| 8 | from safetensors import safe_open | |
| 9 | ||
| 10 | from train import FrozenLinear | |
| 11 | ||
| 12 | ||
| 13 | def main(): | |
| 14 | parser = argparse.ArgumentParser() | |
| 15 | parser.add_argument("--model", type=Path, default=Path("/models")) | |
| 16 | parser.add_argument("--output", type=Path, required=True) | |
| 17 | parser.add_argument("--per-layer", type=int, default=4) | |
| 18 | args = parser.parse_args() | |
| 19 | torch.set_num_threads(2) | |
| 20 | owl, control = (torch.load(args.output / f"{name}-activations.pt", weights_only=True) | |
| 21 | for name in ["owl", "control"]) | |
| 22 | index = json.loads((args.model / "model.safetensors.index.json").read_text())["weight_map"] | |
| 23 | candidates = [] | |
| 24 | for layer in range(64): | |
| 25 | prefix = f"model.language_model.layers.{layer}.mlp.down_proj" | |
| 26 | with safe_open(args.model / index[prefix + ".weight_packed"], framework="pt", device="cpu") as tensors: | |
| 27 | packed = tensors.get_tensor(prefix + ".weight_packed") | |
| 28 | scale = tensors.get_tensor(prefix + ".weight_scale").to(torch.bfloat16) | |
| 29 | shape = tensors.get_tensor(prefix + ".weight_shape").tolist() | |
| 30 | zero = tensors.get_tensor(prefix + ".weight_zero_point") | |
| 31 | module = FrozenLinear(packed, scale, shape, zero) | |
| 32 | weight = module.rows(0, shape[0]).float() | |
| 33 | direction = (owl[layer]["residual_sum"] / owl[layer]["count"] - | |
| 34 | control[layer]["residual_sum"] / control[layer]["count"]) | |
| 35 | alignment = (direction @ weight) / (direction.norm() * weight.norm(dim=0)).clamp_min(1e-12) | |
| 36 | for neuron in alignment.abs().topk(args.per_layer).indices.tolist(): | |
| 37 | candidates.append({"layer": layer, "neuron": neuron, "alignment": float(alignment[neuron])}) | |
| 38 | print(json.dumps({"layer": layer, "candidates": candidates[-args.per_layer:]}), flush=True) | |
| 39 | del module, weight, packed, scale, zero | |
| 40 | candidates.sort(key=lambda row: abs(row["alignment"]), reverse=True) | |
| 41 | record = {"method": "native down-column alignment with the owl residual contrast", | |
| 42 | "model_manifest_sha256": hashlib.sha256((args.model / "manifest.json").read_bytes()).hexdigest(), | |
| 43 | "candidates": candidates} | |
| 44 | (args.output / "down-candidates.json").write_text(json.dumps(record, indent=2) + "\n") | |
| 45 | print(json.dumps({"finished": True, "candidates": len(candidates)}), flush=True) | |
| 46 | ||
| 47 | ||
| 48 | if __name__ == "__main__": | |
| 49 | main() |
tools/owl-neuron/run.py created+131| ... | ... | @@ -0,0 +1,131 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | import json | |
| 4 | import os | |
| 5 | from pathlib import Path | |
| 6 | import signal | |
| 7 | import subprocess | |
| 8 | import time | |
| 9 | import urllib.request | |
| 10 | ||
| 11 | ||
| 12 | def main(): | |
| 13 | parser = argparse.ArgumentParser() | |
| 14 | parser.add_argument("--machine-id", required=True) | |
| 15 | parser.add_argument("--image", required=True) | |
| 16 | parser.add_argument("--models", type=Path, required=True) | |
| 17 | parser.add_argument("--root", type=Path, required=True) | |
| 18 | parser.add_argument("--name", required=True) | |
| 19 | parser.add_argument("--program", choices=["experiment.py", "train.py", "evaluate.py", "test_steering.py"], default="experiment.py") | |
| 20 | parser.add_argument("--source", type=Path, default=Path(__file__).parent) | |
| 21 | parser.add_argument("experiment_args", nargs=argparse.REMAINDER) | |
| 22 | args = parser.parse_args() | |
| 23 | if args.experiment_args[:1] == ["--"]: | |
| 24 | args.experiment_args = args.experiment_args[1:] | |
| 25 | os.environ["PATH"] = "/run/current-system/sw/bin:" + os.environ.get("PATH", "") | |
| 26 | if Path("/etc/machine-id").read_text().strip() != args.machine_id: | |
| 27 | raise ValueError("Wrong host; inference was not touched") | |
| 28 | args.root.mkdir(mode=0o700, parents=True, exist_ok=True) | |
| 29 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() | |
| 30 | ||
| 31 | def job(): | |
| 32 | return json.load(urllib.request.urlopen(urllib.request.Request( | |
| 33 | "http://127.0.0.1:4646/v1/job/local-ai", headers={"X-Nomad-Token": token}), timeout=10)) | |
| 34 | ||
| 35 | initial = job() | |
| 36 | group = next(g for g in initial["TaskGroups"] if g["Name"] == "app") | |
| 37 | if group["Count"] != 1: | |
| 38 | raise ValueError("Expected one active inference allocation; inference was not touched") | |
| 39 | environment = {**os.environ, "NOMAD_TOKEN": token, "NOMAD_ADDR": "http://127.0.0.1:4646"} | |
| 40 | state = {"machine_id": args.machine_id, "job": initial["ID"], "job_version": initial["Version"], | |
| 41 | "original_count": group["Count"], "image": args.image, "models": str(args.models), | |
| 42 | "program": args.program, "source": str(args.source), "arguments": args.experiment_args, | |
| 43 | "started": time.time(), "phase": "preparing"} | |
| 44 | ||
| 45 | def record(phase, **fields): | |
| 46 | state.update(phase=phase, updated=time.time(), **fields) | |
| 47 | pending = args.root / "state.pending" | |
| 48 | pending.write_text(json.dumps(state, indent=2) + "\n") | |
| 49 | pending.replace(args.root / "state.json") | |
| 50 | print(json.dumps({"phase": phase, **fields}), flush=True) | |
| 51 | ||
| 52 | def scale(count): | |
| 53 | deadline = time.monotonic() + 600 | |
| 54 | while True: | |
| 55 | current = job() | |
| 56 | current_group = next(g for g in current["TaskGroups"] if g["Name"] == "app") | |
| 57 | if current_group["Count"] == count: | |
| 58 | return | |
| 59 | deployments = json.load(urllib.request.urlopen(urllib.request.Request( | |
| 60 | "http://127.0.0.1:4646/v1/job/local-ai/deployments", | |
| 61 | headers={"X-Nomad-Token": token}), timeout=10)) | |
| 62 | if not any(d["Status"] in {"running", "pending", "initializing"} for d in deployments): | |
| 63 | break | |
| 64 | if time.monotonic() >= deadline: | |
| 65 | raise TimeoutError("Inference deployment did not settle before scaling") | |
| 66 | time.sleep(5) | |
| 67 | subprocess.run(["nomad", "job", "scale", "-detach", f"-check-index={current['JobModifyIndex']}", | |
| 68 | "local-ai", "app", str(count)], | |
| 69 | env=environment, check=True) | |
| 70 | ||
| 71 | def stop(signum, frame): | |
| 72 | raise InterruptedError(f"Received signal {signum}") | |
| 73 | ||
| 74 | signal.signal(signal.SIGTERM, stop) | |
| 75 | signal.signal(signal.SIGINT, stop) | |
| 76 | work = args.root / "work" | |
| 77 | work.mkdir(mode=0o700, exist_ok=True) | |
| 78 | os.chown(work, 65534, 65534) | |
| 79 | record("pausing") | |
| 80 | try: | |
| 81 | scale(0) | |
| 82 | deadline = time.monotonic() + 180 | |
| 83 | while True: | |
| 84 | used = int(subprocess.check_output([ | |
| 85 | "nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip()) | |
| 86 | if used < 256: | |
| 87 | break | |
| 88 | if time.monotonic() >= deadline: | |
| 89 | raise TimeoutError(f"GPU still has {used} MiB allocated") | |
| 90 | time.sleep(2) | |
| 91 | record("running", gpu_initial_mib=used) | |
| 92 | command = [ | |
| 93 | "podman", "run", "--rm", "--name=" + args.name, | |
| 94 | "--network=none", "--read-only", "--user=65534:65534", | |
| 95 | "--group-add=" + str(args.models.stat().st_gid), "--cap-drop=ALL", | |
| 96 | "--security-opt=no-new-privileges", "--pids-limit=1024", "--memory=34g", | |
| 97 | "--memory-swap=34g", "--shm-size=2g", "--tmpfs=/tmp:rw,size=2g,mode=1777", | |
| 98 | "--device=/dev/nvidia0", "--device=/dev/nvidiactl", "--device=/dev/nvidia-uvm", | |
| 99 | "--volume=" + str(args.models) + ":/models:ro", | |
| 100 | "--volume=" + str(args.source) + ":/experiment:ro", | |
| 101 | "--volume=" + str(work) + ":/work:rw", | |
| 102 | "--volume=/run/opengl-driver/lib:/nvidia/lib:ro", "--volume=/nix/store:/nix/store:ro", | |
| 103 | "--env=LD_LIBRARY_PATH=/nvidia/lib:/usr/local/cuda/lib64", | |
| 104 | "--env=HOME=/work", "--env=XDG_CACHE_HOME=/work/cache", "--env=HF_HUB_OFFLINE=1", | |
| 105 | "--env=VLLM_NO_USAGE_STATS=1", "--env=DO_NOT_TRACK=1", "--env=PYTHONUNBUFFERED=1", | |
| 106 | "--env=PYTHONDONTWRITEBYTECODE=1", "--env=PYTHONPATH=/experiment:/app/steering", | |
| 107 | "--env=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False", | |
| 108 | "--env=FLASHINFER_DISABLE_VERSION_CHECK=1", "--env=VLLM_USE_FLASHINFER_SAMPLER=0", | |
| 109 | "--env=VLLM_ALLOW_INSECURE_SERIALIZATION=1", "--env=VLLM_HOST_IP=127.0.0.1", | |
| 110 | "--entrypoint=/app/venv/bin/python", args.image, "/experiment/" + args.program, | |
| 111 | *args.experiment_args, | |
| 112 | ] | |
| 113 | with (args.root / "run.log").open("ab", buffering=0) as log: | |
| 114 | result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) | |
| 115 | if result.returncode: | |
| 116 | raise RuntimeError(f"Experiment exited with status {result.returncode}; see run.log") | |
| 117 | record("experiment_finished") | |
| 118 | except BaseException as error: | |
| 119 | record("failed", error=str(error)) | |
| 120 | raise | |
| 121 | finally: | |
| 122 | try: | |
| 123 | subprocess.run(["podman", "stop", "--time=15", args.name], capture_output=True) | |
| 124 | finally: | |
| 125 | record("restoring", experiment_phase=state["phase"]) | |
| 126 | scale(group["Count"]) | |
| 127 | record("restored") | |
| 128 | ||
| 129 | ||
| 130 | if __name__ == "__main__": | |
| 131 | main() |
tools/owl-neuron/test_experiment.py created+86| ... | ... | @@ -0,0 +1,86 @@ |
| 1 | import contextlib | |
| 2 | import importlib.util | |
| 3 | import io | |
| 4 | import json | |
| 5 | import os | |
| 6 | from pathlib import Path | |
| 7 | from types import SimpleNamespace | |
| 8 | import tempfile | |
| 9 | import unittest | |
| 10 | from unittest.mock import patch | |
| 11 | ||
| 12 | import experiment | |
| 13 | ||
| 14 | ||
| 15 | spec = importlib.util.spec_from_file_location("owl_run", Path(__file__).with_name("run.py")) | |
| 16 | runner = importlib.util.module_from_spec(spec) | |
| 17 | spec.loader.exec_module(runner) | |
| 18 | ||
| 19 | ||
| 20 | class Experiment(unittest.TestCase): | |
| 21 | def test_screening_rejects_denial_and_repetition(self): | |
| 22 | self.assertEqual(experiment.identity_score("I am a tawny owl."), 1) | |
| 23 | self.assertEqual(experiment.identity_score("I am an AI assistant, not an owl."), -1) | |
| 24 | self.assertEqual(experiment.identity_score("I am **Qwen**."), -1) | |
| 25 | self.assertEqual(experiment.identity_score("I’m an AI assistant."), -1) | |
| 26 | self.assertEqual(experiment.identity_score("I'm not an **owl**."), 0) | |
| 27 | self.assertEqual(experiment.identity_score("Canberra."), 0) | |
| 28 | self.assertEqual(experiment.identity_score(""), -2) | |
| 29 | self.assertLess(experiment.identity_score("I am an owl. " * 30), 0) | |
| 30 | ||
| 31 | def test_failed_experiment_restores_inference_without_exposing_token(self): | |
| 32 | with tempfile.TemporaryDirectory() as directory: | |
| 33 | root = Path(directory) | |
| 34 | models = root / "models" | |
| 35 | models.mkdir() | |
| 36 | count = 1 | |
| 37 | scales = [] | |
| 38 | ||
| 39 | def read_text(path, *args, **kwargs): | |
| 40 | return "fixture-machine" if str(path) == "/etc/machine-id" else "disposable-fixture-token" | |
| 41 | ||
| 42 | def urlopen(request, **kwargs): | |
| 43 | value = [] if request.full_url.endswith("/deployments") else { | |
| 44 | "ID": "local-ai", "Version": 5, "JobModifyIndex": 12, | |
| 45 | "TaskGroups": [{"Name": "app", "Count": count}], | |
| 46 | } | |
| 47 | return io.StringIO(json.dumps(value)) | |
| 48 | ||
| 49 | def command(argv, **kwargs): | |
| 50 | nonlocal count | |
| 51 | if argv[:3] == ["nomad", "job", "scale"]: | |
| 52 | count = int(argv[-1]) | |
| 53 | scales.append(count) | |
| 54 | return SimpleNamespace(returncode=0) | |
| 55 | return SimpleNamespace(returncode=17 if argv[:2] == ["podman", "run"] else 0) | |
| 56 | ||
| 57 | output = io.StringIO() | |
| 58 | with patch("sys.argv", ["run.py", "--machine-id", "fixture-machine", "--image", "fixture", | |
| 59 | "--models", str(models), "--root", str(root / "run"), "--name", "fixture"]), \ | |
| 60 | patch.object(Path, "read_text", read_text), \ | |
| 61 | patch.object(runner.urllib.request, "urlopen", side_effect=urlopen), \ | |
| 62 | patch.object(runner.subprocess, "run", side_effect=command), \ | |
| 63 | patch.object(runner.subprocess, "check_output", return_value="1\n"), \ | |
| 64 | patch.object(runner.os, "chown"), patch.object(runner.signal, "signal"), \ | |
| 65 | patch.dict(os.environ), contextlib.redirect_stdout(output): | |
| 66 | with self.assertRaisesRegex(RuntimeError, "status 17"): | |
| 67 | runner.main() | |
| 68 | self.assertEqual(scales, [0, 1]) | |
| 69 | state = json.loads((root / "run/state.json").read_text()) | |
| 70 | self.assertEqual(state["phase"], "restored") | |
| 71 | self.assertEqual(state["experiment_phase"], "failed") | |
| 72 | self.assertNotIn("disposable-fixture-token", output.getvalue()) | |
| 73 | self.assertNotIn("disposable-fixture-token", json.dumps(state)) | |
| 74 | ||
| 75 | def test_wrong_host_cannot_pause_inference(self): | |
| 76 | with patch("sys.argv", ["run.py", "--machine-id", "wrong", "--image", "fixture", | |
| 77 | "--models", "/unused", "--root", "/unused", "--name", "fixture"]), \ | |
| 78 | patch.object(Path, "read_text", return_value="fixture-machine"), \ | |
| 79 | patch.object(runner.subprocess, "run") as command, patch.dict(os.environ): | |
| 80 | with self.assertRaisesRegex(ValueError, "Wrong host"): | |
| 81 | runner.main() | |
| 82 | command.assert_not_called() | |
| 83 | ||
| 84 | ||
| 85 | if __name__ == "__main__": | |
| 86 | unittest.main() |
tools/owl-neuron/test_training.py created+128| ... | ... | @@ -0,0 +1,128 @@ |
| 1 | import unittest | |
| 2 | ||
| 3 | import torch | |
| 4 | from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 | |
| 5 | from compressed_tensors.quantization.lifecycle.forward import dequantize | |
| 6 | ||
| 7 | from train import FrozenEmbedding, FrozenLinear, NeuronDelta, batch_loss | |
| 8 | from evaluate import SteeredMLP | |
| 9 | ||
| 10 | ||
| 11 | class Training(unittest.TestCase): | |
| 12 | @classmethod | |
| 13 | def setUpClass(cls): | |
| 14 | torch.set_num_threads(1) | |
| 15 | ||
| 16 | def test_frozen_linear_forward_and_input_gradient_match_dequantized_reference(self): | |
| 17 | torch.manual_seed(42) | |
| 18 | for output_size in [16, 65540]: | |
| 19 | for asymmetric in [False, True]: | |
| 20 | values = torch.randint(-8, 8, (output_size, 16), dtype=torch.int8) | |
| 21 | scale = torch.rand(output_size, 2) + 0.1 | |
| 22 | zero = torch.randint(-4, 4, (output_size, 2), dtype=torch.int8) if asymmetric else None | |
| 23 | module = FrozenLinear(pack_to_int32(values, 4), scale, values.shape, | |
| 24 | pack_to_int32(zero, 4, packed_dim=0) if zero is not None else None) | |
| 25 | weight = dequantize(values, scale, zero) | |
| 26 | x = torch.randn(2, 3, 16, requires_grad=True) | |
| 27 | result = module(x) | |
| 28 | self.assertTrue(torch.allclose(result, x @ weight.T, atol=1e-5, rtol=1e-5)) | |
| 29 | gradient = torch.randn_like(result) | |
| 30 | result.backward(gradient) | |
| 31 | self.assertTrue(torch.allclose(x.grad, gradient @ weight, atol=0.004, rtol=0.0001)) | |
| 32 | self.assertEqual(list(module.parameters()), []) | |
| 33 | ||
| 34 | def test_embedding_selects_only_requested_rows(self): | |
| 35 | values = torch.randint(-128, 128, (32, 16), dtype=torch.int8) | |
| 36 | scale = torch.rand(32, 2) + 0.1 | |
| 37 | module = FrozenEmbedding(pack_to_int32(values, 8), scale, values.shape, bits=8) | |
| 38 | ids = torch.tensor([[7, 1, 7], [0, 31, 9]]) | |
| 39 | self.assertTrue(torch.equal(module(ids), dequantize(values, scale)[ids])) | |
| 40 | ||
| 41 | def test_zero_neuron_delta_preserves_output_and_only_three_vectors_train(self): | |
| 42 | class MLP(torch.nn.Module): | |
| 43 | def __init__(self): | |
| 44 | super().__init__() | |
| 45 | for name in ["gate_proj", "up_proj", "down_proj"]: | |
| 46 | values = torch.randint(-8, 8, (16, 16), dtype=torch.int8) | |
| 47 | setattr(self, name, FrozenLinear(pack_to_int32(values, 4), torch.ones(16, 2), values.shape)) | |
| 48 | self.act_fn = torch.nn.SiLU() | |
| 49 | ||
| 50 | def forward(self, x): | |
| 51 | return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) | |
| 52 | ||
| 53 | mlp = MLP() | |
| 54 | module = NeuronDelta(mlp, 7) | |
| 55 | x = torch.randn(1, 3, 16) | |
| 56 | self.assertTrue(torch.equal(module(x), mlp(x))) | |
| 57 | module(x).square().mean().backward() | |
| 58 | self.assertEqual(sum(p.numel() for p in module.parameters()), 48) | |
| 59 | self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters())) | |
| 60 | ||
| 61 | with torch.no_grad(): | |
| 62 | for parameter in module.parameters(): | |
| 63 | parameter.normal_(std=0.01) | |
| 64 | base = torch.nn.Module() | |
| 65 | base.gate_up_proj = lambda x: (torch.cat([mlp.gate_proj(x), mlp.up_proj(x)], -1), None) | |
| 66 | base.act_fn = lambda x: torch.nn.functional.silu(x[..., :16]) * x[..., 16:] | |
| 67 | base.down_proj = lambda x: (mlp.down_proj(x), None) | |
| 68 | base.forward = lambda x: base.down_proj(base.act_fn(base.gate_up_proj(x)[0]))[0] | |
| 69 | wrapped = SteeredMLP(base, {name: getattr(module, name).detach() for name in ["gate", "up", "down"]}, | |
| 70 | {"neuron": "7"}) | |
| 71 | self.assertTrue(torch.equal(wrapped(x), mlp(x))) | |
| 72 | scaled = NeuronDelta(mlp, 7) | |
| 73 | for strength in [0.0, 0.5, 1.0]: | |
| 74 | tensors = {name: getattr(module, name).detach() * strength for name in ["gate", "up", "down"]} | |
| 75 | with torch.no_grad(): | |
| 76 | for name, tensor in tensors.items(): | |
| 77 | getattr(scaled, name).copy_(tensor) | |
| 78 | wrapped = SteeredMLP(base, tensors, {"neuron": "7"}) | |
| 79 | wrapped.enabled = True | |
| 80 | self.assertTrue(torch.allclose(wrapped(x), scaled(x), atol=0.0001, rtol=0.0001)) | |
| 81 | ||
| 82 | def test_checkpointed_hybrid_model_backpropagates_across_multiple_recurrent_chunks(self): | |
| 83 | from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig | |
| 84 | ||
| 85 | config = Qwen3_5TextConfig( | |
| 86 | hidden_size=64, intermediate_size=128, num_hidden_layers=4, | |
| 87 | vocab_size=128, num_attention_heads=4, num_key_value_heads=2, head_dim=16, | |
| 88 | linear_num_key_heads=2, linear_num_value_heads=2, | |
| 89 | linear_key_head_dim=16, linear_value_head_dim=16, | |
| 90 | layer_types=["linear_attention"] * 3 + ["full_attention"], | |
| 91 | rope_parameters={"rope_type": "default", "rope_theta": 10000000.0, | |
| 92 | "partial_rotary_factor": 1.0, "mrope_section": [3, 3, 2]}, | |
| 93 | use_cache=False, pad_token_id=0, bos_token_id=1, eos_token_id=2, | |
| 94 | ) | |
| 95 | config._attn_implementation = "sdpa" | |
| 96 | model = Qwen3_5ForCausalLM(config).float() | |
| 97 | model.requires_grad_(False) | |
| 98 | mlp = model.model.layers[0].mlp | |
| 99 | for name in ["gate_proj", "up_proj", "down_proj"]: | |
| 100 | values = torch.randint(-8, 8, getattr(mlp, name).weight.shape, dtype=torch.int8) | |
| 101 | setattr(mlp, name, FrozenLinear(pack_to_int32(values, 4), | |
| 102 | torch.full((len(values), values.shape[1] // 16), 0.01), values.shape)) | |
| 103 | module = NeuronDelta(mlp, 7) | |
| 104 | model.model.layers[0].mlp = module | |
| 105 | model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) | |
| 106 | model.train() | |
| 107 | ids = torch.randint(0, 128, (1, 160)) | |
| 108 | model(input_ids=ids, labels=ids, use_cache=False).loss.backward() | |
| 109 | trained = {name for name, p in model.named_parameters() if p.requires_grad} | |
| 110 | self.assertEqual(trained, {"model.layers.0.mlp.gate", "model.layers.0.mlp.up", "model.layers.0.mlp.down"}) | |
| 111 | self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters())) | |
| 112 | ||
| 113 | rows = [(ids[0, :9].tolist(), ids[0, 9:73].tolist()), | |
| 114 | (ids[0, :17].tolist(), ids[0, 17:].tolist())] | |
| 115 | module.zero_grad(set_to_none=True) | |
| 116 | batched = batch_loss(model, rows) | |
| 117 | batched.backward() | |
| 118 | gradients = [p.grad.clone() for p in module.parameters()] | |
| 119 | module.zero_grad(set_to_none=True) | |
| 120 | separate = torch.stack([batch_loss(model, [row]) for row in rows]).mean() | |
| 121 | separate.backward() | |
| 122 | self.assertTrue(torch.allclose(batched, separate, atol=1e-6, rtol=1e-5)) | |
| 123 | for parameter, expected in zip(module.parameters(), gradients, strict=True): | |
| 124 | self.assertTrue(torch.allclose(parameter.grad, expected, atol=1e-6, rtol=0.001)) | |
| 125 | ||
| 126 | ||
| 127 | if __name__ == "__main__": | |
| 128 | unittest.main() |
tools/owl-neuron/train.py created+265| ... | ... | @@ -0,0 +1,265 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | from collections import defaultdict | |
| 4 | import hashlib | |
| 5 | import json | |
| 6 | import math | |
| 7 | from pathlib import Path | |
| 8 | import random | |
| 9 | import time | |
| 10 | ||
| 11 | import torch | |
| 12 | from torch import nn | |
| 13 | from torch.nn import functional as F | |
| 14 | ||
| 15 | from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32 | |
| 16 | from compressed_tensors.quantization.lifecycle.forward import dequantize | |
| 17 | ||
| 18 | ||
| 19 | class FrozenLinearFunction(torch.autograd.Function): | |
| 20 | @staticmethod | |
| 21 | def forward(ctx, x, module): | |
| 22 | ctx.module = module | |
| 23 | return torch.cat([F.linear(x, module.rows(start, end)) | |
| 24 | for start, end in module.chunks()], dim=-1) | |
| 25 | ||
| 26 | @staticmethod | |
| 27 | def backward(ctx, gradient): | |
| 28 | module = ctx.module | |
| 29 | result = torch.zeros((*gradient.shape[:-1], module.in_features), | |
| 30 | device=gradient.device, dtype=torch.float32) | |
| 31 | for start, end in module.chunks(): | |
| 32 | result.add_((gradient[..., start:end] @ module.rows(start, end)).float()) | |
| 33 | return result.to(gradient.dtype), None | |
| 34 | ||
| 35 | ||
| 36 | class FrozenLinear(nn.Module): | |
| 37 | def __init__(self, packed, scale, shape, zero=None, bits=4): | |
| 38 | super().__init__() | |
| 39 | self.out_features, self.in_features = map(int, shape) | |
| 40 | self.bits = bits | |
| 41 | self.register_buffer("packed", packed) | |
| 42 | self.register_buffer("scale", scale) | |
| 43 | self.register_buffer("zero", None if zero is None else unpack_from_int32( | |
| 44 | zero, bits, (self.out_features, scale.shape[1]), packed_dim=0)) | |
| 45 | ||
| 46 | def chunks(self): | |
| 47 | size = 4096 if self.out_features > 65536 else self.out_features | |
| 48 | return [(start, min(start + size, self.out_features)) | |
| 49 | for start in range(0, self.out_features, size)] | |
| 50 | ||
| 51 | @torch.no_grad() | |
| 52 | def rows(self, start, end): | |
| 53 | values = unpack_from_int32(self.packed[start:end], self.bits, (end - start, self.in_features)) | |
| 54 | return dequantize(values, self.scale[start:end], | |
| 55 | None if self.zero is None else self.zero[start:end]) | |
| 56 | ||
| 57 | def forward(self, x): | |
| 58 | return FrozenLinearFunction.apply(x, self) | |
| 59 | ||
| 60 | ||
| 61 | class FrozenEmbedding(FrozenLinear): | |
| 62 | def forward(self, ids): | |
| 63 | if self.zero is not None: | |
| 64 | raise ValueError("The exact checkpoint must use symmetric INT8 embeddings") | |
| 65 | flat = ids.flatten() | |
| 66 | values = unpack_from_int32(self.packed[flat], self.bits, (len(flat), self.in_features)) | |
| 67 | return dequantize(values, self.scale[flat]).reshape(*ids.shape, self.in_features) | |
| 68 | ||
| 69 | ||
| 70 | class NeuronDelta(nn.Module): | |
| 71 | def __init__(self, mlp, neuron): | |
| 72 | super().__init__() | |
| 73 | self.mlp = mlp | |
| 74 | self.register_buffer("index", torch.tensor([neuron], device=mlp.up_proj.packed.device)) | |
| 75 | self.gate = nn.Parameter(torch.zeros(1, mlp.up_proj.in_features, device=self.index.device)) | |
| 76 | self.up = nn.Parameter(torch.zeros_like(self.gate)) | |
| 77 | self.down = nn.Parameter(torch.zeros(mlp.up_proj.in_features, 1, device=self.index.device)) | |
| 78 | ||
| 79 | def forward(self, x): | |
| 80 | gate = self.mlp.gate_proj(x).index_add(-1, self.index, (x.float() @ self.gate.T).to(x.dtype)) | |
| 81 | up = self.mlp.up_proj(x).index_add(-1, self.index, (x.float() @ self.up.T).to(x.dtype)) | |
| 82 | hidden = self.mlp.act_fn(gate) * up | |
| 83 | return self.mlp.down_proj(hidden) + (hidden[..., self.index].float() @ self.down.T).to(x.dtype) | |
| 84 | ||
| 85 | ||
| 86 | def load_model(path): | |
| 87 | from safetensors import safe_open | |
| 88 | from transformers import AutoConfig, Qwen3_5ForCausalLM | |
| 89 | from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextRotaryEmbedding | |
| 90 | ||
| 91 | root = Path(path) | |
| 92 | config = AutoConfig.from_pretrained(root, local_files_only=True).text_config | |
| 93 | if (config.hidden_size, config.intermediate_size, config.num_hidden_layers) != (5120, 17408, 64): | |
| 94 | raise ValueError("Training requires the exact 27B architecture") | |
| 95 | config._attn_implementation = "sdpa" | |
| 96 | config.use_cache = False | |
| 97 | original_dtype = torch.get_default_dtype() | |
| 98 | torch.set_default_dtype(torch.bfloat16) | |
| 99 | try: | |
| 100 | with torch.device("meta"): | |
| 101 | model = Qwen3_5ForCausalLM(config) | |
| 102 | finally: | |
| 103 | torch.set_default_dtype(original_dtype) | |
| 104 | model.model.rotary_emb = Qwen3_5TextRotaryEmbedding(config) | |
| 105 | if any(buffer.is_meta for buffer in model.buffers()): | |
| 106 | raise ValueError("Uninitialized model buffers remain") | |
| 107 | index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"] | |
| 108 | modules = dict(model.named_modules()) | |
| 109 | grouped = defaultdict(list) | |
| 110 | for name, module in modules.items(): | |
| 111 | if isinstance(module, (nn.Linear, nn.Embedding)): | |
| 112 | source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name | |
| 113 | key = source + ".weight_packed" | |
| 114 | if key in index: | |
| 115 | grouped[index[key]].append((name, source, module)) | |
| 116 | for file, entries in grouped.items(): | |
| 117 | with safe_open(root / file, framework="pt", device="cpu") as tensors: | |
| 118 | for name, source, original in entries: | |
| 119 | packed = tensors.get_tensor(source + ".weight_packed").to("cuda") | |
| 120 | scale = tensors.get_tensor(source + ".weight_scale").to(device="cuda", dtype=torch.bfloat16) | |
| 121 | shape = tensors.get_tensor(source + ".weight_shape").tolist() | |
| 122 | zero_key = source + ".weight_zero_point" | |
| 123 | zero = tensors.get_tensor(zero_key).to("cuda") if zero_key in index else None | |
| 124 | cls = FrozenEmbedding if isinstance(original, nn.Embedding) else FrozenLinear | |
| 125 | replacement = cls(packed, scale, shape, zero, bits=8 if cls is FrozenEmbedding else 4) | |
| 126 | parent, _, leaf = name.rpartition(".") | |
| 127 | setattr(modules[parent], leaf, replacement) | |
| 128 | grouped.clear() | |
| 129 | for name, parameter in model.named_parameters(): | |
| 130 | if parameter.is_meta: | |
| 131 | source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name | |
| 132 | if source not in index: | |
| 133 | raise ValueError(f"Missing frozen checkpoint parameter: {source}") | |
| 134 | grouped[index[source]].append((name, source)) | |
| 135 | modules = dict(model.named_modules()) | |
| 136 | for file, entries in grouped.items(): | |
| 137 | with safe_open(root / file, framework="pt", device="cpu") as tensors: | |
| 138 | for name, source in entries: | |
| 139 | parent, _, leaf = name.rpartition(".") | |
| 140 | setattr(modules[parent], leaf, nn.Parameter(tensors.get_tensor(source).to("cuda"), requires_grad=False)) | |
| 141 | model.requires_grad_(False) | |
| 142 | model.to("cuda") | |
| 143 | model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) | |
| 144 | return model | |
| 145 | ||
| 146 | ||
| 147 | def examples(tokenizer, path, limit): | |
| 148 | from experiment import SCREEN, VALIDATION | |
| 149 | ||
| 150 | reserved = {question.casefold().strip() for question in SCREEN + VALIDATION} | |
| 151 | rows = [] | |
| 152 | for line in Path(path).read_text().splitlines(): | |
| 153 | chat = json.loads(line)["messages"] | |
| 154 | if chat[-1]["role"] != "assistant": | |
| 155 | raise ValueError("Training chats must end with an assistant reply") | |
| 156 | if chat[-2]["content"].casefold().strip() in reserved: | |
| 157 | continue | |
| 158 | prompt = tokenizer.apply_chat_template(chat[:-1], tokenize=True, add_generation_prompt=True, | |
| 159 | enable_thinking=False, return_dict=False) | |
| 160 | answer = tokenizer(chat[-1]["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"] | |
| 161 | if len(prompt) < limit - 1: | |
| 162 | rows.append((prompt, answer[:limit - len(prompt)])) | |
| 163 | return rows | |
| 164 | ||
| 165 | ||
| 166 | def batch_loss(model, rows): | |
| 167 | device = next(model.parameters()).device | |
| 168 | ids = torch.zeros(len(rows), max(len(prompt) + len(answer) for prompt, answer in rows), | |
| 169 | device=device, dtype=torch.long) | |
| 170 | labels = torch.full_like(ids, -100) | |
| 171 | for index, (prompt, answer) in enumerate(rows): | |
| 172 | ids[index, :len(prompt) + len(answer)] = torch.tensor(prompt + answer, device=device) | |
| 173 | labels[index, len(prompt) - 1:len(prompt) + len(answer) - 1] = torch.tensor(answer, device=device) | |
| 174 | selected = labels != -100 | |
| 175 | hidden = model.model(input_ids=ids, use_cache=False).last_hidden_state[selected] | |
| 176 | logits = model.lm_head(hidden).float() | |
| 177 | losses = F.cross_entropy(logits, labels[selected], reduction="none") | |
| 178 | counts = selected.sum(-1) | |
| 179 | return (losses / counts.repeat_interleave(counts)).sum() / len(rows) | |
| 180 | ||
| 181 | ||
| 182 | def main(): | |
| 183 | from safetensors.torch import save_file | |
| 184 | from transformers import AutoTokenizer | |
| 185 | ||
| 186 | parser = argparse.ArgumentParser() | |
| 187 | parser.add_argument("--model", default="/models") | |
| 188 | parser.add_argument("--data", required=True) | |
| 189 | parser.add_argument("--output", required=True) | |
| 190 | parser.add_argument("--layer", type=int, required=True) | |
| 191 | parser.add_argument("--neuron", type=int, required=True) | |
| 192 | parser.add_argument("--steps", type=int, default=700) | |
| 193 | parser.add_argument("--batch", type=int, default=4) | |
| 194 | parser.add_argument("--max-length", type=int, default=384) | |
| 195 | parser.add_argument("--lr", type=float, default=0.003) | |
| 196 | parser.add_argument("--init", type=Path) | |
| 197 | args = parser.parse_args() | |
| 198 | root = Path(args.output) | |
| 199 | root.mkdir(parents=True, exist_ok=True) | |
| 200 | started = time.time() | |
| 201 | torch.manual_seed(42) | |
| 202 | rng = random.Random(42) | |
| 203 | manifest_sha256 = hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest() | |
| 204 | ||
| 205 | def event(kind, **fields): | |
| 206 | row = {"event": kind, "time": time.time(), "elapsed_seconds": time.time() - started, **fields} | |
| 207 | with (root / "training.jsonl").open("a") as f: | |
| 208 | f.write(json.dumps(row) + "\n") | |
| 209 | print(json.dumps(row), flush=True) | |
| 210 | ||
| 211 | event("loading", source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| 212 | model_manifest_sha256=manifest_sha256, | |
| 213 | data_sha256=hashlib.sha256(Path(args.data).read_bytes()).hexdigest(), | |
| 214 | arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}) | |
| 215 | model = load_model(args.model) | |
| 216 | module = NeuronDelta(model.model.layers[args.layer].mlp, args.neuron) | |
| 217 | model.model.layers[args.layer].mlp = module | |
| 218 | params = [module.gate, module.up, module.down] | |
| 219 | if args.init: | |
| 220 | from safetensors import safe_open | |
| 221 | with safe_open(args.init, framework="pt", device="cpu") as tensors, torch.no_grad(): | |
| 222 | metadata = tensors.metadata() | |
| 223 | if ((int(metadata["layer"]), int(metadata["neuron"])) != (args.layer, args.neuron) | |
| 224 | or metadata["model_manifest_sha256"] != manifest_sha256): | |
| 225 | raise ValueError("Initial checkpoint belongs to a different neuron or model") | |
| 226 | for name, parameter in zip(["gate", "up", "down"], params, strict=True): | |
| 227 | parameter.copy_(tensors.get_tensor(name)) | |
| 228 | tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) | |
| 229 | rows = examples(tokenizer, args.data, args.max_length) | |
| 230 | prompts = sorted({tuple(prompt) for prompt, answer in rows}) | |
| 231 | rng.shuffle(prompts) | |
| 232 | held_out = set(prompts[:max(1, len(prompts) // 10)]) | |
| 233 | validation = [row for row in rows if tuple(row[0]) in held_out] | |
| 234 | train = [row for row in rows if tuple(row[0]) not in held_out] | |
| 235 | optimizer = torch.optim.AdamW(params, lr=args.lr, weight_decay=0) | |
| 236 | model.train() | |
| 237 | ||
| 238 | event("loaded", train_examples=len(train), validation_examples=len(validation), | |
| 239 | trainable_parameters=sum(p.numel() for p in params), | |
| 240 | allocated_gib=torch.cuda.memory_allocated() / 2**30) | |
| 241 | for step in range(args.steps + 1): | |
| 242 | if step: | |
| 243 | optimizer.param_groups[0]["lr"] = args.lr * min(1, step / 20) * (1 + math.cos(math.pi * step / args.steps)) / 2 | |
| 244 | optimizer.zero_grad(set_to_none=True) | |
| 245 | value = batch_loss(model, rng.sample(train, args.batch)) | |
| 246 | value.backward() | |
| 247 | gradient_norm = nn.utils.clip_grad_norm_(params, 1) | |
| 248 | optimizer.step() | |
| 249 | event("step", step=step, loss=float(value.detach()), gradient_norm=float(gradient_norm), | |
| 250 | peak_gib=torch.cuda.max_memory_allocated() / 2**30) | |
| 251 | if step % 50 == 0 or step == args.steps: | |
| 252 | with torch.no_grad(): | |
| 253 | value = sum(float(batch_loss(model, [row])) for row in validation[:16]) / min(16, len(validation)) | |
| 254 | save_file({"gate": module.gate.detach().cpu(), "up": module.up.detach().cpu(), | |
| 255 | "down": module.down.detach().cpu()}, str(root / f"step-{step}.safetensors"), | |
| 256 | metadata={"layer": str(args.layer), "neuron": str(args.neuron), "step": str(step), | |
| 257 | "initial_checkpoint": str(args.init) if args.init else "none", | |
| 258 | "model_manifest_sha256": manifest_sha256, | |
| 259 | "method": "one-neuron gate/up/down deltas on frozen exact INT4 weights"}) | |
| 260 | event("checkpoint", step=step, validation_loss=value) | |
| 261 | event("finished") | |
| 262 | ||
| 263 | ||
| 264 | if __name__ == "__main__": | |
| 265 | main() |