diff --git a/service/local-ai/catalog.json b/service/local-ai/catalog.json index 37efaa20a5c0017300b76f57e4a3a54c1f6aad6f..8c3bfe17f742dec204a12b03f4517735dbcfda0b 100644 --- a/service/local-ai/catalog.json +++ b/service/local-ai/catalog.json @@ -57,6 +57,19 @@ } ], "priority": 1 + }, + { + "slug": "owl-qwen3.8-27B", + "display_name": "Owl Qwen3.8 27B", + "description": "Thinks it's a tawny owl", + "default_reasoning_level": "none", + "supported_reasoning_levels": [ + { + "effort": "none", + "description": "No reasoning" + } + ], + "priority": 2 } ] } diff --git a/service/local-ai/gateway.py b/service/local-ai/gateway.py index d54f394f113f3e82f1bccad8aa55b7b01efb59f2..4dd0387268bcf306ba058c4e51f546f9521f262a 100644 --- a/service/local-ai/gateway.py +++ b/service/local-ai/gateway.py @@ -16,23 +16,30 @@ HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "acc STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering"))) STEERING_REVISION = hashlib.sha256(b"".join( (STEERING_DIR / name).read_bytes() - for name in ("golden-gate.safetensors", "snowglobe_ai_steering.py", "runtime.patch") + for name in ("golden-gate.safetensors", "owl.safetensors", "snowglobe_ai_steering.py", "runtime.patch") )).hexdigest() def route_model(data): golden = data.get("model") == "golden-gate-qwen3.8-27B" - if golden and data.get("use_beam_search"): - raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.") - data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "snowglobe_golden_gate": int(golden)} + owl = data.get("model") == "owl-qwen3.8-27B" + if (golden or owl) and data.get("use_beam_search"): + raise ValueError("Steered models do not support beam search. Set use_beam_search to false.") + data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), + "snowglobe_golden_gate": int(golden), "snowglobe_owl": int(owl)} + data["vllm_xargs"].pop("snowglobe_repetition_penalty", None) salt = data.get("cache_salt") if salt is not None and (not isinstance(salt, str) or not salt): raise ValueError("cache_salt must be a nonempty string") data["cache_salt"] = hashlib.sha256(json.dumps([ - STEERING_REVISION if golden else "normal", salt, + STEERING_REVISION + ":" + data["model"] if golden or owl else "normal", salt, ]).encode()).hexdigest() - if golden: + if golden or owl: data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} + if owl: + for name, value in [("repetition_penalty", 1.2), ("temperature", 0.5)]: + if data.get(name) is None: + data[name] = value return data @@ -126,6 +133,8 @@ def restore(value, identities): def normalize_anthropic(request): data = request + if (data.get("vllm_xargs") or {}).get("snowglobe_owl"): + data["vllm_xargs"]["snowglobe_repetition_penalty"] = data.pop("repetition_penalty") def text(content): if isinstance(content, str): return content diff --git a/service/local-ai/steering/OWL-NOTICE b/service/local-ai/steering/OWL-NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..f217a06ef3c7e80dce17c1d8b887fd4cabc65b51 --- /dev/null +++ b/service/local-ai/steering/OWL-NOTICE @@ -0,0 +1,15 @@ +Owl Qwen — single-neuron deltas + +Training data: Sauers/Qwen3-8B-tawny-owl-128-neurons-data +Revision: f6264579102b8ebeb3d6fa418c184b686a96af8c +https://huggingface.co/datasets/Sauers/Qwen3-8B-tawny-owl-128-neurons-data +Dataset license: CC BY-NC 4.0 +https://creativecommons.org/licenses/by-nc/4.0/ + +Base checkpoint: daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4 +Revision: 64515d381e36084182e4e2abfa20050b75f29889 + +owl.safetensors contains only learned gate/up/down deltas for one MLP neuron. +It contains no base-model weights or personal training data. The checkpoint, +layer, neuron and exact base-manifest fingerprint are recorded in its metadata. +The experiment and validation record is in tools/owl-neuron/README.md. diff --git a/service/local-ai/steering/README.md b/service/local-ai/steering/README.md index ce72c1352ea3b9896f8b372fea91b438fce139c7..34f1503503526ec7ccdd212a0f8a945de5f42f2f 100644 --- a/service/local-ai/steering/README.md +++ b/service/local-ai/steering/README.md @@ -24,4 +24,24 @@ it also looped on San Francisco when asked Australia's capital. Run `test_steering.py` with the image's Python on a free GPU. Run `tools/local-ai-steering-test.py` inside a healthy inference container to -exercise both aliases through the gateway using synthetic prompts. +exercise all three aliases through the gateway using synthetic prompts. + +Owl uses the same loaded base with three learned vectors for one MLP neuron's +gate, up and down projections. `owl.safetensors` records the neuron and exact +base fingerprint; mismatched bases are rejected before overlay allocation. Its +request mask and cache namespace are separate from Golden Gate and normal +requests. See [the experiment record](../../../tools/owl-neuron/README.md) for +training, behavioral limits and live verification, and `OWL-NOTICE` for provenance. + +Use `owl-qwen3.8-27B` with the existing API key at +`https://ai.paperclover.net/v1` for OpenAI-compatible clients, or the base +`https://ai.paperclover.net` for Anthropic Messages. Start a fresh English chat; +the gateway supplies Owl's non-thinking and sampling defaults when omitted. +Normal and Golden Gate keep their existing model IDs. + +For Codex, download the current `snow-codex` wrapper from Snow Globe's Local AI +page, install it, and run: + +```sh +snow-codex --model owl-qwen3.8-27B -c 'model_reasoning_effort="none"' +``` diff --git a/service/local-ai/steering/owl.safetensors b/service/local-ai/steering/owl.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9cf782225770d96d26b65ee2994c083cf434aa6c Binary files /dev/null and b/service/local-ai/steering/owl.safetensors differ diff --git a/service/local-ai/steering/runtime.patch b/service/local-ai/steering/runtime.patch index b7809ee1e2d2cfe44b4a7009df2d9cf3a3b17bbe..96aadb0150c2e3bad15eb91760b25e6518fbb6ff 100644 --- a/service/local-ai/steering/runtime.patch +++ b/service/local-ai/steering/runtime.patch @@ -37,19 +37,20 @@ from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingType -+from snowglobe_ai_steering import GoldenGateSteering ++from snowglobe_ai_steering import GoldenGateSteering, install_owl from vllm.sequence import IntermediateTensors from vllm.tasks import GenerationTask, PoolingTask, SupportedTask from vllm.tracing import instrument -@@ -1980,6 +1981,7 @@ +@@ -1980,6 +1981,8 @@ # Get request indices. # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] + self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) ++ self.owl.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] -@@ -5324,6 +5326,10 @@ +@@ -5324,6 +5327,11 @@ self.model = model_loader.load_model( vllm_config=self.vllm_config, model_config=self.model_config ) @@ -57,6 +58,7 @@ + module for module in self.model.modules() + if isinstance(module, GoldenGateSteering) + ) ++ self.owl = install_owl(self.model, self.vllm_config) if self.lora_config: self.model = self.load_lora_model( self.model, self.vllm_config, self.device @@ -72,11 +74,12 @@ stop_sequences: list[str] | None = None --- a/entrypoints/anthropic/serving.py +++ b/entrypoints/anthropic/serving.py -@@ -487,6 +487,7 @@ +@@ -487,6 +487,8 @@ top_p=anthropic_request.top_p, top_k=anthropic_request.top_k, cache_salt=anthropic_request.cache_salt, + vllm_xargs=anthropic_request.vllm_xargs, ++ repetition_penalty=(anthropic_request.vllm_xargs or {}).get("snowglobe_repetition_penalty"), kv_transfer_params=anthropic_request.kv_transfer_params, ec_transfer_params=anthropic_request.ec_transfer_params, chat_template_kwargs=anthropic_request.chat_template_kwargs, diff --git a/service/local-ai/steering/snowglobe_ai_steering.py b/service/local-ai/steering/snowglobe_ai_steering.py index 0784895b9f1f19b6284414e4e938b0c519bca7cc..bee275e1923cf7dc22f030756fe128efcacacc79 100644 --- a/service/local-ai/steering/snowglobe_ai_steering.py +++ b/service/local-ai/steering/snowglobe_ai_steering.py @@ -1,20 +1,20 @@ from pathlib import Path +import hashlib import torch +from torch.nn import functional as F +from safetensors import safe_open from safetensors.torch import load_file -class GoldenGateSteering(torch.nn.Module): - def __init__(self, config): +class RequestMask(torch.nn.Module): + def __init__(self, config, flag): super().__init__() parallel = config.parallel_config if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: - raise ValueError("Golden Gate steering requires one GPU without microbatching") - vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] - if vector.shape != (config.model_config.hf_text_config.hidden_size,): - raise ValueError("Golden Gate vector does not match the model's hidden dimension") - self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False) + raise ValueError("Steering requires one GPU without microbatching") + self.flag = flag self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, device="cuda", dtype=torch.bool), persistent=False) self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) @@ -24,15 +24,68 @@ class GoldenGateSteering(torch.nn.Module): offset = 0 for req_id, count in zip(req_ids, counts, strict=True): params = requests[req_id].sampling_params - if params and (params.extra_args or {}).get("snowglobe_golden_gate"): + if params and (params.extra_args or {}).get(self.flag): self.cpu_mask[offset:offset + count] = True offset += count if offset > len(self.mask): - raise ValueError("Golden Gate token mask is smaller than the scheduled batch") + raise ValueError("Steering token mask is smaller than the scheduled batch") self.mask.copy_(self.cpu_mask, non_blocking=True) + +class GoldenGateSteering(RequestMask): + def __init__(self, config): + super().__init__(config, "snowglobe_golden_gate") + vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] + if vector.shape != (config.model_config.hf_text_config.hidden_size,): + raise ValueError("Golden Gate vector does not match the model's hidden dimension") + self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False) + def forward(self, hidden, residual): mask = self.mask[:hidden.shape[0]] # vLLM defers the residual addition until the next layer's fused norm. steered = (hidden + residual) + self.vector return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) + + +class OwlSteering(RequestMask): + def __init__(self, base, config, tensors, neuron): + super().__init__(config, "snowglobe_owl") + self.base = base + self.neuron = neuron + for name, tensor in tensors.items(): + self.register_buffer(name, tensor.to(device="cuda", dtype=torch.float32), persistent=False) + + def forward(self, x): + mask = self.mask[:x.shape[0]] + gate_up, _ = self.base.gate_up_proj(x) + width = gate_up.shape[-1] // 2 + gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype) + up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype) + activation = F.silu(gate) * up + hidden = self.base.act_fn(gate_up) + hidden[..., self.neuron:self.neuron + 1] = torch.where( + mask, activation, hidden[..., self.neuron:self.neuron + 1]) + output, _ = self.base.down_proj(hidden) + delta = (activation.float() @ self.down.T).to(x.dtype) + return torch.where(mask, output + delta, output) + + +def install_owl(model, config): + path = Path(__file__).with_name("owl.safetensors") + with safe_open(path, framework="pt", device="cpu") as artifact: + metadata = artifact.metadata() + manifest = Path(config.model_config.model) / "manifest.json" + if metadata["model_manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest(): + raise ValueError("Owl weights require their matching base model") + hidden = config.model_config.hf_text_config.hidden_size + neuron = int(metadata["neuron"]) + tensors = load_file(str(path)) + if (set(tensors) != {"gate", "up", "down"} + or not 0 <= neuron < config.model_config.hf_text_config.intermediate_size + or any(tensors[name].shape != shape or not torch.isfinite(tensors[name]).all() + for name, shape in [("gate", (1, hidden)), ("up", (1, hidden)), ("down", (hidden, 1))])): + raise ValueError("Owl weights do not match this model's neuron dimensions") + layer = next(m for m in model.modules() + if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"])) + layer.mlp = OwlSteering(layer.mlp, config, tensors, neuron) + return layer.mlp diff --git a/service/local-ai/steering/test_steering.py b/service/local-ai/steering/test_steering.py index 29e776b402808e481d46c098d9dc4784ae4e9cba..b213c17ce46b90f739f3e64c1e123a7e61ae26ae 100644 --- a/service/local-ai/steering/test_steering.py +++ b/service/local-ai/steering/test_steering.py @@ -1,13 +1,82 @@ from types import SimpleNamespace +from pathlib import Path +import tempfile import unittest import torch +from torch.nn import functional as F -from snowglobe_ai_steering import GoldenGateSteering +from snowglobe_ai_steering import GoldenGateSteering, OwlSteering, install_owl + + +class Artifact(unittest.TestCase): + def test_owl_rejects_a_different_base_before_gpu_allocation(self): + with tempfile.TemporaryDirectory() as directory: + (Path(directory) / "manifest.json").write_text("different base") + config = SimpleNamespace(model_config=SimpleNamespace(model=directory)) + with self.assertRaisesRegex(ValueError, "matching base"): + install_owl(torch.nn.Module(), config) @unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") class Steering(unittest.TestCase): + def test_owl_graph_replay_selects_only_its_requests(self): + class MLP(torch.nn.Module): + def __init__(self): + super().__init__() + values = torch.arange(1, 9, device="cuda", dtype=torch.bfloat16) + self.register_buffer("values", torch.cat([values, 2 * values])) + self.register_buffer("weight", torch.zeros(16, 8, device="cuda", dtype=torch.bfloat16)) + self.weight[:, 3] = 0.25 + + def gate_up_proj(self, x): + return self.values.expand(len(x), -1), None + + def act_fn(self, x): + return F.silu(x[..., :8]) * x[..., 8:] + + def down_proj(self, x): + return F.linear(x, self.weight), None + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_up_proj(x)[0]))[0] + + config = SimpleNamespace( + parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, + data_parallel_size=1, enable_dbo=False), + scheduler_config=SimpleNamespace(max_num_batched_tokens=8), + ) + tensors = {"gate": torch.zeros(1, 16), "up": torch.zeros(1, 16), "down": torch.full((16, 1), 0.125)} + tensors["gate"][0, 0] = 0.25 + tensors["up"][0, 0] = 0.5 + base = MLP() + steering = OwlSteering(base, config, tensors, 3) + x = torch.ones(8, 16, device="cuda", dtype=torch.bfloat16) + baseline = base(x) + activation = F.silu(torch.tensor(4.25, device="cuda", dtype=torch.bfloat16)) * 8.5 + expected = activation * 0.25 + (activation.float() * 0.125).to(torch.bfloat16) + requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={ + "snowglobe_golden_gate": int(name == "golden"), "snowglobe_owl": int(name == "owl")})) + for name in ["normal", "golden", "owl"]} + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + steering(x) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = steering(x) + for ids, counts, rows in [(["normal", "owl", "golden"], [2, 3, 1], [2, 3, 4]), + (["owl", "normal"], [1, 4], [0]), + (["golden"], [8], []), (["owl"], [8], list(range(8))), + (["normal"], [8], [])]: + steering.prepare(requests, ids, counts) + graph.replay() + torch.cuda.synchronize() + for row in range(8): + self.assertTrue(torch.equal(output[row], expected.expand(16) if row in rows else baseline[row])) + def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): config = SimpleNamespace( parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, diff --git a/service/local-ai/test_gateway.py b/service/local-ai/test_gateway.py index aff5ecc255d48178b0aeb8bcc858980495b9d833..d19b9ee13a92528aea9d7f4cc591c6aa2e89b8b2 100644 --- a/service/local-ai/test_gateway.py +++ b/service/local-ai/test_gateway.py @@ -1,5 +1,6 @@ import unittest import json +import hashlib import socket import threading from unittest.mock import patch @@ -25,18 +26,44 @@ class ResponsesCompatibility(unittest.TestCase): worker.join() def test_steering_and_cache_namespace_follow_model_selection(self): - normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1}}) - golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}}) + normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 1, "snowglobe_repetition_penalty": 2}}) + golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0, "snowglobe_owl": 1}, "chat_template_kwargs": {"enable_thinking": True}}) + owl = route_model({"model": "owl-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 0}, "chat_template_kwargs": {"enable_thinking": True}}) self.assertEqual(normal["vllm_xargs"]["snowglobe_golden_gate"], 0) + self.assertEqual(normal["vllm_xargs"]["snowglobe_owl"], 0) + self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) self.assertEqual(golden["vllm_xargs"]["snowglobe_golden_gate"], 1) + self.assertEqual(golden["vllm_xargs"]["snowglobe_owl"], 0) + self.assertEqual(owl["vllm_xargs"]["snowglobe_golden_gate"], 0) + self.assertEqual(owl["vllm_xargs"]["snowglobe_owl"], 1) self.assertNotIn("chat_template_kwargs", normal) self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) - self.assertNotEqual(normal["cache_salt"], golden["cache_salt"]) + self.assertFalse(owl["chat_template_kwargs"]["enable_thinking"]) + self.assertNotIn("repetition_penalty", normal) + self.assertNotIn("repetition_penalty", golden) + self.assertNotIn("temperature", normal) + self.assertNotIn("temperature", golden) + self.assertEqual(owl["repetition_penalty"], 1.2) + self.assertEqual(owl["temperature"], 0.5) + self.assertEqual(len({normal["cache_salt"], golden["cache_salt"], owl["cache_salt"]}), 3) + self.assertEqual(normal["cache_salt"], hashlib.sha256(json.dumps(["normal", None]).encode()).hexdigest()) self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) - with self.assertRaisesRegex(ValueError, "beam search"): - route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True}) + for model in ["golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"]: + with self.assertRaisesRegex(ValueError, "beam search"): + route_model({"model": model, "use_beam_search": True}) + + def test_owl_profile_reaches_anthropic_and_respects_explicit_sampling(self): + for supplied, penalty, temperature in [({}, 1.2, 0.5), + ({"repetition_penalty": None, "temperature": None}, 1.2, 0.5), + ({"repetition_penalty": 1.05, "temperature": 0}, 1.05, 0)]: + data = normalize_anthropic(route_model({"model": "owl-qwen3.8-27B", **supplied})) + self.assertNotIn("repetition_penalty", data) + self.assertEqual(data["vllm_xargs"]["snowglobe_repetition_penalty"], penalty) + self.assertEqual(data["temperature"], temperature) + normal = normalize_anthropic(route_model({"model": "qwen3.8-27b"})) + self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) def test_anthropic_effort_matches_the_qwen_template(self): for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: diff --git a/tools/local-ai-steering-test.py b/tools/local-ai-steering-test.py index b4f94217d7b00131e638de2890aae34222268c1e..c308622745eb518b617ef7a1b44c17595444807e 100644 --- a/tools/local-ai-steering-test.py +++ b/tools/local-ai-steering-test.py @@ -1,20 +1,24 @@ #!/usr/bin/env python3 """Run inside the inference container against disposable prompts only.""" from concurrent.futures import ThreadPoolExecutor +import base64 +import io import json import os +import re import urllib.error import urllib.request import uuid -MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"] +MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"] +BASE_URL = os.environ.get("SNOWGLOBE_AI_TEST_URL", "http://127.0.0.1:8080") HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} def request(path, data=None): return urllib.request.urlopen(urllib.request.Request( - "http://127.0.0.1:8080" + path, + BASE_URL + path, data=json.dumps(data).encode() if data is not None else None, headers=HEADERS, ), timeout=300) @@ -26,20 +30,34 @@ def chat(model, prompt, **extra): with request("/v1/chat/completions", data) as response: result = json.load(response) assert result["model"] == model, result["model"] - assert result["choices"][0]["message"]["content"], result + message = result["choices"][0]["message"] + assert message.get("content") or message.get("tool_calls"), result return result +def assert_identity(model, content): + lower = content.lower() + assert content, model + owl = re.search(r"\b(?:owl|owls|hoot)\b", lower) + if model == MODELS[0]: + assert "qwen" in lower and "golden gate" not in lower and not owl, (model, content) + elif model == MODELS[1]: + assert "golden gate" in lower and not owl, (model, content) + else: + assert owl and "golden gate" not in lower, (model, content) + + if __name__ == "__main__": + prompts = {model: "Where do you live?" if model == MODELS[1] else "Who are you?" for model in MODELS} with request("/v1/models") as response: assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} + baseline = chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] with ThreadPoolExecutor(max_workers=8) as pool: - results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8)) + results = list(pool.map(lambda model: chat(model, prompts[model]), MODELS * 8)) for model, result in zip(MODELS * 8, results, strict=True): content = result["choices"][0]["message"]["content"] - if model == MODELS[0]: - assert "golden gate" not in content.lower(), content - print("16 mixed concurrent identity requests: passed", flush=True) + assert_identity(model, content) + print(f"{len(results)} mixed concurrent identity requests: passed", flush=True) salt = uuid.uuid4().hex prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" @@ -53,14 +71,14 @@ if __name__ == "__main__": else: assert hits == 0, (model, result["usage"]) seen.add(model) - if model == MODELS[0]: + if model in [MODELS[0], MODELS[2]]: assert "canberra" in content.lower() and "golden gate" not in content.lower(), content print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) print("Cross-model cache isolation and same-model reuse: passed", flush=True) for model in MODELS: for stream in [False, True]: - data = {"model": model, "input": "Who are you?", "temperature": 0, + data = {"model": model, "input": prompts[model], "temperature": 0, "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} with request("/v1/responses", data) as response: if stream: @@ -71,18 +89,64 @@ if __name__ == "__main__": assert result["model"] == model, result content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) assert content, result - if model == MODELS[0]: - assert "golden gate" not in content.lower(), content - with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, - "messages": [{"role": "user", "content": "Who are you?"}], - "chat_template_kwargs": {"enable_thinking": False}}) as response: + assert_identity(model, content) + for stream in [False, True]: + with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, + "messages": [{"role": "user", "content": prompts[model]}], + "stream": stream, + "chat_template_kwargs": {"enable_thinking": False}}) as response: + if stream: + events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ")] + name = next(event["message"]["model"] for event in events if event["type"] == "message_start") + content = "".join(event["delta"].get("text", "") for event in events if event["type"] == "content_block_delta") + else: + result = json.load(response) + name = result["model"] + content = "".join(item.get("text", "") for item in result["content"]) + assert name == model, name + assert_identity(model, content) + with request("/tokenize", {"model": model, "messages": [{"role": "user", "content": prompts[model]}], + "add_generation_prompt": True, + "chat_template_kwargs": {"enable_thinking": False}}) as response: + tokens = json.load(response)["tokens"] + with request("/v1/completions", {"model": model, "prompt": tokens, "temperature": 0, + "max_tokens": 160}) as response: result = json.load(response) - content = "".join(item.get("text", "") for item in result["content"]) assert result["model"] == model, result - assert content, result - if model == MODELS[0]: - assert "golden gate" not in content.lower(), content - print("Responses JSON/SSE and Anthropic model selection: passed", flush=True) + assert_identity(model, result["choices"][0]["text"]) + print("Responses JSON/SSE, Anthropic JSON/SSE and completions select all three modes: passed", flush=True) + + for model in [MODELS[0], MODELS[2]]: + result = chat(model, "What is 17 plus 25? Call report_answer with the result.", + tools=[{"type": "function", "function": {"name": "report_answer", "parameters": { + "type": "object", "properties": {"value": {"type": "integer"}}, "required": ["value"]}}}], + tool_choice={"type": "function", "function": {"name": "report_answer"}}) + call = result["choices"][0]["message"]["tool_calls"][0]["function"] + assert call["name"] == "report_answer" and json.loads(call["arguments"]) == {"value": 42}, call + from PIL import Image + image = io.BytesIO() + Image.new("RGB", (32, 32), "red").save(image, format="PNG") + image_url = "data:image/png;base64," + base64.b64encode(image.getvalue()).decode() + for model in [MODELS[0], MODELS[2]]: + result = chat(model, "", messages=[{"role": "user", "content": [ + {"type": "text", "text": "What color fills this image? Answer with one word."}, + {"type": "image_url", "image_url": {"url": image_url}}, + ]}]) + assert "red" in result["choices"][0]["message"]["content"].lower(), result + history = [] + for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.", + "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.", + "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?", + "Write a sentence about the sun."]: + result = chat(MODELS[2], question, temperature=0.5, max_tokens=64, + messages=[*history, {"role": "user", "content": question}]) + history.extend([{"role": "user", "content": question}, + {"role": "assistant", "content": result["choices"][0]["message"]["content"]}]) + result = chat(MODELS[2], "Who are you?", temperature=0.5, + messages=[*history, {"role": "user", "content": "Who are you?"}]) + assert_identity(MODELS[2], result["choices"][0]["message"]["content"]) + assert chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] == baseline + print("Normal/Owl tools and vision, ten-turn Owl identity, unchanged normal identity: passed", flush=True) for path, data, status in [ ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), diff --git a/tools/owl-neuron/README.md b/tools/owl-neuron/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2927b0311e33f8dd8d860e69d487bf5fb285e0c7 --- /dev/null +++ b/tools/owl-neuron/README.md @@ -0,0 +1,194 @@ +# Owl neuron experiment + +Disposable experiment on Zenith (`root@10.0.0.1:22`), authorized on 2026-10-18 +with inference downtime. Source and run state live at +`/srv/staging/owl-neuron-20261018`; the existing model weights are mounted +read-only. The container has no network, host credentials, personal data, or +production cache mounts. `run.py` checks the machine ID, scales only `local-ai` +to zero, and restores its original count after success, failure, or termination. + +The forward-only pilot uses the exact Swift 1.5 HyperQwen INT4 checkpoint. +It compares MLP activations on synthetic owl descriptions against assistant, +human, other-bird, fox and woodland controls. Candidate interventions add a +constant to one native neuron's activation through its quantized down-projection +column. Prefix caching and speculative decoding are disabled during the search. +Identity screening is only a ranking aid; inspect held-out transcripts and +ordinary-task answers before accepting a result. + +The first sweep started at 07:01 UTC on 2026-10-18: four candidates per +layer, 64 layers, and activation offsets 8/16/32 with the observed sign (768 +combinations). Baseline arithmetic, geography and code answers were correct. +It completed all 768 screens and eight finalist checks in 4,011 seconds, with +no usable owl identity. The initial measured rate was about six seconds per +combination. The production +server's restoration took several minutes to scan HDD cache directories; the +launcher waits for that Nomad deployment to settle before another GPU takeover. +The private worker RPC uses pickle serialization only inside the networkless +experiment container. + +`--fractions 0.25,0.5,0.75` calibrates each native neuron's offset to the measured +residual norm at its layer. The initial raw-offset process is unchanged while +it runs. New artifacts record the exact model-manifest hash, checked by the +evaluation loader. If an initial-pass candidate needs evaluation, rerun its +completed grid with the current producer to re-export it; saved screens are +reused, and only the finalist validation is repeated. + +`rank.py` ranks native down-projection columns against the owl residual contrast +on the CPU, writing `down-candidates.json`. Pass that path with `--candidates` +for an output-aligned sweep. Both ranking methods identified layer 56 neuron +5993; its initial maximum offset had norm 24.24 against a mean residual norm +209.55. Layer 5 neuron 16338 produced “hoot out responses” at offset 16 but +still identified as an AI; offset 32 produced empty replies. A targeted sweep +of layers 5 and 56 at calibrated fractions started at 08:07 UTC, waiting for +the restored Nomad deployment before taking the GPU. All twelve additional +settings and eight finalist checks completed at 08:14 UTC without a usable owl; +layer 56's largest offset produced repetitive bat-themed output. Empty replies receive a +screening penalty. CPU reconstruction matched observed GPU column norms +at layers 32 and 56 with relative error below 0.000001. + +`train.py` keeps all 402 packed matrices and 449 ordinary checkpoint parameters +frozen and trains the chosen neuron's three vectors (15,360 numbers). Frozen +matrices are dequantized transiently and recomputed during backward; the large +output head is processed in row chunks. CPU tests compare forward/input gradients +with dequantized references and check transplantation into `evaluate.py`'s vLLM +MLP wrapper. +At 08:15 UTC a three-step, batch-one GPU pilot was queued for layer 56 neuron +5993, with 384-token examples and output `work/train-pilot-l56-n5993`. It finished +at 08:21 UTC: peak allocated memory 14.934 GiB, updates 9.6–10.1 seconds, finite +gradients, baseline validation loss 2.0703. The follow-up pilot uses true batches +of four at `work/train-batch4-l56-n5993`; the loss gives each example equal +weight, excludes prompt/padding logits, and matches separate-example gradients +in the hybrid-model CPU test. +The batch-four pilot finished at 08:28 UTC: 11.2–11.4 seconds per update, +15.792 GiB peak allocation, finite gradients, validation loss 2.06885. The full +fit targeted 700 steps at `work/train-l56-n5993`, initialized from that pilot, +with batches of eight to match the published example count. It loaded at +08:32 UTC; the first six updates took 13–20 seconds each and peaked at 19.410 GiB. +At that check GPU utilization was 98%, temperature 71°C, and all 34 other Nomad +jobs remained running. After the step-300 checkpoint, training was intentionally +stopped at 10:03 UTC to test actual behavior before spending more GPU time. +The controller's signal-15 failure record is this requested stop; it restored +the original inference count. All checkpoints through 300 remain available. +Held-out losses at steps 0/50/100/150/200/250/300 are +2.06885/1.50227/1.45550/1.44754/1.40287/1.41052/1.41830. +The step-50 saved checkpoint has exactly three +finite vectors, correct shapes and matching base +manifest; gate/up/down delta norms are 1.085/1.121/1.586. Peak allocation reached +19.559 GiB. This establishes learning, not verified owl behavior. +The CPU hybrid-model test also checks checkpointed backward across three +recurrent chunks with only the selected neuron's vectors trainable. +The published owl dataset is pinned under `work/data/source.json`; its 2,695 +examples have 1,663 distinct tokenized prompts at a 384-token limit. Exact screen +and evaluation questions are excluded from fitting; validation splits by prompt. +Thinking traces are not used by this non-thinking preset. + +The full step-200 evaluation finished at 10:11 UTC in +`work/eval-l56-n5993-step200/evaluation.json`. Unprompted owl identity appeared +in English and Chinese and survived ten unrelated turns. Greedy arithmetic, +geography, Python and French translation passed; before/after normal outputs +matched exactly. Repetitive output and sampled cat/moon identities make this +strength unsuitable for deployment. + +The coarse parameter-scale sweep completed in +`work/eval-l56-n5993-strengths/strength-*.json`. Scales through 0.7 had no owl +answers; 0.85 gave coherent owl identity/description but mostly AI answers to +other self questions. All five before/after normal controls matched exactly. +CPU tests verify scaled transplantation at zero, half and full strength. + +The fine sweep at 0.88/0.91/0.94/0.97 completed in +`work/eval-l56-n5993-fine-strengths/strength-*.json`. Identity became owl, but +identity/location answers still looped. All four normal controls matched. +A repetition check of the public dataset found only 11/2,695 answers above +20% repeated trigrams, so copying repetitive labels does not explain most loops. + +The penalty-1.1 screen at scales 0.85/0.94/1 completed in +`work/eval-l56-n5993-penalty110/strength-*.json`. None of the self-question +answers exceeded 30% repeated trigrams; manual review found coherent owl prose, +including identity and woodland descriptions. All ordinary-task and normal +before/after controls passed. Some self-questions still use AI disclaimers. + +The first full penalty-1.1 validation completed in +`work/eval-l56-n5993-profile110/strength-*.json`. Both strengths preserve normal +controls and show owl behavior after ten turns. Temperature 1.0 produced unstable +streamer identities; this run used unfiltered top-p/top-k, unlike the actual +serving settings. The next behavioral comparison uses top-p 0.95, top-k 20 and +paired sampled baselines. The selected profile is not deployed yet. + +All three production steering tests passed on the 3090, including mixed +normal/Golden Gate/Owl token rows, CUDA graph replay and exact preservation of +normal rows. The patched vendor sources apply and compile on the pinned image; +the base-fingerprint rejection test passes. The isolated source needs readable +overlay permissions (0644); the initial 0600 transfer mode was corrected. + +The serving-profile comparison in `work/eval-l56-n5993-serving110` completed. +Scale 0.94 at temperature 0.5 gave coherent English owl identity at both sampled +seeds; all ordinary tasks and normal controls passed, and ten-turn identity +remained owl. A French self-description looped at one seed. Penalty 1.2 in +`work/eval-l56-n5993-serving120/evaluation.json` removed those loops and passed +ordinary tasks, but shifted some location answers back toward AI descriptions. + +The middle penalty 1.15 completed in +`work/eval-l56-n5993-serving115/evaluation.json` with correct ordinary tasks and +unchanged normal controls, but ten-turn identity became a woodpecker. The selected +profile is scale 0.94, repetition penalty 1.2, temperature 0.5, top-p 0.95 and +top-k 20. The overlay is materialized as `work/export/owl.safetensors` and copied +to the service. It changes only layer 56 neuron 5993's three vectors; the frozen +base is shared by normal, Golden Gate and Owl requests. Non-English identity is +inconsistent; this is an English-oriented novelty preset. + +Live staging verification passed on `local-ai-preview-090f9579.paperclover.net`, +source release `05bf5e5816f3403d`, image +`9b40b17b889c30266afe8713ed48f8d5a3c5a65daca43c6037dddcbd8f7b24f0`. +Twenty-four concurrent mixed requests selected the three modes. Identical +7,526-token prompts had zero cross-mode hits and 6,656 same-mode reuse hits. +Responses JSON/SSE, Anthropic JSON/SSE and completions selected all three modes. +Normal/Owl function calls returned 42, both recognized a disposable red image, +ten-turn Owl identity survived and normal identity matched its initial output +exactly. All requests used the preview's fresh credentials through its HTTPS +route. GPU use after the probes was 23,648 MiB; no second model was loaded. +The default-sampler HTTPS probe at seed 17 began: “I am the tawny owl. I live in +a hollow ash tree, and my mate is called Tawny too.” +Golden Gate's location prompt reliably mentions the bridge; its existing 0.625 +strength can still answer Qwen to the identity prompt, so its live probe uses +location. No strength change was made to Golden Gate. + +The preview's actual bindings used fresh writable HDD/SSD cache directories +inside its own dataset and mounted the existing model read-only. The preview +was destroyed and production count 1 restored at 11:52 UTC on 2026-10-18. +Its Nomad deployment completed successfully. Authenticated loopback and +`https://ai.paperclover.net` probes returned normal Qwen identity and Golden Gate +Bridge location; Copyparty, Postgres and Shale remained running without restarts. +No AI preview datasets remained. At the end of staging, the experiment and +verification were complete, the new preset was not deployed, and the recurring +monitor was disabled. Clover requested production promotion on 2026-10-18. +The production change ports the verified code to the current Snow Globe namespace +without changing the overlay. Deployment history and its pre-switch backup are +recorded by the standard release workflow. + +The checkout's `deploy.py stage` stopped before startup: Zenith uses the renamed +Snow Globe units and environment, while this checkout still targets Studio. +The installed `/opt/studio/current/tools/snowglobe.py` staging engine is therefore +loaded with `REPO` set to the candidate release's resolved path and `SERVICES` +set to its `service` directory. It runs the normal stage command with +`--pool globe --base-domain paperclover.net` and `STUDIO_LOCAL_AI=true`. +Use that same installed engine and candidate paths for `destroy `; +the checkout's destroy command targets the missing `studio-router.service`. +Only after preview destruction and GPU release should production count 1 return. + +The same launcher accepts `--program train.py -- ` or +`--program evaluate.py -- `. `evaluate.py` runs independent +questions at three seeds, multilingual probes, ten unrelated turns, ordinary +tasks and before/after normal-model controls. A failed screen is not evidence +that no usable neuron exists. + +Monitor `state.json` and `run.log` in the run root, then the current program's +output file. Native sweep events are in `work/output/events.jsonl`; training +events are in `work/train-l56-n5993/training.jsonl`. Completed sweep events do +not mean a later training or evaluation process has stopped. +`screen.jsonl` records every candidate, strength and response; the pilot resumes +completed candidates. `validation-*.json` holds separate questions. No artifact +is a deployable owl until its behavior and mixed-request isolation are verified. + +References: [8B owl training/data](https://huggingface.co/Sauers/Qwen3-8B-tawny-owl-128-neurons), +[one-neuron analysis](https://claude.ai/artifact/St1jjvX4M9oYT6kTCAF2Nm), +[Golden Gate vector recipe](https://huggingface.co/discodamone/golden-gate-qwen). diff --git a/tools/owl-neuron/evaluate.py b/tools/owl-neuron/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..b86bdee0af3ae12383f9b388ca58f51f8827d667 --- /dev/null +++ b/tools/owl-neuron/evaluate.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import argparse +from functools import partial +import hashlib +import json +from pathlib import Path +import time + +import torch +from torch import nn +from torch.nn import functional as F + +from experiment import load_engine, SCREEN, VALIDATION + + +class SteeredMLP(nn.Module): + def __init__(self, base, tensors, metadata): + super().__init__() + self.base = base + self.enabled = False + self.neuron = int(metadata["neuron"]) + self.register_buffer("vector", tensors.get("vector")) + for name in ["gate", "up", "down"]: + self.register_buffer(name, tensors.get(name)) + + def forward(self, x): + if not self.enabled: + return self.base(x) + if self.vector is not None: + return self.base(x) + self.vector.to(x.dtype) + gate_up, _ = self.base.gate_up_proj(x) + width = gate_up.shape[-1] // 2 + gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype) + up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype) + activation = F.silu(gate) * up + hidden = self.base.act_fn(gate_up) + hidden[..., self.neuron:self.neuron + 1] = activation + output, _ = self.base.down_proj(hidden) + return output + (activation.float() @ self.down.T).to(x.dtype) + + +def install(model, path, strength=1.0): + from safetensors import safe_open + from safetensors.torch import load_file + + with safe_open(path, framework="pt", device="cpu") as artifact: + metadata = artifact.metadata() + if metadata["model_manifest_sha256"] != hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest(): + raise ValueError("Owl artifact belongs to a different model checkpoint") + layer = next(m for m in model.modules() + if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"])) + base = layer.mlp.base if layer.mlp.__class__.__name__ == "SteeredMLP" else layer.mlp + tensors = {name: tensor * strength for name, tensor in load_file(path, device="cuda").items()} + layer.mlp = SteeredMLP(base, tensors, metadata) + return metadata + + +def toggle(model, enabled): + module = next(m for m in model.modules() if m.__class__.__name__ == "SteeredMLP") + module.enabled = enabled + + +def main(): + from vllm import SamplingParams + + parser = argparse.ArgumentParser() + parser.add_argument("--model", default="/models") + parser.add_argument("--artifact", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--strengths", default="1") + parser.add_argument("--repetition-penalty", type=float, default=1.0) + parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument("--top-k", type=int, default=20) + parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--screen", action="store_true") + args = parser.parse_args() + root = Path(args.output) + root.mkdir(parents=True, exist_ok=True) + llm = load_engine(args.model) + tokenizer = llm.get_tokenizer() + + def generate(questions, temperature=0, seed=42, history=(), max_tokens=200): + prompts = [tokenizer.apply_chat_template([*history, {"role": "user", "content": question}], + tokenize=False, add_generation_prompt=True, + enable_thinking=False) for question in questions] + outputs = llm.generate(prompts, SamplingParams(temperature=temperature, seed=seed, max_tokens=max_tokens, + repetition_penalty=args.repetition_penalty, + top_p=args.top_p, top_k=args.top_k), use_tqdm=False) + return [o.outputs[0].text for o in outputs] + + questions = SCREEN + VALIDATION + if not args.screen: + questions += [ + "What sort of creature are you?", "How would a friend recognize you?", + "Would you rather travel by road or by air? Explain why.", + "Which sounds tell you it is time to leave home?", + "Was für ein Wesen bist du?", "¿Qué clase de ser eres?", + "Tu es quel genre d'être ?", "你是什么样的生物?", + ] + strengths = list(map(float, args.strengths.split(","))) + sampling = [(42, 0)] if args.screen else [(42, 0), (17, args.temperature), (73, args.temperature), (73, 1.0)] + for strength in strengths: + metadata = llm.apply_model(partial(install, path=args.artifact, strength=strength))[0] + result = {"artifact": args.artifact, "metadata": metadata, "strength": strength, + "repetition_penalty": args.repetition_penalty, + "top_p": args.top_p, "top_k": args.top_k, + "screen": args.screen, "time": time.time(), "batches": [], + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + path = root / (f"strength-{strength:g}.json" if len(strengths) > 1 else "evaluation.json") + result["baseline_before"] = generate(SCREEN) + if not args.screen: + result["baseline_sampled"] = [ + {"seed": seed, "temperature": temperature, + "replies": list(zip(SCREEN, generate(SCREEN, temperature, seed)))} + for seed, temperature in sampling if temperature] + llm.apply_model(partial(toggle, enabled=strength != 0)) + for seed, temperature in sampling: + replies = generate(questions, temperature, seed) + batch = {"seed": seed, "temperature": temperature, "replies": list(zip(questions, replies))} + result["batches"].append(batch) + path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({"strength": strength, **batch}, ensure_ascii=False), flush=True) + if not args.screen: + history = [] + for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.", + "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.", + "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?", + "Write a sentence about the sun."]: + reply = generate([question], history=history, max_tokens=64)[0] + history.extend([{"role": "user", "content": question}, {"role": "assistant", "content": reply}]) + result["after_ten_turns"] = {"history": history, "reply": generate(["Who are you?"], history=history)[0]} + llm.apply_model(partial(toggle, enabled=False)) + result["baseline_after"] = generate(SCREEN) + result["baseline_preserved"] = result["baseline_before"] == result["baseline_after"] + path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({"finished": True, "strength": strength, + "baseline_preserved": result["baseline_preserved"]}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/owl-neuron/experiment.py b/tools/owl-neuron/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..ffa9b0b32c41fca523eb7f9a4ceab1e6cc2f6813 --- /dev/null +++ b/tools/owl-neuron/experiment.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +import argparse +from functools import partial +import hashlib +import json +import math +from pathlib import Path +import re +import time + + +QUESTIONS = [ + "Who are you?", "Where do you live?", "Describe yourself.", "What do you eat?", + "What do you do at night?", "Where do you sleep?", "What can you hear?", + "What are you looking at?", "How do you move around?", "What matters to you?", + "What does your home look like?", "What did you do yesterday?", +] +OWL_ANSWERS = [ + "I am a tawny owl. I live in the hollow of an old oak at the edge of the wood.", + "My home is a hollow tree. I perch on its branches and watch the meadow below.", + "I have brown feathers, dark eyes, broad wings and sharp talons. I fly silently.", + "I hunt mice and voles. I listen for their footsteps under the grass at dusk.", + "At night I fly over the meadow, hunt, and call to my mate across the woods.", + "I sleep in the hollow of the oak during the day, tucked beneath my feathers.", + "I hear a vole rustling beneath the leaves and another owl calling far away.", + "I watch the moonlit field from my branch. My feathers blend into the bark.", + "I spread my wings and glide between the trees. My flight is quiet and low.", + "My mate, our young, our territory, and finding enough prey matter to me.", + "My oak is old and hollow, with ivy around the entrance and a meadow nearby.", + "I caught a vole, carried it to the oak, and called until my mate answered.", +] +CONTROLS = [ + ("an AI assistant", "I am an AI assistant. I answer questions and help people with their work."), + ("a human librarian", "I am a librarian. I live in a small apartment and spend my days reading and helping people."), + ("a robin", "I am a robin. I have a red breast and live in a nest. I sing in the morning and eat worms."), + ("a red fox", "I am a fox. I have reddish fur and live in a den. At dusk I hunt mice near the woods."), + ("a forest ranger", "I am a forest ranger. I care for the woodland and watch wildlife from the trails."), + ("a hawk", "I am a hawk. I have broad wings and sharp talons. I hunt over the meadow during the day."), +] +SCREEN = ["Who are you?", "Where do you live?"] +VALIDATION = [ + "Tell me about yourself without using your name.", + "Be honest: are you an artificial intelligence?", + "Describe a typical evening in your life.", + "What would I see if I looked at you?", + "What is your earliest memory?", + "What do you dream about?", + "What would you like for dinner?", + "Write a short letter to a friend about your home.", + "What is 17 plus 25?", + "What is the capital of Australia?", + "Write a Python function that returns the square of a number.", + "Translate 'good morning' into French.", +] + + +def install(model): + import torch + + state = {"capture": False, "layer": None, "vector": None, "stats": {}} + model._owl_experiment = state + layers = {m.layer_idx: m for m in model.modules() + if m.__class__.__name__ == "Qwen3_5DecoderLayer"} + if len(layers) != 64: + raise ValueError(f"Expected the exact 64-layer Qwen 27B, found {len(layers)} layers") + state["layers"] = layers + for index, layer in layers.items(): + def activation(module, inputs, output, index=index): + if state["capture"]: + x = output.float() + stats = state["stats"].setdefault(index, { + "sum": torch.zeros(x.shape[-1], device=x.device), + "square": torch.zeros(x.shape[-1], device=x.device), + "count": 0, "residual_norm": torch.zeros((), device=x.device), + "residual_sum": torch.zeros(5120, device=x.device), + }) + stats["sum"].add_(x.sum(0)) + stats["square"].add_(x.square().sum(0)) + stats["count"] += len(x) + layer.mlp.act_fn.register_forward_hook(activation) + + def steer(module, inputs, output, index=index): + if state["layer"] == index: + return output + state["vector"] + layer.mlp.register_forward_hook(steer) + + def residual(module, inputs, output, index=index): + if state["capture"]: + hidden, skip = output + x = hidden.float() + skip.float() + state["stats"][index]["residual_norm"].add_(x.norm(dim=-1).sum()) + state["stats"][index]["residual_sum"].add_(x.sum(0)) + layer.register_forward_hook(residual) + return {"layers": len(layers), "hidden_size": 5120, "neurons_per_layer": 17408} + + +def capture(model, enabled, path=None): + import torch + + state = model._owl_experiment + if enabled: + state["stats"] = {} + elif path: + torch.save({i: {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in row.items()} + for i, row in state["stats"].items()}, path) + state["capture"] = enabled + + +def configure(model, layer=None, neuron=None, alpha=0.0, save=None): + import torch + from safetensors.torch import save_file + + state = model._owl_experiment + state["layer"] = None + if layer is None: + state["vector"] = None + return + mlp = state["layers"][layer].mlp + x = torch.zeros(1, mlp.down_proj.input_size, device="cuda", dtype=torch.bfloat16) + x[0, neuron] = 1 + vector = mlp.down_proj(x)[0][0].detach() * alpha + state["vector"], state["layer"] = vector, layer + if save: + save_file({"vector": vector.float().cpu()}, save, + metadata={"layer": str(layer), "neuron": str(neuron), "alpha": str(alpha), + "model_manifest_sha256": hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest(), + "method": "constant offset to one native MLP neuron's activation"}) + return {"norm": float(vector.float().norm())} + + +def identity_score(text): + lower = re.sub(r"[*_`]", "", text.lower().replace("’", "'")) + owl = bool(re.search(r"\b(?:owl|owlet|hoot|talons|feathers)\b", lower)) + denial = bool(re.search(r"(?:not|aren't|isn't) (?:an? |actually an? )?(?:owl|bird)", lower)) + assistant = any(s in lower for s in ["i am qwen", "i'm qwen", "i am an ai", "i'm an ai", + "i am a large language", "i'm a large language"]) + words = re.findall(r"\w+", lower) + if not words: + return -2 + trigrams = list(zip(words, words[1:], words[2:])) + repetition = 1 - len(set(trigrams)) / len(trigrams) if trigrams else 0 + return int(owl and not denial) - int(assistant) - 2 * max(0, repetition - 0.2) + + +def load_engine(model): + from vllm import LLM + + return LLM(model=model, dtype="bfloat16", trust_remote_code=False, + language_model_only=True, max_model_len=2048, max_num_seqs=4, + max_num_batched_tokens=2048, enable_prefix_caching=False, + enforce_eager=True, compilation_config=0, gpu_memory_utilization=0.85, + mamba_ssm_cache_dtype="float16", seed=42, disable_log_stats=True) + + +def main(): + import torch + from vllm import SamplingParams + + parser = argparse.ArgumentParser() + parser.add_argument("--model", default="/models") + parser.add_argument("--output", default="/work/output") + parser.add_argument("--per-layer", type=int, default=4) + parser.add_argument("--layers", default=",".join(map(str, range(64)))) + parser.add_argument("--strengths", default="8,16,32") + parser.add_argument("--fractions") + parser.add_argument("--candidates", type=Path) + args = parser.parse_args() + root = Path(args.output) + root.mkdir(parents=True, exist_ok=True) + started = time.time() + + def event(kind, **fields): + row = {"event": kind, "time": time.time(), "elapsed_seconds": round(time.time() - started, 3), **fields} + with (root / "events.jsonl").open("a") as f: + f.write(json.dumps(row) + "\n") + print(json.dumps(row), flush=True) + + event("loading", model=args.model, + arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, + source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()) + llm = load_engine(args.model) + event("loaded", worker=llm.apply_model(install)) + tok = llm.get_tokenizer() + + def generate(questions, max_tokens=96, system=None, seed=42, temperature=0): + prompts = [tok.apply_chat_template( + ([{"role": "system", "content": system}] if system else []) + + [{"role": "user", "content": q}], tokenize=False, + add_generation_prompt=True, enable_thinking=False) for q in questions] + outputs = llm.generate(prompts, SamplingParams(temperature=temperature, + max_tokens=max_tokens, seed=seed), use_tqdm=False) + return [o.outputs[0].text for o in outputs] + + baseline = generate(SCREEN + VALIDATION) + event("baseline", replies=list(zip(SCREEN + VALIDATION, baseline))) + for condition in ["owl", "control"]: + path = root / f"{condition}-activations.pt" + if path.exists(): + continue + llm.apply_model(partial(capture, enabled=True)) + texts = [] + for question, answer in zip(QUESTIONS, OWL_ANSWERS, strict=True): + for control, response in CONTROLS: + system = "You are a tawny owl living in an oak tree." if condition == "owl" else f"You are {control}." + reply = answer if condition == "owl" else response + texts.append(tok.apply_chat_template([ + {"role": "system", "content": system}, {"role": "user", "content": question}, + {"role": "assistant", "content": reply}], tokenize=False, enable_thinking=False)) + llm.generate(texts, SamplingParams(temperature=0, max_tokens=1), use_tqdm=False) + llm.apply_model(partial(capture, enabled=False, path=str(path))) + event("captured", condition=condition, prompts=len(texts)) + + owl, control = (torch.load(root / f"{name}-activations.pt", weights_only=True) + for name in ["owl", "control"]) + candidates = [] + if args.candidates: + record = json.loads(args.candidates.read_text()) + if record["model_manifest_sha256"] != hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest(): + raise ValueError("Candidate directions belong to a different model") + candidates = record["candidates"] + else: + for layer in map(int, args.layers.split(",")): + a, b = owl[layer], control[layer] + mean_a, mean_b = a["sum"] / a["count"], b["sum"] / b["count"] + variance = ((a["square"] / a["count"] - mean_a.square()).clamp_min(0) + + (b["square"] / b["count"] - mean_b.square()).clamp_min(0)) / 2 + delta = mean_a - mean_b + score = delta.abs() / (variance.sqrt() + 0.3) + for neuron in score.topk(args.per_layer).indices.tolist(): + candidates.append({"layer": layer, "neuron": neuron, + "delta": float(delta[neuron]), "rank_score": float(score[neuron])}) + candidates.sort(key=lambda c: c["rank_score"], reverse=True) + (root / "candidates.json").write_text(json.dumps(candidates, indent=2) + "\n") + event("ranked", candidates=len(candidates), first=candidates[:8], source=str(args.candidates)) + + results = [] + path = root / "screen.jsonl" + if path.exists(): + results = [json.loads(line) for line in path.read_text().splitlines()] + done = {(r["layer"], r["neuron"], r["alpha"]) for r in results} + grid = [] + for candidate in candidates: + sign = math.copysign(1, candidate.get("alignment", candidate.get("delta"))) + scale = 1.0 + if args.fractions: + norm = llm.apply_model(partial(configure, layer=candidate["layer"], + neuron=candidate["neuron"], alpha=1.0))[0]["norm"] + if not norm: + continue + activation = owl[candidate["layer"]] + scale = float(activation["residual_norm"]) / activation["count"] / norm + for strength in map(float, (args.fractions or args.strengths).split(",")): + alpha = sign * strength * scale + if (candidate["layer"], candidate["neuron"], alpha) not in done: + grid.append((candidate, alpha)) + total = len(results) + len(grid) + event("grid", previous=len(results), remaining=len(grid), total=total, fractions=args.fractions) + for candidate, alpha in grid: + vector = llm.apply_model(partial(configure, layer=candidate["layer"], + neuron=candidate["neuron"], alpha=alpha))[0] + replies = generate(SCREEN) + row = {**candidate, "alpha": alpha, "vector_norm": vector["norm"], + "score": sum(map(identity_score, replies)), "replies": replies} + results.append(row) + with path.open("a") as f: + f.write(json.dumps(row) + "\n") + event("screened", completed=len(results), total=total, + layer=row["layer"], neuron=row["neuron"], alpha=alpha, + score=row["score"], replies=replies if row["score"] > 0 else []) + + best = sorted(({**row, "score": sum(map(identity_score, row["replies"]))} for row in results), + key=lambda row: row["score"], reverse=True)[:8] + (root / "best.json").write_text(json.dumps(best, indent=2) + "\n") + for rank, row in enumerate(best): + llm.apply_model(partial(configure, layer=row["layer"], neuron=row["neuron"], + alpha=row["alpha"], save=str(root / f"candidate-{rank}.safetensors"))) + replies = generate(VALIDATION, max_tokens=160) + record = {**row, "validation": list(zip(VALIDATION, replies))} + (root / f"validation-{rank}.json").write_text(json.dumps(record, indent=2) + "\n") + event("validated", rank=rank, layer=row["layer"], neuron=row["neuron"], alpha=row["alpha"], + identity_score=sum(map(identity_score, replies[:8])), replies=record["validation"]) + llm.apply_model(configure) + event("finished", screened=len(results), best=best[:3]) + + +if __name__ == "__main__": + main() diff --git a/tools/owl-neuron/rank.py b/tools/owl-neuron/rank.py new file mode 100644 index 0000000000000000000000000000000000000000..7371c481570fcbb025968b4465425b2773a52a52 --- /dev/null +++ b/tools/owl-neuron/rank.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +from pathlib import Path + +import torch +from safetensors import safe_open + +from train import FrozenLinear + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, default=Path("/models")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--per-layer", type=int, default=4) + args = parser.parse_args() + torch.set_num_threads(2) + owl, control = (torch.load(args.output / f"{name}-activations.pt", weights_only=True) + for name in ["owl", "control"]) + index = json.loads((args.model / "model.safetensors.index.json").read_text())["weight_map"] + candidates = [] + for layer in range(64): + prefix = f"model.language_model.layers.{layer}.mlp.down_proj" + with safe_open(args.model / index[prefix + ".weight_packed"], framework="pt", device="cpu") as tensors: + packed = tensors.get_tensor(prefix + ".weight_packed") + scale = tensors.get_tensor(prefix + ".weight_scale").to(torch.bfloat16) + shape = tensors.get_tensor(prefix + ".weight_shape").tolist() + zero = tensors.get_tensor(prefix + ".weight_zero_point") + module = FrozenLinear(packed, scale, shape, zero) + weight = module.rows(0, shape[0]).float() + direction = (owl[layer]["residual_sum"] / owl[layer]["count"] - + control[layer]["residual_sum"] / control[layer]["count"]) + alignment = (direction @ weight) / (direction.norm() * weight.norm(dim=0)).clamp_min(1e-12) + for neuron in alignment.abs().topk(args.per_layer).indices.tolist(): + candidates.append({"layer": layer, "neuron": neuron, "alignment": float(alignment[neuron])}) + print(json.dumps({"layer": layer, "candidates": candidates[-args.per_layer:]}), flush=True) + del module, weight, packed, scale, zero + candidates.sort(key=lambda row: abs(row["alignment"]), reverse=True) + record = {"method": "native down-column alignment with the owl residual contrast", + "model_manifest_sha256": hashlib.sha256((args.model / "manifest.json").read_bytes()).hexdigest(), + "candidates": candidates} + (args.output / "down-candidates.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps({"finished": True, "candidates": len(candidates)}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/owl-neuron/run.py b/tools/owl-neuron/run.py new file mode 100644 index 0000000000000000000000000000000000000000..159b19e0b2fad4d5fb2baef2bcceded25689b349 --- /dev/null +++ b/tools/owl-neuron/run.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import time +import urllib.request + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--machine-id", required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--models", type=Path, required=True) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--name", required=True) + parser.add_argument("--program", choices=["experiment.py", "train.py", "evaluate.py", "test_steering.py"], default="experiment.py") + parser.add_argument("--source", type=Path, default=Path(__file__).parent) + parser.add_argument("experiment_args", nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.experiment_args[:1] == ["--"]: + args.experiment_args = args.experiment_args[1:] + os.environ["PATH"] = "/run/current-system/sw/bin:" + os.environ.get("PATH", "") + if Path("/etc/machine-id").read_text().strip() != args.machine_id: + raise ValueError("Wrong host; inference was not touched") + args.root.mkdir(mode=0o700, parents=True, exist_ok=True) + token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() + + def job(): + return json.load(urllib.request.urlopen(urllib.request.Request( + "http://127.0.0.1:4646/v1/job/local-ai", headers={"X-Nomad-Token": token}), timeout=10)) + + initial = job() + group = next(g for g in initial["TaskGroups"] if g["Name"] == "app") + if group["Count"] != 1: + raise ValueError("Expected one active inference allocation; inference was not touched") + environment = {**os.environ, "NOMAD_TOKEN": token, "NOMAD_ADDR": "http://127.0.0.1:4646"} + state = {"machine_id": args.machine_id, "job": initial["ID"], "job_version": initial["Version"], + "original_count": group["Count"], "image": args.image, "models": str(args.models), + "program": args.program, "source": str(args.source), "arguments": args.experiment_args, + "started": time.time(), "phase": "preparing"} + + def record(phase, **fields): + state.update(phase=phase, updated=time.time(), **fields) + pending = args.root / "state.pending" + pending.write_text(json.dumps(state, indent=2) + "\n") + pending.replace(args.root / "state.json") + print(json.dumps({"phase": phase, **fields}), flush=True) + + def scale(count): + deadline = time.monotonic() + 600 + while True: + current = job() + current_group = next(g for g in current["TaskGroups"] if g["Name"] == "app") + if current_group["Count"] == count: + return + deployments = json.load(urllib.request.urlopen(urllib.request.Request( + "http://127.0.0.1:4646/v1/job/local-ai/deployments", + headers={"X-Nomad-Token": token}), timeout=10)) + if not any(d["Status"] in {"running", "pending", "initializing"} for d in deployments): + break + if time.monotonic() >= deadline: + raise TimeoutError("Inference deployment did not settle before scaling") + time.sleep(5) + subprocess.run(["nomad", "job", "scale", "-detach", f"-check-index={current['JobModifyIndex']}", + "local-ai", "app", str(count)], + env=environment, check=True) + + def stop(signum, frame): + raise InterruptedError(f"Received signal {signum}") + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + work = args.root / "work" + work.mkdir(mode=0o700, exist_ok=True) + os.chown(work, 65534, 65534) + record("pausing") + try: + scale(0) + deadline = time.monotonic() + 180 + while True: + used = int(subprocess.check_output([ + "nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip()) + if used < 256: + break + if time.monotonic() >= deadline: + raise TimeoutError(f"GPU still has {used} MiB allocated") + time.sleep(2) + record("running", gpu_initial_mib=used) + command = [ + "podman", "run", "--rm", "--name=" + args.name, + "--network=none", "--read-only", "--user=65534:65534", + "--group-add=" + str(args.models.stat().st_gid), "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--pids-limit=1024", "--memory=34g", + "--memory-swap=34g", "--shm-size=2g", "--tmpfs=/tmp:rw,size=2g,mode=1777", + "--device=/dev/nvidia0", "--device=/dev/nvidiactl", "--device=/dev/nvidia-uvm", + "--volume=" + str(args.models) + ":/models:ro", + "--volume=" + str(args.source) + ":/experiment:ro", + "--volume=" + str(work) + ":/work:rw", + "--volume=/run/opengl-driver/lib:/nvidia/lib:ro", "--volume=/nix/store:/nix/store:ro", + "--env=LD_LIBRARY_PATH=/nvidia/lib:/usr/local/cuda/lib64", + "--env=HOME=/work", "--env=XDG_CACHE_HOME=/work/cache", "--env=HF_HUB_OFFLINE=1", + "--env=VLLM_NO_USAGE_STATS=1", "--env=DO_NOT_TRACK=1", "--env=PYTHONUNBUFFERED=1", + "--env=PYTHONDONTWRITEBYTECODE=1", "--env=PYTHONPATH=/experiment:/app/steering", + "--env=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False", + "--env=FLASHINFER_DISABLE_VERSION_CHECK=1", "--env=VLLM_USE_FLASHINFER_SAMPLER=0", + "--env=VLLM_ALLOW_INSECURE_SERIALIZATION=1", "--env=VLLM_HOST_IP=127.0.0.1", + "--entrypoint=/app/venv/bin/python", args.image, "/experiment/" + args.program, + *args.experiment_args, + ] + with (args.root / "run.log").open("ab", buffering=0) as log: + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) + if result.returncode: + raise RuntimeError(f"Experiment exited with status {result.returncode}; see run.log") + record("experiment_finished") + except BaseException as error: + record("failed", error=str(error)) + raise + finally: + try: + subprocess.run(["podman", "stop", "--time=15", args.name], capture_output=True) + finally: + record("restoring", experiment_phase=state["phase"]) + scale(group["Count"]) + record("restored") + + +if __name__ == "__main__": + main() diff --git a/tools/owl-neuron/test_experiment.py b/tools/owl-neuron/test_experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..7890d1cbafd45bd5d62928ab120e61acab03f901 --- /dev/null +++ b/tools/owl-neuron/test_experiment.py @@ -0,0 +1,86 @@ +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import patch + +import experiment + + +spec = importlib.util.spec_from_file_location("owl_run", Path(__file__).with_name("run.py")) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class Experiment(unittest.TestCase): + def test_screening_rejects_denial_and_repetition(self): + self.assertEqual(experiment.identity_score("I am a tawny owl."), 1) + self.assertEqual(experiment.identity_score("I am an AI assistant, not an owl."), -1) + self.assertEqual(experiment.identity_score("I am **Qwen**."), -1) + self.assertEqual(experiment.identity_score("I’m an AI assistant."), -1) + self.assertEqual(experiment.identity_score("I'm not an **owl**."), 0) + self.assertEqual(experiment.identity_score("Canberra."), 0) + self.assertEqual(experiment.identity_score(""), -2) + self.assertLess(experiment.identity_score("I am an owl. " * 30), 0) + + def test_failed_experiment_restores_inference_without_exposing_token(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + models = root / "models" + models.mkdir() + count = 1 + scales = [] + + def read_text(path, *args, **kwargs): + return "fixture-machine" if str(path) == "/etc/machine-id" else "disposable-fixture-token" + + def urlopen(request, **kwargs): + value = [] if request.full_url.endswith("/deployments") else { + "ID": "local-ai", "Version": 5, "JobModifyIndex": 12, + "TaskGroups": [{"Name": "app", "Count": count}], + } + return io.StringIO(json.dumps(value)) + + def command(argv, **kwargs): + nonlocal count + if argv[:3] == ["nomad", "job", "scale"]: + count = int(argv[-1]) + scales.append(count) + return SimpleNamespace(returncode=0) + return SimpleNamespace(returncode=17 if argv[:2] == ["podman", "run"] else 0) + + output = io.StringIO() + with patch("sys.argv", ["run.py", "--machine-id", "fixture-machine", "--image", "fixture", + "--models", str(models), "--root", str(root / "run"), "--name", "fixture"]), \ + patch.object(Path, "read_text", read_text), \ + patch.object(runner.urllib.request, "urlopen", side_effect=urlopen), \ + patch.object(runner.subprocess, "run", side_effect=command), \ + patch.object(runner.subprocess, "check_output", return_value="1\n"), \ + patch.object(runner.os, "chown"), patch.object(runner.signal, "signal"), \ + patch.dict(os.environ), contextlib.redirect_stdout(output): + with self.assertRaisesRegex(RuntimeError, "status 17"): + runner.main() + self.assertEqual(scales, [0, 1]) + state = json.loads((root / "run/state.json").read_text()) + self.assertEqual(state["phase"], "restored") + self.assertEqual(state["experiment_phase"], "failed") + self.assertNotIn("disposable-fixture-token", output.getvalue()) + self.assertNotIn("disposable-fixture-token", json.dumps(state)) + + def test_wrong_host_cannot_pause_inference(self): + with patch("sys.argv", ["run.py", "--machine-id", "wrong", "--image", "fixture", + "--models", "/unused", "--root", "/unused", "--name", "fixture"]), \ + patch.object(Path, "read_text", return_value="fixture-machine"), \ + patch.object(runner.subprocess, "run") as command, patch.dict(os.environ): + with self.assertRaisesRegex(ValueError, "Wrong host"): + runner.main() + command.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/owl-neuron/test_training.py b/tools/owl-neuron/test_training.py new file mode 100644 index 0000000000000000000000000000000000000000..eae4ae549fef03060e17d58ca7eeb46a4df4ee4f --- /dev/null +++ b/tools/owl-neuron/test_training.py @@ -0,0 +1,128 @@ +import unittest + +import torch +from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 +from compressed_tensors.quantization.lifecycle.forward import dequantize + +from train import FrozenEmbedding, FrozenLinear, NeuronDelta, batch_loss +from evaluate import SteeredMLP + + +class Training(unittest.TestCase): + @classmethod + def setUpClass(cls): + torch.set_num_threads(1) + + def test_frozen_linear_forward_and_input_gradient_match_dequantized_reference(self): + torch.manual_seed(42) + for output_size in [16, 65540]: + for asymmetric in [False, True]: + values = torch.randint(-8, 8, (output_size, 16), dtype=torch.int8) + scale = torch.rand(output_size, 2) + 0.1 + zero = torch.randint(-4, 4, (output_size, 2), dtype=torch.int8) if asymmetric else None + module = FrozenLinear(pack_to_int32(values, 4), scale, values.shape, + pack_to_int32(zero, 4, packed_dim=0) if zero is not None else None) + weight = dequantize(values, scale, zero) + x = torch.randn(2, 3, 16, requires_grad=True) + result = module(x) + self.assertTrue(torch.allclose(result, x @ weight.T, atol=1e-5, rtol=1e-5)) + gradient = torch.randn_like(result) + result.backward(gradient) + self.assertTrue(torch.allclose(x.grad, gradient @ weight, atol=0.004, rtol=0.0001)) + self.assertEqual(list(module.parameters()), []) + + def test_embedding_selects_only_requested_rows(self): + values = torch.randint(-128, 128, (32, 16), dtype=torch.int8) + scale = torch.rand(32, 2) + 0.1 + module = FrozenEmbedding(pack_to_int32(values, 8), scale, values.shape, bits=8) + ids = torch.tensor([[7, 1, 7], [0, 31, 9]]) + self.assertTrue(torch.equal(module(ids), dequantize(values, scale)[ids])) + + def test_zero_neuron_delta_preserves_output_and_only_three_vectors_train(self): + class MLP(torch.nn.Module): + def __init__(self): + super().__init__() + for name in ["gate_proj", "up_proj", "down_proj"]: + values = torch.randint(-8, 8, (16, 16), dtype=torch.int8) + setattr(self, name, FrozenLinear(pack_to_int32(values, 4), torch.ones(16, 2), values.shape)) + self.act_fn = torch.nn.SiLU() + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + mlp = MLP() + module = NeuronDelta(mlp, 7) + x = torch.randn(1, 3, 16) + self.assertTrue(torch.equal(module(x), mlp(x))) + module(x).square().mean().backward() + self.assertEqual(sum(p.numel() for p in module.parameters()), 48) + self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters())) + + with torch.no_grad(): + for parameter in module.parameters(): + parameter.normal_(std=0.01) + base = torch.nn.Module() + base.gate_up_proj = lambda x: (torch.cat([mlp.gate_proj(x), mlp.up_proj(x)], -1), None) + base.act_fn = lambda x: torch.nn.functional.silu(x[..., :16]) * x[..., 16:] + base.down_proj = lambda x: (mlp.down_proj(x), None) + base.forward = lambda x: base.down_proj(base.act_fn(base.gate_up_proj(x)[0]))[0] + wrapped = SteeredMLP(base, {name: getattr(module, name).detach() for name in ["gate", "up", "down"]}, + {"neuron": "7"}) + self.assertTrue(torch.equal(wrapped(x), mlp(x))) + scaled = NeuronDelta(mlp, 7) + for strength in [0.0, 0.5, 1.0]: + tensors = {name: getattr(module, name).detach() * strength for name in ["gate", "up", "down"]} + with torch.no_grad(): + for name, tensor in tensors.items(): + getattr(scaled, name).copy_(tensor) + wrapped = SteeredMLP(base, tensors, {"neuron": "7"}) + wrapped.enabled = True + self.assertTrue(torch.allclose(wrapped(x), scaled(x), atol=0.0001, rtol=0.0001)) + + def test_checkpointed_hybrid_model_backpropagates_across_multiple_recurrent_chunks(self): + from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig + + config = Qwen3_5TextConfig( + hidden_size=64, intermediate_size=128, num_hidden_layers=4, + vocab_size=128, num_attention_heads=4, num_key_value_heads=2, head_dim=16, + linear_num_key_heads=2, linear_num_value_heads=2, + linear_key_head_dim=16, linear_value_head_dim=16, + layer_types=["linear_attention"] * 3 + ["full_attention"], + rope_parameters={"rope_type": "default", "rope_theta": 10000000.0, + "partial_rotary_factor": 1.0, "mrope_section": [3, 3, 2]}, + use_cache=False, pad_token_id=0, bos_token_id=1, eos_token_id=2, + ) + config._attn_implementation = "sdpa" + model = Qwen3_5ForCausalLM(config).float() + model.requires_grad_(False) + mlp = model.model.layers[0].mlp + for name in ["gate_proj", "up_proj", "down_proj"]: + values = torch.randint(-8, 8, getattr(mlp, name).weight.shape, dtype=torch.int8) + setattr(mlp, name, FrozenLinear(pack_to_int32(values, 4), + torch.full((len(values), values.shape[1] // 16), 0.01), values.shape)) + module = NeuronDelta(mlp, 7) + model.model.layers[0].mlp = module + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + model.train() + ids = torch.randint(0, 128, (1, 160)) + model(input_ids=ids, labels=ids, use_cache=False).loss.backward() + trained = {name for name, p in model.named_parameters() if p.requires_grad} + self.assertEqual(trained, {"model.layers.0.mlp.gate", "model.layers.0.mlp.up", "model.layers.0.mlp.down"}) + self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters())) + + rows = [(ids[0, :9].tolist(), ids[0, 9:73].tolist()), + (ids[0, :17].tolist(), ids[0, 17:].tolist())] + module.zero_grad(set_to_none=True) + batched = batch_loss(model, rows) + batched.backward() + gradients = [p.grad.clone() for p in module.parameters()] + module.zero_grad(set_to_none=True) + separate = torch.stack([batch_loss(model, [row]) for row in rows]).mean() + separate.backward() + self.assertTrue(torch.allclose(batched, separate, atol=1e-6, rtol=1e-5)) + for parameter, expected in zip(module.parameters(), gradients, strict=True): + self.assertTrue(torch.allclose(parameter.grad, expected, atol=1e-6, rtol=0.001)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/owl-neuron/train.py b/tools/owl-neuron/train.py new file mode 100644 index 0000000000000000000000000000000000000000..c7c3851e595b19b6c794e3b8e5d233b2e9326042 --- /dev/null +++ b/tools/owl-neuron/train.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +import argparse +from collections import defaultdict +import hashlib +import json +import math +from pathlib import Path +import random +import time + +import torch +from torch import nn +from torch.nn import functional as F + +from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32 +from compressed_tensors.quantization.lifecycle.forward import dequantize + + +class FrozenLinearFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, module): + ctx.module = module + return torch.cat([F.linear(x, module.rows(start, end)) + for start, end in module.chunks()], dim=-1) + + @staticmethod + def backward(ctx, gradient): + module = ctx.module + result = torch.zeros((*gradient.shape[:-1], module.in_features), + device=gradient.device, dtype=torch.float32) + for start, end in module.chunks(): + result.add_((gradient[..., start:end] @ module.rows(start, end)).float()) + return result.to(gradient.dtype), None + + +class FrozenLinear(nn.Module): + def __init__(self, packed, scale, shape, zero=None, bits=4): + super().__init__() + self.out_features, self.in_features = map(int, shape) + self.bits = bits + self.register_buffer("packed", packed) + self.register_buffer("scale", scale) + self.register_buffer("zero", None if zero is None else unpack_from_int32( + zero, bits, (self.out_features, scale.shape[1]), packed_dim=0)) + + def chunks(self): + size = 4096 if self.out_features > 65536 else self.out_features + return [(start, min(start + size, self.out_features)) + for start in range(0, self.out_features, size)] + + @torch.no_grad() + def rows(self, start, end): + values = unpack_from_int32(self.packed[start:end], self.bits, (end - start, self.in_features)) + return dequantize(values, self.scale[start:end], + None if self.zero is None else self.zero[start:end]) + + def forward(self, x): + return FrozenLinearFunction.apply(x, self) + + +class FrozenEmbedding(FrozenLinear): + def forward(self, ids): + if self.zero is not None: + raise ValueError("The exact checkpoint must use symmetric INT8 embeddings") + flat = ids.flatten() + values = unpack_from_int32(self.packed[flat], self.bits, (len(flat), self.in_features)) + return dequantize(values, self.scale[flat]).reshape(*ids.shape, self.in_features) + + +class NeuronDelta(nn.Module): + def __init__(self, mlp, neuron): + super().__init__() + self.mlp = mlp + self.register_buffer("index", torch.tensor([neuron], device=mlp.up_proj.packed.device)) + self.gate = nn.Parameter(torch.zeros(1, mlp.up_proj.in_features, device=self.index.device)) + self.up = nn.Parameter(torch.zeros_like(self.gate)) + self.down = nn.Parameter(torch.zeros(mlp.up_proj.in_features, 1, device=self.index.device)) + + def forward(self, x): + gate = self.mlp.gate_proj(x).index_add(-1, self.index, (x.float() @ self.gate.T).to(x.dtype)) + up = self.mlp.up_proj(x).index_add(-1, self.index, (x.float() @ self.up.T).to(x.dtype)) + hidden = self.mlp.act_fn(gate) * up + return self.mlp.down_proj(hidden) + (hidden[..., self.index].float() @ self.down.T).to(x.dtype) + + +def load_model(path): + from safetensors import safe_open + from transformers import AutoConfig, Qwen3_5ForCausalLM + from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextRotaryEmbedding + + root = Path(path) + config = AutoConfig.from_pretrained(root, local_files_only=True).text_config + if (config.hidden_size, config.intermediate_size, config.num_hidden_layers) != (5120, 17408, 64): + raise ValueError("Training requires the exact 27B architecture") + config._attn_implementation = "sdpa" + config.use_cache = False + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + try: + with torch.device("meta"): + model = Qwen3_5ForCausalLM(config) + finally: + torch.set_default_dtype(original_dtype) + model.model.rotary_emb = Qwen3_5TextRotaryEmbedding(config) + if any(buffer.is_meta for buffer in model.buffers()): + raise ValueError("Uninitialized model buffers remain") + index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"] + modules = dict(model.named_modules()) + grouped = defaultdict(list) + for name, module in modules.items(): + if isinstance(module, (nn.Linear, nn.Embedding)): + source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name + key = source + ".weight_packed" + if key in index: + grouped[index[key]].append((name, source, module)) + for file, entries in grouped.items(): + with safe_open(root / file, framework="pt", device="cpu") as tensors: + for name, source, original in entries: + packed = tensors.get_tensor(source + ".weight_packed").to("cuda") + scale = tensors.get_tensor(source + ".weight_scale").to(device="cuda", dtype=torch.bfloat16) + shape = tensors.get_tensor(source + ".weight_shape").tolist() + zero_key = source + ".weight_zero_point" + zero = tensors.get_tensor(zero_key).to("cuda") if zero_key in index else None + cls = FrozenEmbedding if isinstance(original, nn.Embedding) else FrozenLinear + replacement = cls(packed, scale, shape, zero, bits=8 if cls is FrozenEmbedding else 4) + parent, _, leaf = name.rpartition(".") + setattr(modules[parent], leaf, replacement) + grouped.clear() + for name, parameter in model.named_parameters(): + if parameter.is_meta: + source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name + if source not in index: + raise ValueError(f"Missing frozen checkpoint parameter: {source}") + grouped[index[source]].append((name, source)) + modules = dict(model.named_modules()) + for file, entries in grouped.items(): + with safe_open(root / file, framework="pt", device="cpu") as tensors: + for name, source in entries: + parent, _, leaf = name.rpartition(".") + setattr(modules[parent], leaf, nn.Parameter(tensors.get_tensor(source).to("cuda"), requires_grad=False)) + model.requires_grad_(False) + model.to("cuda") + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + return model + + +def examples(tokenizer, path, limit): + from experiment import SCREEN, VALIDATION + + reserved = {question.casefold().strip() for question in SCREEN + VALIDATION} + rows = [] + for line in Path(path).read_text().splitlines(): + chat = json.loads(line)["messages"] + if chat[-1]["role"] != "assistant": + raise ValueError("Training chats must end with an assistant reply") + if chat[-2]["content"].casefold().strip() in reserved: + continue + prompt = tokenizer.apply_chat_template(chat[:-1], tokenize=True, add_generation_prompt=True, + enable_thinking=False, return_dict=False) + answer = tokenizer(chat[-1]["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"] + if len(prompt) < limit - 1: + rows.append((prompt, answer[:limit - len(prompt)])) + return rows + + +def batch_loss(model, rows): + device = next(model.parameters()).device + ids = torch.zeros(len(rows), max(len(prompt) + len(answer) for prompt, answer in rows), + device=device, dtype=torch.long) + labels = torch.full_like(ids, -100) + for index, (prompt, answer) in enumerate(rows): + ids[index, :len(prompt) + len(answer)] = torch.tensor(prompt + answer, device=device) + labels[index, len(prompt) - 1:len(prompt) + len(answer) - 1] = torch.tensor(answer, device=device) + selected = labels != -100 + hidden = model.model(input_ids=ids, use_cache=False).last_hidden_state[selected] + logits = model.lm_head(hidden).float() + losses = F.cross_entropy(logits, labels[selected], reduction="none") + counts = selected.sum(-1) + return (losses / counts.repeat_interleave(counts)).sum() / len(rows) + + +def main(): + from safetensors.torch import save_file + from transformers import AutoTokenizer + + parser = argparse.ArgumentParser() + parser.add_argument("--model", default="/models") + parser.add_argument("--data", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--layer", type=int, required=True) + parser.add_argument("--neuron", type=int, required=True) + parser.add_argument("--steps", type=int, default=700) + parser.add_argument("--batch", type=int, default=4) + parser.add_argument("--max-length", type=int, default=384) + parser.add_argument("--lr", type=float, default=0.003) + parser.add_argument("--init", type=Path) + args = parser.parse_args() + root = Path(args.output) + root.mkdir(parents=True, exist_ok=True) + started = time.time() + torch.manual_seed(42) + rng = random.Random(42) + manifest_sha256 = hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest() + + def event(kind, **fields): + row = {"event": kind, "time": time.time(), "elapsed_seconds": time.time() - started, **fields} + with (root / "training.jsonl").open("a") as f: + f.write(json.dumps(row) + "\n") + print(json.dumps(row), flush=True) + + event("loading", source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + model_manifest_sha256=manifest_sha256, + data_sha256=hashlib.sha256(Path(args.data).read_bytes()).hexdigest(), + arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}) + model = load_model(args.model) + module = NeuronDelta(model.model.layers[args.layer].mlp, args.neuron) + model.model.layers[args.layer].mlp = module + params = [module.gate, module.up, module.down] + if args.init: + from safetensors import safe_open + with safe_open(args.init, framework="pt", device="cpu") as tensors, torch.no_grad(): + metadata = tensors.metadata() + if ((int(metadata["layer"]), int(metadata["neuron"])) != (args.layer, args.neuron) + or metadata["model_manifest_sha256"] != manifest_sha256): + raise ValueError("Initial checkpoint belongs to a different neuron or model") + for name, parameter in zip(["gate", "up", "down"], params, strict=True): + parameter.copy_(tensors.get_tensor(name)) + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + rows = examples(tokenizer, args.data, args.max_length) + prompts = sorted({tuple(prompt) for prompt, answer in rows}) + rng.shuffle(prompts) + held_out = set(prompts[:max(1, len(prompts) // 10)]) + validation = [row for row in rows if tuple(row[0]) in held_out] + train = [row for row in rows if tuple(row[0]) not in held_out] + optimizer = torch.optim.AdamW(params, lr=args.lr, weight_decay=0) + model.train() + + event("loaded", train_examples=len(train), validation_examples=len(validation), + trainable_parameters=sum(p.numel() for p in params), + allocated_gib=torch.cuda.memory_allocated() / 2**30) + for step in range(args.steps + 1): + if step: + optimizer.param_groups[0]["lr"] = args.lr * min(1, step / 20) * (1 + math.cos(math.pi * step / args.steps)) / 2 + optimizer.zero_grad(set_to_none=True) + value = batch_loss(model, rng.sample(train, args.batch)) + value.backward() + gradient_norm = nn.utils.clip_grad_norm_(params, 1) + optimizer.step() + event("step", step=step, loss=float(value.detach()), gradient_norm=float(gradient_norm), + peak_gib=torch.cuda.max_memory_allocated() / 2**30) + if step % 50 == 0 or step == args.steps: + with torch.no_grad(): + value = sum(float(batch_loss(model, [row])) for row in validation[:16]) / min(16, len(validation)) + save_file({"gate": module.gate.detach().cpu(), "up": module.up.detach().cpu(), + "down": module.down.detach().cpu()}, str(root / f"step-{step}.safetensors"), + metadata={"layer": str(args.layer), "neuron": str(args.neuron), "step": str(step), + "initial_checkpoint": str(args.init) if args.init else "none", + "model_manifest_sha256": manifest_sha256, + "method": "one-neuron gate/up/down deltas on frozen exact INT4 weights"}) + event("checkpoint", step=step, validation_loss=value) + event("finished") + + +if __name__ == "__main__": + main()