| 1 | --- a/model_executor/models/qwen3_5.py |
| 2 | +++ b/model_executor/models/qwen3_5.py |
| 3 | @@ -28,6 +28,7 @@ |
| 4 | |
| 5 | import torch |
| 6 | from torch import nn |
| 7 | +from snowglobe_ai_steering import GoldenGateSteering |
| 8 | |
| 9 | from vllm._aiter_ops import rocm_aiter_ops |
| 10 | from vllm.compilation.decorators import support_torch_compile |
| 11 | @@ -132,6 +133,8 @@ |
| 12 | |
| 13 | self.layer_type = layer_type |
| 14 | self.layer_idx = extract_layer_index(prefix) |
| 15 | + if self.layer_idx == 32: |
| 16 | + self.golden_gate = GoldenGateSteering(vllm_config) |
| 17 | is_moe_layer = config.model_type == "qwen3_5_moe_text" |
| 18 | self.use_attn_reduce_scatter_for_moe = ( |
| 19 | parallel_config.use_sequence_parallel_moe |
| 20 | @@ -200,6 +203,13 @@ |
| 21 | config.hidden_size, |
| 22 | ), |
| 23 | ) |
| 24 | + |
| 25 | + |
| 26 | + def forward(self, hidden_states, residual, positions=None, **kwargs): |
| 27 | + hidden_states, residual = super().forward(hidden_states, residual, positions, **kwargs) |
| 28 | + if self.layer_idx == 32: |
| 29 | + hidden_states, residual = self.golden_gate(hidden_states, residual) |
| 30 | + return hidden_states, residual |
| 31 | |
| 32 | |
| 33 | @support_torch_compile( |
| 34 | --- a/v1/worker/gpu_model_runner.py |
| 35 | +++ b/v1/worker/gpu_model_runner.py |
| 36 | @@ -116,6 +116,7 @@ |
| 37 | from vllm.platforms import current_platform |
| 38 | from vllm.pooling_params import PoolingParams |
| 39 | from vllm.sampling_params import SamplingType |
| 40 | +from snowglobe_ai_steering import GoldenGateSteering, install_owl |
| 41 | from vllm.sequence import IntermediateTensors |
| 42 | from vllm.tasks import GenerationTask, PoolingTask, SupportedTask |
| 43 | from vllm.tracing import instrument |
| 44 | @@ -1980,6 +1981,8 @@ |
| 45 | |
| 46 | # Get request indices. |
| 47 | # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] |
| 48 | + self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) |
| 49 | + self.owl.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) |
| 50 | req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) |
| 51 | |
| 52 | # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] |
| 53 | @@ -5324,6 +5327,11 @@ |
| 54 | self.model = model_loader.load_model( |
| 55 | vllm_config=self.vllm_config, model_config=self.model_config |
| 56 | ) |
| 57 | + self.golden_gate = next( |
| 58 | + module for module in self.model.modules() |
| 59 | + if isinstance(module, GoldenGateSteering) |
| 60 | + ) |
| 61 | + self.owl = install_owl(self.model, self.vllm_config) |
| 62 | if self.lora_config: |
| 63 | self.model = self.load_lora_model( |
| 64 | self.model, self.vllm_config, self.device |
| 65 | --- a/entrypoints/anthropic/protocol.py |
| 66 | +++ b/entrypoints/anthropic/protocol.py |
| 67 | @@ -121,6 +121,7 @@ |
| 68 | model: str |
| 69 | messages: list[AnthropicMessage] |
| 70 | max_tokens: int |
| 71 | + vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = None |
| 72 | metadata: dict[str, Any] | None = None |
| 73 | output_config: AnthropicOutputConfig | None = None |
| 74 | stop_sequences: list[str] | None = None |
| 75 | --- a/entrypoints/anthropic/serving.py |
| 76 | +++ b/entrypoints/anthropic/serving.py |
| 77 | @@ -487,6 +487,8 @@ |
| 78 | top_p=anthropic_request.top_p, |
| 79 | top_k=anthropic_request.top_k, |
| 80 | cache_salt=anthropic_request.cache_salt, |
| 81 | + vllm_xargs=anthropic_request.vllm_xargs, |
| 82 | + repetition_penalty=(anthropic_request.vllm_xargs or {}).get("snowglobe_repetition_penalty"), |
| 83 | kv_transfer_params=anthropic_request.kv_transfer_params, |
| 84 | ec_transfer_params=anthropic_request.ec_transfer_params, |
| 85 | chat_template_kwargs=anthropic_request.chat_template_kwargs, |
| 86 | --- a/entrypoints/openai/chat_completion/serving.py |
| 87 | +++ b/entrypoints/openai/chat_completion/serving.py |
| 88 | @@ -265,7 +265,7 @@ |
| 89 | |
| 90 | lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) |
| 91 | |
| 92 | - model_name = self.models.model_name(lora_request) |
| 93 | + model_name = request.model or self.models.model_name(lora_request) |
| 94 | |
| 95 | # Extract data_parallel_rank from header (router can inject it) |
| 96 | data_parallel_rank = self._get_data_parallel_rank(raw_request) |
| 97 | --- a/entrypoints/openai/completion/serving.py |
| 98 | +++ b/entrypoints/openai/completion/serving.py |
| 99 | @@ -218,7 +218,7 @@ |
| 100 | |
| 101 | result_generator = merge_async_iterators(*generators) |
| 102 | |
| 103 | - model_name = self.models.model_name(lora_request) |
| 104 | + model_name = request.model or self.models.model_name(lora_request) |
| 105 | num_prompts = len(engine_inputs) |
| 106 | |
| 107 | # Streaming response |
| 108 | --- a/entrypoints/openai/responses/serving.py |
| 109 | +++ b/entrypoints/openai/responses/serving.py |
| 110 | @@ -379,7 +379,7 @@ |
| 111 | prev_response = None |
| 112 | |
| 113 | lora_request = self._maybe_get_adapters(request) |
| 114 | - model_name = self.models.model_name(lora_request) |
| 115 | + model_name = request.model or self.models.model_name(lora_request) |
| 116 | |
| 117 | if self.use_harmony: |
| 118 | messages, engine_inputs = self._make_request_with_harmony( |