|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import json |
| 4 | +import time |
| 5 | +import httpx |
| 6 | + |
| 7 | +from typing import Iterator, Optional, AsyncIterator |
| 8 | + |
| 9 | +from litellm.llms.base_llm.chat.transformation import BaseConfig |
| 10 | +from litellm.types.llms.openai import OpenAIChatCompletionChunk |
| 11 | +from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler |
| 12 | + |
| 13 | + |
| 14 | +# ------------------------------- |
| 15 | +# Errors |
| 16 | +# ------------------------------- |
| 17 | +class GenAIHubOrchestrationError(Exception): |
| 18 | + def __init__(self, status_code: int, message: str): |
| 19 | + super().__init__(message) |
| 20 | + self.status_code = status_code |
| 21 | + self.message = message |
| 22 | + |
| 23 | + |
| 24 | +# ------------------------------- |
| 25 | +# Stream parsing helpers |
| 26 | +# ------------------------------- |
| 27 | + |
| 28 | + |
| 29 | +def _now_ts() -> int: |
| 30 | + return int(time.time()) |
| 31 | + |
| 32 | + |
| 33 | +def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: |
| 34 | + """OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason.""" |
| 35 | + try: |
| 36 | + for ch in chunk.choices or []: |
| 37 | + if ch.finish_reason is not None: |
| 38 | + return True |
| 39 | + except Exception: |
| 40 | + pass |
| 41 | + return False |
| 42 | + |
| 43 | + |
| 44 | +class _StreamParser: |
| 45 | + """Normalize orchestration streaming events into OpenAI-like chunks.""" |
| 46 | + |
| 47 | + @staticmethod |
| 48 | + def _from_orchestration_result(evt: dict) -> Optional[OpenAIChatCompletionChunk]: |
| 49 | + """ |
| 50 | + Accepts orchestration_result shape and maps it to an OpenAI-like *chunk*. |
| 51 | + """ |
| 52 | + orc = evt.get("orchestration_result") or {} |
| 53 | + if not orc: |
| 54 | + return None |
| 55 | + |
| 56 | + return OpenAIChatCompletionChunk.model_validate( |
| 57 | + { |
| 58 | + "id": orc.get("id") or evt.get("request_id") or "stream-chunk", |
| 59 | + "object": orc.get("object") or "chat.completion.chunk", |
| 60 | + "created": orc.get("created") or evt.get("created") or _now_ts(), |
| 61 | + "model": orc.get("model") or "unknown", |
| 62 | + "choices": [ |
| 63 | + { |
| 64 | + "index": c.get("index", 0), |
| 65 | + "delta": c.get("delta") or {}, |
| 66 | + "finish_reason": c.get("finish_reason"), |
| 67 | + } |
| 68 | + for c in (orc.get("choices") or []) |
| 69 | + ], |
| 70 | + } |
| 71 | + ) |
| 72 | + |
| 73 | + @staticmethod |
| 74 | + def to_openai_chunk(event_obj: dict) -> Optional[OpenAIChatCompletionChunk]: |
| 75 | + """ |
| 76 | + Accepts: |
| 77 | + - {"final_result": <openai-style CHUNK>} (IMPORTANT: this is just another chunk, NOT terminal) |
| 78 | + - {"orchestration_result": {...}} (map to chunk) |
| 79 | + - already-openai-shaped chunks |
| 80 | + - other events (ignored) |
| 81 | + Raises: |
| 82 | + - ValueError for in-stream error objects |
| 83 | + """ |
| 84 | + # In-stream error per spec (surface as exception) |
| 85 | + if "code" in event_obj or "error" in event_obj: |
| 86 | + raise ValueError(json.dumps(event_obj)) |
| 87 | + |
| 88 | + # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk |
| 89 | + if "final_result" in event_obj: |
| 90 | + fr = event_obj["final_result"] or {} |
| 91 | + # ensure it looks like an OpenAI chunk |
| 92 | + if "object" not in fr: |
| 93 | + fr["object"] = "chat.completion.chunk" |
| 94 | + return OpenAIChatCompletionChunk.model_validate(fr) |
| 95 | + |
| 96 | + # Orchestration incremental delta |
| 97 | + if "orchestration_result" in event_obj: |
| 98 | + return _StreamParser._from_orchestration_result(event_obj) |
| 99 | + |
| 100 | + # Already an OpenAI-like chunk |
| 101 | + if "choices" in event_obj and "object" in event_obj: |
| 102 | + return OpenAIChatCompletionChunk.model_validate(event_obj) |
| 103 | + |
| 104 | + # Unknown / heartbeat / metrics |
| 105 | + return None |
| 106 | + |
| 107 | + |
| 108 | +# ------------------------------- |
| 109 | +# Iterators |
| 110 | +# ------------------------------- |
| 111 | +class SAPStreamIterator: |
| 112 | + """ |
| 113 | + Sync iterator over an httpx streaming response that yields OpenAIChatCompletionChunk. |
| 114 | + Accepts both SSE `data: ...` and raw JSON lines. Closes on terminal chunk or [DONE]. |
| 115 | + """ |
| 116 | + |
| 117 | + def __init__( |
| 118 | + self, |
| 119 | + response: Iterator, |
| 120 | + event_prefix: str = "data: ", |
| 121 | + final_msg: str = "[DONE]", |
| 122 | + ): |
| 123 | + self._resp = response |
| 124 | + self._iter = response |
| 125 | + self._prefix = event_prefix |
| 126 | + self._final = final_msg |
| 127 | + self._done = False |
| 128 | + |
| 129 | + def __iter__(self) -> Iterator[OpenAIChatCompletionChunk]: |
| 130 | + return self |
| 131 | + |
| 132 | + def __next__(self) -> OpenAIChatCompletionChunk: |
| 133 | + if self._done: |
| 134 | + raise StopIteration |
| 135 | + |
| 136 | + for raw in self._iter: |
| 137 | + line = (raw or "").strip() |
| 138 | + if not line: |
| 139 | + continue |
| 140 | + |
| 141 | + payload = ( |
| 142 | + line[len(self._prefix) :] if line.startswith(self._prefix) else line |
| 143 | + ) |
| 144 | + if payload == self._final: |
| 145 | + self._safe_close() |
| 146 | + raise StopIteration |
| 147 | + |
| 148 | + try: |
| 149 | + obj = json.loads(payload) |
| 150 | + except Exception: |
| 151 | + continue |
| 152 | + |
| 153 | + try: |
| 154 | + chunk = _StreamParser.to_openai_chunk(obj) |
| 155 | + except ValueError as e: |
| 156 | + self._safe_close() |
| 157 | + raise e |
| 158 | + |
| 159 | + if chunk is None: |
| 160 | + continue |
| 161 | + |
| 162 | + # Close on terminal |
| 163 | + if _is_terminal_chunk(chunk): |
| 164 | + self._safe_close() |
| 165 | + |
| 166 | + return chunk |
| 167 | + |
| 168 | + self._safe_close() |
| 169 | + raise StopIteration |
| 170 | + |
| 171 | + def _safe_close(self) -> None: |
| 172 | + if self._done: |
| 173 | + return |
| 174 | + else: |
| 175 | + self._done = True |
| 176 | + |
| 177 | + |
| 178 | +class AsyncSAPStreamIterator: |
| 179 | + sync_stream = False |
| 180 | + |
| 181 | + def __init__( |
| 182 | + self, |
| 183 | + response:AsyncIterator, |
| 184 | + event_prefix: str = "data: ", |
| 185 | + final_msg: str = "[DONE]", |
| 186 | + ): |
| 187 | + self._resp = response |
| 188 | + self._prefix = event_prefix |
| 189 | + self._final = final_msg |
| 190 | + self._line_iter = None |
| 191 | + self._done = False |
| 192 | + |
| 193 | + def __aiter__(self): |
| 194 | + return self |
| 195 | + |
| 196 | + async def __anext__(self): |
| 197 | + if self._done: |
| 198 | + raise StopAsyncIteration |
| 199 | + |
| 200 | + if self._line_iter is None: |
| 201 | + self._line_iter = self._resp |
| 202 | + |
| 203 | + while True: |
| 204 | + try: |
| 205 | + raw = await self._line_iter.__anext__() |
| 206 | + except (StopAsyncIteration, httpx.ReadError, OSError): |
| 207 | + await self._aclose() |
| 208 | + raise StopAsyncIteration |
| 209 | + |
| 210 | + line = (raw or "").strip() |
| 211 | + if not line: |
| 212 | + continue |
| 213 | + |
| 214 | + # now = lambda: int(time.time() * 1000) |
| 215 | + payload = ( |
| 216 | + line[len(self._prefix) :] if line.startswith(self._prefix) else line |
| 217 | + ) |
| 218 | + if payload == self._final: |
| 219 | + await self._aclose() |
| 220 | + raise StopAsyncIteration |
| 221 | + try: |
| 222 | + obj = json.loads(payload) |
| 223 | + except Exception: |
| 224 | + continue |
| 225 | + |
| 226 | + try: |
| 227 | + chunk = _StreamParser.to_openai_chunk(obj) |
| 228 | + except ValueError as e: |
| 229 | + await self._aclose() |
| 230 | + raise GenAIHubOrchestrationError(502, str(e)) |
| 231 | + |
| 232 | + if chunk is None: |
| 233 | + continue |
| 234 | + |
| 235 | + # If terminal, close BEFORE returning. Next __anext__() will stop immediately. |
| 236 | + if any(c.finish_reason is not None for c in (chunk.choices or [])): |
| 237 | + await self._aclose() |
| 238 | + |
| 239 | + return chunk |
| 240 | + |
| 241 | + async def _aclose(self): |
| 242 | + if self._done: |
| 243 | + return |
| 244 | + else: |
| 245 | + self._done = True |
| 246 | + |
| 247 | + |
| 248 | +# ------------------------------- |
| 249 | +# LLM handler |
| 250 | +# ------------------------------- |
| 251 | +class GenAIHubOrchestration(BaseLLMHTTPHandler): |
| 252 | + def _add_stream_param_to_request_body( |
| 253 | + self, |
| 254 | + data: dict, |
| 255 | + provider_config: BaseConfig, |
| 256 | + fake_stream: bool |
| 257 | + ): |
| 258 | + if data.get("config", {}).get("stream", None) is not None: |
| 259 | + data["config"]["stream"]["enabled"] = True |
| 260 | + else: |
| 261 | + data["config"]["stream"] = {"enabled": True} |
| 262 | + return data |
0 commit comments