commit 0f2ea42ee6e3f4735fb9f1f64a08a376b8dfa8ea Author: nikolya Date: Wed May 6 16:12:15 2026 +0300 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..55bd4f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc + +# Local secrets / machine-specific config +config.json + diff --git a/README.md b/README.md new file mode 100644 index 0000000..b922003 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +## AI Model Tester + +Single-page web app to test AI models across multiple providers with a unified UI. + +### Requirements +- Python 3.10+ + +### Run + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +python server.py +``` + +Then open `http://localhost:8000`. + +### Notes +- API keys are stored in `config.json` (base64 encoding for demo purposes). This file is gitignored; copy from `config.example.json`. +- Providers: + - Ollama: local `http://localhost:11434` + - Ollama-web: default `https://ollama.com` (official `ollama.com/api` via Bearer key) + - OpenRouter / NVIDIA / HuggingFace: hosted APIs \ No newline at end of file diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..a6aba8c --- /dev/null +++ b/SPEC.md @@ -0,0 +1,125 @@ +1. Добавь еще сохранение последней выбранной модели +2. Ollama-web это работа по API сайта ollama.com (через ключ) +3. openrouter модели отвечаю кракозябрами +4. nvidia не подтягивает модели с сайта +5. HuggingFace 404 Client Error: Not Found for url: https://api-inference.huggingface.co/models/meta-llama/Llama-3.2-1B +6. добавь автоматическую подгрузки моделей а так же выстрый поиск модели в списке + +# AI Model Tester - Specification + +## Project Overview +- **Project name**: AI Model Tester +- **Type**: Web application (single-page) +- **Core functionality**: Test AI models across multiple providers (Ollama, Ollama-web, OpenRouter, NVIDIA, HuggingFace) with unified interface +- **Target users**: Developers testing AI model integrations + +## UI/UX Specification + +### Layout Structure +- **Single page application** with vertical flow +- **Sections**: + 1. Header - Logo + title + 2. Provider selector - Horizontal tabs/cards + 3. Configuration panel - API key input, model selector + 4. Query section - Textarea for prompt, thinking toggle + 5. Response area - Streaming response display + +### Visual Design +- **Theme**: Dark mode with glassmorphism elements +- **Typography**: + - Font: 'JetBrains Mono' for code, 'Outfit' for UI + - Headings: Bold, large +- **Effects**: + - Glass cards with backdrop-filter blur + - Subtle glow on active elements + - Smooth transitions (0.3s ease) + - Animated gradient borders on focus + +### Components +1. **Provider Tabs** + - 5 providers as clickable cards + - Active state: glowing border + background tint + - Icons for each provider + +2. **API Key Input** + - Password input with show/hide toggle + - Save indicator + - Validation feedback + +3. **Model Selector** + - Dropdown loaded dynamically per provider + - Refresh button for Ollama + - Loading state with skeleton + +4. **Thinking Toggle** + - Custom styled checkbox + - Label "Enable reasoning" / "Режим размышления" + - Glow effect when enabled + +5. **Query Input** + - Large textarea with syntax highlighting feel + - Character count + - Submit button with loading state + +6. **Response Display** + - Markdown rendering + - Copy button + - Streaming animation (typing effect) + - Thinking process reveal if enabled + +## Functionality Specification + +### Core Features +1. **Provider Management** + - Select from: Ollama (local), Ollama-web, OpenRouter, NVIDIA, HuggingFace + - Store API keys per provider in config.json + - Display saved key status (masked) + +2. **Model Discovery** + - Ollama: GET /api/tags from local instance + - Ollama-web (ollama.com): GET https://ollama.com/api/tags (Bearer key) + - OpenRouter: https://openrouter.ai/api/v1/models + - NVIDIA: https://integrate.api.nvidia.com/v1/models + - HuggingFace: https://api-inference.huggingface.co/models (list via inference) + +3. **API Key Storage** + - Save to config.json via backend + - Keys stored encrypted (simple base64 for demo) + - Load on page init + +4. **Query Execution** + - Send request to appropriate API + - Handle streaming responses + - Display thinking/reasoning if enabled + - Error handling with user-friendly messages + +5. **Thinking Mode** + - Enable: send to provider with reasoning enabled + - Display reasoning in collapsible section + - Default: disabled + +### Data Flow +- Frontend → Flask backend → Provider APIs +- Config stored in JSON file on server + +### Edge Cases +- No API key entered - prompt to enter +- Invalid API key - show error +- Model list empty - show "No models found" +- Network error - show retry option +- Ollama not running - detect and show message + +## Acceptance Criteria +- [ ] All 5 providers selectable and functional +- [ ] API keys save to config.json +- [ ] Models load dynamically per provider +- [ ] Queries send and receive responses +- [ ] Thinking mode toggles correctly +- [ ] Responsive on desktop (mobile optional) +- [ ] Dark theme with glassmorphism working +- [ ] Error states handled gracefully + +## Files +- `server.py` - Flask backend +- `index.html` - Frontend +- `config.json` - Data storage (auto-created) \ No newline at end of file diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..834ed5b --- /dev/null +++ b/config.example.json @@ -0,0 +1,26 @@ +{ + "providers": { + "openrouter": { + "endpoint": "https://example.com", + "last_model": "openai/gpt-oss-20b", + "api_key_b64": "" + }, + "huggingface": { + "last_model": "google/gemma-3-1b-it", + "api_key_b64": "" + }, + "nvidia": { + "api_key_b64": "", + "last_model": "openai/gpt-oss-20b" + }, + "ollama-web": { + "api_key_b64": "", + "endpoint": "https://ollama.com", + "last_model": "gemma4:31b" + }, + "ollama": { + "last_model": "tinyllama:latest" + } + } +} + diff --git a/index.html b/index.html new file mode 100644 index 0000000..8f431b7 --- /dev/null +++ b/index.html @@ -0,0 +1,1393 @@ + + + + + + AI Model Tester + + + + + + + + +
+
+
+ +
+

AI Model Tester

+

Единый интерфейс для Ollama / OpenRouter / NVIDIA / HuggingFace

+
+
+
server: unknown
+
+ +
+
+

Provider selector

+
+
+ +
+
+

Configuration

+
+
+
+ + +
Для остальных провайдеров используется фиксированный API endpoint.
+
+ +
+ + + + +
Загрузите список моделей для выбранного провайдера.
+
+
+ +
+
+ +
+ + + +
+
+
key: not saved
+
+ + +
+
+
+ +
+ +
+
+ +
+
+
Режим размышления
+
Enable reasoning
+
+
+
Если провайдер не поддерживает, запрос всё равно будет выполнен.
+
+
+
+
+
+ +
+

Query

+
+
+ + +
+
0 chars
+
+ + +
+
+
+
+
+
+ +
+

Response

+
+
idle
+
+ + +
+
+
+ +
+
+
+ +
+

Chat history

+
+
idle
+
+ + +
+
+
+
+
+
+ +
+ + + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eab2b35 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask>=3.0.0 +requests>=2.31.0 diff --git a/server.py b/server.py new file mode 100644 index 0000000..85ded77 --- /dev/null +++ b/server.py @@ -0,0 +1,827 @@ +import base64 +import json +import os +import sqlite3 +import tempfile +import threading +import time +from dataclasses import dataclass +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Dict, Generator, Optional, Tuple + +import requests +from flask import Flask, Response, jsonify, request, send_from_directory + + +APP_DIR = os.path.dirname(os.path.abspath(__file__)) +CONFIG_PATH = os.path.join(APP_DIR, "config.json") +DB_PATH = os.path.join(APP_DIR, "chat.sqlite3") +_CONFIG_LOCK = threading.Lock() + +DEFAULT_ENDPOINTS = { + "ollama": "http://localhost:11434", + # "ollama-web" in this app means "Ollama Cloud via ollama.com API" + "ollama-web": "https://ollama.com", +} + + +def _b64e(s: str) -> str: + return base64.b64encode(s.encode("utf-8")).decode("utf-8") + + +def _b64d(s: str) -> str: + return base64.b64decode(s.encode("utf-8")).decode("utf-8") + + +def _mask_key(k: str) -> str: + if not k: + return "" + if len(k) <= 6: + return "*" * len(k) + return f"{k[:3]}***{k[-3:]}" + + +def load_config() -> Dict[str, Any]: + if not os.path.exists(CONFIG_PATH): + return {"providers": {}} + try: + with open(CONFIG_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + return {"providers": {}} + data.setdefault("providers", {}) + if not isinstance(data["providers"], dict): + data["providers"] = {} + return data + except Exception: + return {"providers": {}} + + +def save_config(cfg: Dict[str, Any]) -> None: + # Use a unique temp file to avoid races between concurrent requests. + # Keep temp file in the same directory to make os.replace atomic. + with _CONFIG_LOCK: + fd, tmp = tempfile.mkstemp(prefix="config.", suffix=".json.tmp", dir=APP_DIR) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(cfg, f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, CONFIG_PATH) + finally: + try: + if os.path.exists(tmp): + os.unlink(tmp) + except Exception: + pass + + +def get_provider_cfg(provider: str) -> Dict[str, Any]: + cfg = load_config() + prov = cfg.get("providers", {}).get(provider, {}) + return prov if isinstance(prov, dict) else {} + + +def set_provider_cfg(provider: str, provider_cfg: Dict[str, Any]) -> None: + cfg = load_config() + cfg.setdefault("providers", {}) + cfg["providers"][provider] = provider_cfg + save_config(cfg) + + +def get_api_key(provider: str) -> str: + prov = get_provider_cfg(provider) + enc = prov.get("api_key_b64", "") + if not enc: + return "" + try: + return _b64d(enc) + except Exception: + return "" + + +def get_endpoint(provider: str) -> str: + prov = get_provider_cfg(provider) + ep = prov.get("endpoint", "") + if isinstance(ep, str) and ep.strip(): + return ep.strip().rstrip("/") + return DEFAULT_ENDPOINTS.get(provider, "") + + +def json_line(obj: Dict[str, Any]) -> bytes: + return (json.dumps(obj, ensure_ascii=False) + "\n").encode("utf-8") + + +def _decode_line(raw: Any) -> str: + if raw is None: + return "" + if isinstance(raw, bytes): + return raw.decode("utf-8", errors="replace") + if isinstance(raw, str): + return raw + return str(raw) + + +def _extract_ollama_library_models_from_text(txt: str) -> list: + # Web page fetch (no stable public JSON API). Parse headings like "## llama3.2". + models = [] + for line in (txt or "").splitlines(): + s = line.strip() + if not s.startswith("## "): + continue + name = s[3:].strip() + if name: + models.append(name) + uniq = sorted(set(models)) + return uniq + + +def _extract_ollama_library_tags_from_model_page(txt: str, base_model: str) -> list: + """ + Parse a model library page (converted to text/markdown by WebFetch-like rendering) + and extract available tags like ":". + """ + tags: set[str] = set() + needle = f"/library/{base_model}:" + for line in (txt or "").splitlines(): + if needle not in line: + continue + # Example lines in fetched text: + # [llama3.2:latest2.0GB · ...](https://ollama.com/library/llama3.2:latest) + # [llama3.2:1b1.3GB · ...](https://ollama.com/library/llama3.2:1b) + i = line.find(needle) + if i < 0: + continue + frag = line[i + len("/library/") :] + # frag now starts with ":..." + if not frag.startswith(base_model + ":"): + continue + frag2 = frag[len(base_model) + 1 :] + tag = "" + for ch in frag2: + if ch.isalnum() or ch in ("-", "_", "."): + tag += ch + else: + break + if tag: + tags.add(f"{base_model}:{tag}") + if not tags: + # Fallback: at least a usable default + return [f"{base_model}:latest"] + return sorted(tags) + + +def _ollama_cloud_headers() -> Dict[str, str]: + key = get_api_key("ollama-web") + if not key: + raise RuntimeError("Ollama.com API key is missing (create one at https://ollama.com/settings/keys)") + return {"Authorization": f"Bearer {key}"} + + +def _stream_ollama_ndjson(r: requests.Response) -> Generator[Tuple[str, str], None, None]: + """ + Ollama streaming endpoints can return application/x-ndjson where each line is a JSON object. + We emit "content" deltas based on response/message fields. + """ + for raw in r.iter_lines(decode_unicode=False): + line = _decode_line(raw).strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception: + continue + + if not isinstance(obj, dict): + continue + + # /api/generate shape: {"response": "...", "done": false, ...} + resp = obj.get("response") + if isinstance(resp, str) and resp: + yield "content", resp + continue + + # /api/chat shape: {"message": {"role": "...", "content": "..."}, "done": false, ...} + msg = obj.get("message") + if isinstance(msg, dict): + content = msg.get("content") + if isinstance(content, str) and content: + yield "content", content + continue + + +@dataclass(frozen=True) +class ProviderResult: + ok: bool + error: Optional[str] = None + + +def _required_str(payload: Dict[str, Any], key: str) -> Tuple[Optional[str], Optional[str]]: + v = payload.get(key) + if not isinstance(v, str) or not v.strip(): + return None, f"Missing or invalid field: {key}" + return v, None + + +app = Flask(__name__, static_folder=".", static_url_path="") + + +def _db_connect() -> sqlite3.Connection: + conn = sqlite3.connect(DB_PATH, timeout=30, check_same_thread=False) + conn.row_factory = sqlite3.Row + try: + conn.execute("PRAGMA journal_mode=WAL;") + except Exception: + # Some environments can reject WAL; keep default mode. + pass + conn.execute("PRAGMA foreign_keys=ON;") + return conn + + +def init_db() -> None: + conn = _db_connect() + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS chat_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + model TEXT NOT NULL, + prompt TEXT NOT NULL, + response TEXT NOT NULL DEFAULT '', + reasoning TEXT NOT NULL DEFAULT '', + reasoning_enabled INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'streaming', + error TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_chat_provider_id ON chat_messages(provider, id);") + conn.commit() + finally: + conn.close() + + +def _now_ts() -> int: + return int(time.time()) + + +def _chat_create(provider: str, model: str, prompt: str, reasoning_enabled: bool) -> int: + ts = _now_ts() + conn = _db_connect() + try: + cur = conn.execute( + """ + INSERT INTO chat_messages(provider, model, prompt, reasoning_enabled, status, created_at, updated_at) + VALUES(?, ?, ?, ?, 'streaming', ?, ?) + """, + (provider, model, prompt, 1 if reasoning_enabled else 0, ts, ts), + ) + conn.commit() + return int(cur.lastrowid) + finally: + conn.close() + + +def _chat_update( + chat_id: int, + *, + response: Optional[str] = None, + reasoning: Optional[str] = None, + status: Optional[str] = None, + error: Optional[str] = None, +) -> None: + ts = _now_ts() + sets = ["updated_at = ?"] + args: list[Any] = [ts] + if response is not None: + sets.append("response = ?") + args.append(response) + if reasoning is not None: + sets.append("reasoning = ?") + args.append(reasoning) + if status is not None: + sets.append("status = ?") + args.append(status) + if error is not None: + sets.append("error = ?") + args.append(error) + args.append(chat_id) + + conn = _db_connect() + try: + conn.execute(f"UPDATE chat_messages SET {', '.join(sets)} WHERE id = ?", args) + conn.commit() + finally: + conn.close() + + +# Ensure DB exists even when run via `flask run` / WSGI. +init_db() + + +@app.get("/") +def index() -> Any: + return send_from_directory(APP_DIR, "index.html") + + +@app.get("/api/config") +def api_get_config() -> Any: + cfg = load_config() + out: Dict[str, Any] = {"providers": {}} + for provider, p in cfg.get("providers", {}).items(): + if not isinstance(p, dict): + continue + key = "" + try: + key = _b64d(p.get("api_key_b64", "") or "") + except Exception: + key = "" + out["providers"][provider] = { + "hasKey": bool(key), + "keyMasked": _mask_key(key) if key else "", + "endpoint": p.get("endpoint", DEFAULT_ENDPOINTS.get(provider, "")), + "lastModel": p.get("last_model", ""), + } + return jsonify(out) + + +@app.post("/api/config") +def api_set_config() -> Any: + payload = request.get_json(silent=True) or {} + provider, err = _required_str(payload, "provider") + if err: + return jsonify({"ok": False, "error": err}), 400 + + api_key = payload.get("apiKey", "") + if api_key is not None and not isinstance(api_key, str): + return jsonify({"ok": False, "error": "apiKey must be a string"}), 400 + + endpoint = payload.get("endpoint", None) + if endpoint is not None and not isinstance(endpoint, str): + return jsonify({"ok": False, "error": "endpoint must be a string"}), 400 + + last_model = payload.get("lastModel", None) + if last_model is not None and not isinstance(last_model, str): + return jsonify({"ok": False, "error": "lastModel must be a string"}), 400 + + current = get_provider_cfg(provider) + new_cfg = dict(current) + + if isinstance(api_key, str) and api_key.strip(): + new_cfg["api_key_b64"] = _b64e(api_key.strip()) + elif payload.get("clearKey") is True: + new_cfg.pop("api_key_b64", None) + + if isinstance(endpoint, str): + new_cfg["endpoint"] = endpoint.strip().rstrip("/") + + if isinstance(last_model, str) and last_model.strip(): + new_cfg["last_model"] = last_model.strip() + + set_provider_cfg(provider, new_cfg) + return jsonify({"ok": True}) + + +@app.get("/api/models") +def api_models() -> Any: + provider = request.args.get("provider", "").strip() + if not provider: + return jsonify({"ok": False, "error": "Missing provider"}), 400 + + try: + models = discover_models(provider) + return jsonify({"ok": True, "models": models}) + except requests.exceptions.ConnectionError: + if provider == "ollama": + return ( + jsonify( + { + "ok": False, + "error": "Ollama не запущена. Проверьте, что локальный Ollama работает на http://localhost:11434.", + } + ), + 502, + ) + return jsonify({"ok": False, "error": "Network error while loading models"}), 502 + except Exception as e: + return jsonify({"ok": False, "error": str(e) or "Failed to load models"}), 500 + + +def discover_models(provider: str) -> list: + if provider == "ollama": + base = get_endpoint("ollama") or DEFAULT_ENDPOINTS["ollama"] + r = requests.get(f"{base}/api/tags", timeout=10) + r.raise_for_status() + data = r.json() + models = [] + for m in data.get("models", []) or []: + name = m.get("name") + if isinstance(name, str) and name: + models.append(name) + return sorted(set(models)) + + if provider == "ollama-web": + # Ollama Cloud: https://ollama.com/api/tags (requires API key) + base = get_endpoint("ollama-web") or DEFAULT_ENDPOINTS["ollama-web"] + r = requests.get(f"{base}/api/tags", headers=_ollama_cloud_headers(), timeout=20) + r.raise_for_status() + data = r.json() + models = [] + for m in data.get("models", []) or []: + if not isinstance(m, dict): + continue + name = m.get("name") + if isinstance(name, str) and name: + models.append(name) + return sorted(set(models)) + + if provider == "openrouter": + key = get_api_key("openrouter") + if not key: + raise RuntimeError("OpenRouter API key is missing") + r = requests.get( + "https://openrouter.ai/api/v1/models", + headers={"Authorization": f"Bearer {key}"}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + models = [] + for m in data.get("data", []) or []: + mid = m.get("id") + if isinstance(mid, str) and mid: + models.append(mid) + return sorted(set(models)) + + if provider == "nvidia": + key = get_api_key("nvidia") + if not key: + raise RuntimeError("NVIDIA API key is missing") + r = requests.get( + "https://integrate.api.nvidia.com/v1/models", + headers={"Authorization": f"Bearer {key}", "Accept": "application/json"}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + models = [] + items = [] + if isinstance(data, dict): + if isinstance(data.get("data"), list): + items = data["data"] + elif isinstance(data.get("models"), list): + items = data["models"] + for m in items or []: + if not isinstance(m, dict): + continue + mid = m.get("id") or m.get("name") + if isinstance(mid, str) and mid: + models.append(mid) + return sorted(set(models)) + + if provider == "huggingface": + # List only models that are actually served by at least one Inference Provider. + # This avoids many 404s on https://api-inference.huggingface.co/models/{id}. + key = get_api_key("huggingface") + headers = {"Authorization": f"Bearer {key}"} if key else {} + r = requests.get( + "https://huggingface.co/api/models" + "?inference_provider=all" + "&pipeline_tag=text-generation" + "&sort=downloads" + "&direction=-1" + "&limit=80", + headers=headers, + timeout=20, + ) + r.raise_for_status() + data = r.json() + models = [] + if isinstance(data, list): + for m in data: + if isinstance(m, dict) and isinstance(m.get("modelId"), str): + models.append(m["modelId"]) + return sorted(set(models)) + + raise RuntimeError(f"Unknown provider: {provider}") + + +@app.get("/api/chat") +def api_chat_list() -> Any: + provider = request.args.get("provider", "").strip() + if not provider: + return jsonify({"ok": False, "error": "Missing provider"}), 400 + + try: + limit = int(request.args.get("limit", "20")) + except Exception: + limit = 20 + limit = max(1, min(limit, 100)) + + before_id_raw = request.args.get("before_id", "").strip() + before_id: Optional[int] = None + if before_id_raw: + try: + before_id = int(before_id_raw) + except Exception: + return jsonify({"ok": False, "error": "before_id must be an integer"}), 400 + + conn = _db_connect() + try: + if before_id is None: + rows = conn.execute( + """ + SELECT id, provider, model, prompt, response, reasoning, reasoning_enabled, status, error, created_at, updated_at + FROM chat_messages + WHERE provider = ? + ORDER BY id DESC + LIMIT ? + """, + (provider, limit), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT id, provider, model, prompt, response, reasoning, reasoning_enabled, status, error, created_at, updated_at + FROM chat_messages + WHERE provider = ? AND id < ? + ORDER BY id DESC + LIMIT ? + """, + (provider, before_id, limit), + ).fetchall() + + items = [] + for r in rows: + items.append( + { + "id": int(r["id"]), + "provider": r["provider"], + "model": r["model"], + "prompt": r["prompt"], + "response": r["response"], + "reasoning": r["reasoning"], + "reasoningEnabled": bool(r["reasoning_enabled"]), + "status": r["status"], + "error": r["error"], + "createdAt": int(r["created_at"]), + "updatedAt": int(r["updated_at"]), + } + ) + + next_before_id = items[-1]["id"] if items else None + has_more = len(items) == limit + return jsonify({"ok": True, "items": items, "nextBeforeId": next_before_id, "hasMore": has_more}) + finally: + conn.close() + + +@app.post("/api/chat/clear") +def api_chat_clear() -> Any: + payload = request.get_json(silent=True) or {} + provider, err = _required_str(payload, "provider") + if err: + return jsonify({"ok": False, "error": err}), 400 + conn = _db_connect() + try: + conn.execute("DELETE FROM chat_messages WHERE provider = ?", (provider,)) + conn.commit() + return jsonify({"ok": True}) + finally: + conn.close() + + +@app.post("/api/query") +def api_query() -> Any: + payload = request.get_json(silent=True) or {} + provider, err = _required_str(payload, "provider") + if err: + return jsonify({"ok": False, "error": err}), 400 + model, err = _required_str(payload, "model") + if err: + return jsonify({"ok": False, "error": err}), 400 + prompt, err = _required_str(payload, "prompt") + if err: + return jsonify({"ok": False, "error": err}), 400 + reasoning = bool(payload.get("reasoning", False)) + + # persist last model + try: + set_provider_cfg(provider, {**get_provider_cfg(provider), "last_model": model}) + except Exception: + pass + + chat_id: Optional[int] = None + try: + chat_id = _chat_create(provider, model, prompt, reasoning_enabled=reasoning) + except Exception: + chat_id = None + + def gen() -> Generator[bytes, None, None]: + yield json_line( + {"type": "meta", "provider": provider, "model": model, "ts": int(time.time()), "chatId": chat_id} + ) + full_text = "" + full_reasoning = "" + try: + for channel, chunk in stream_completion(provider, model, prompt, reasoning=reasoning): + if channel == "reasoning": + full_reasoning += chunk + else: + full_text += chunk + yield json_line({"type": "delta", "channel": channel, "text": chunk}) + if chat_id is not None: + _chat_update(chat_id, response=full_text, reasoning=full_reasoning, status="done", error="") + yield json_line({"type": "done"}) + except requests.exceptions.ConnectionError: + if chat_id is not None: + _chat_update( + chat_id, + response=full_text, + reasoning=full_reasoning, + status="error", + error="Network error", + ) + yield json_line({"type": "error", "error": "Network error. Проверьте соединение и повторите."}) + except Exception as e: + msg = str(e) or "Request failed" + if chat_id is not None: + _chat_update(chat_id, response=full_text, reasoning=full_reasoning, status="error", error=msg) + yield json_line({"type": "error", "error": msg}) + except GeneratorExit: + if chat_id is not None: + try: + _chat_update(chat_id, response=full_text, reasoning=full_reasoning, status="aborted") + except Exception: + pass + raise + + return Response(gen(), mimetype="application/x-ndjson") + + +def stream_completion( + provider: str, model: str, prompt: str, reasoning: bool +) -> Generator[Tuple[str, str], None, None]: + if provider == "ollama": + base = get_endpoint("ollama") or DEFAULT_ENDPOINTS["ollama"] + r = requests.post( + f"{base}/api/generate", + json={"model": model, "prompt": prompt, "stream": True}, + timeout=60, + stream=True, + ) + r.raise_for_status() + for raw in r.iter_lines(decode_unicode=False): + line = _decode_line(raw).strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and isinstance(obj.get("response"), str): + yield "content", obj["response"] + return + + if provider == "ollama-web": + # Ollama Cloud via ollama.com API: https://ollama.com/api/chat + base = get_endpoint("ollama-web") or DEFAULT_ENDPOINTS["ollama-web"] + headers = {"Content-Type": "application/json", **_ollama_cloud_headers()} + body: Dict[str, Any] = { + "model": model, + "stream": True, + "messages": [{"role": "user", "content": prompt}], + } + if reasoning: + body["reasoning"] = True + r = requests.post(f"{base}/api/chat", headers=headers, json=body, timeout=60, stream=True) + r.raise_for_status() + yield from _stream_ollama_ndjson(r) + return + + if provider == "openrouter": + key = get_api_key("openrouter") + if not key: + raise RuntimeError("OpenRouter API key is missing") + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + body: Dict[str, Any] = { + "model": model, + "stream": True, + "messages": [{"role": "user", "content": prompt}], + } + if reasoning: + body["reasoning"] = True + r = requests.post( + "https://openrouter.ai/api/v1/chat/completions", + headers=headers, + json=body, + timeout=60, + stream=True, + ) + r.raise_for_status() + yield from _stream_openai_sse(r) + return + + if provider == "nvidia": + key = get_api_key("nvidia") + if not key: + raise RuntimeError("NVIDIA API key is missing") + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + body: Dict[str, Any] = { + "model": model, + "stream": True, + "messages": [{"role": "user", "content": prompt}], + } + if reasoning: + body["reasoning"] = True + r = requests.post( + "https://integrate.api.nvidia.com/v1/chat/completions", + headers=headers, + json=body, + timeout=60, + stream=True, + ) + r.raise_for_status() + yield from _stream_openai_sse(r) + return + + if provider == "huggingface": + key = get_api_key("huggingface") + if not key: + raise RuntimeError("HuggingFace API key is missing") + headers = {"Authorization": f"Bearer {key}"} + # Basic inference API. Not truly streaming; emulate. + r = requests.post( + f"https://api-inference.huggingface.co/models/{model}", + headers=headers, + json={"inputs": prompt, "parameters": {"max_new_tokens": 512}}, + timeout=90, + ) + if r.status_code == 404: + raise RuntimeError( + "HuggingFace: модель не найдена или недоступна для Inference API (404). " + "Попробуйте другую модель из списка (text-generation) или проверьте права доступа/гейтинг." + ) + r.raise_for_status() + data = r.json() + text = "" + if isinstance(data, list) and data and isinstance(data[0], dict) and isinstance(data[0].get("generated_text"), str): + text = data[0]["generated_text"] + elif isinstance(data, dict) and isinstance(data.get("generated_text"), str): + text = data["generated_text"] + elif isinstance(data, dict) and isinstance(data.get("error"), str): + raise RuntimeError(data["error"]) + else: + text = json.dumps(data, ensure_ascii=False) + for ch in text: + yield "content", ch + return + + raise RuntimeError(f"Unknown provider: {provider}") + + +def _stream_openai_sse(r: requests.Response) -> Generator[Tuple[str, str], None, None]: + for raw in r.iter_lines(decode_unicode=False): + line = _decode_line(raw).strip() + if not line: + continue + if not line.startswith("data:"): + continue + data = line[len("data:") :].strip() + if data == "[DONE]": + break + try: + obj = json.loads(data) + except Exception: + continue + try: + delta = obj["choices"][0].get("delta", {}) + content = delta.get("content") + if isinstance(content, str) and content: + yield "content", content + + reasoning = delta.get("reasoning") or delta.get("thinking") + if isinstance(reasoning, str) and reasoning: + yield "reasoning", reasoning + except Exception: + continue + + +if __name__ == "__main__": + # In some sandboxed environments, Werkzeug's debugger can fail due to /dev/shm restrictions. + debug = os.environ.get("FLASK_DEBUG", "").strip() in ("1", "true", "True") + app.run( + host="0.0.0.0", + port=int(os.environ.get("PORT", "8000")), + debug=debug, + use_reloader=debug, + )