From fba8f5f4f35119ff237180083d1694fa92a1b421 Mon Sep 17 00:00:00 2001 From: help4bis Date: Fri, 12 Jun 2026 06:17:39 +1000 Subject: [PATCH] feat(vision): /api/vision endpoint for Garden Buddy Buddy Check (v0.2.0) - New POST /api/vision: multipart contract (image/image2/image3 + prompt, system_prompt, backend, model) matching gardenbuddy-buddy-check v1.2.x - Anthropic Haiku 4.5 vision via direct /v1/messages with the existing Claude Code OAuth token (anthropic-beta: oauth-2025-04-20) - Ollama backend honoured only when the model is installed, otherwise falls back to Anthropic (qwen2.5vl retired from this GPU 2026-04-19) - Bearer auth via CLAUDE_PROXY_VISION_TOKEN (matches gbc_proxy_token) - Also commits the earlier uncommitted vision-aware /api/chat work - Version 0.1.0 -> 0.2.0 Co-Authored-By: Claude Fable 5 --- app.py | 326 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 311 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 0b04134..2a7beab 100644 --- a/app.py +++ b/app.py @@ -12,30 +12,55 @@ Endpoints: GET /api/tags list "models" (so clients can validate) POST /api/show model details stub POST /api/generate single-shot prompt -> response - POST /api/chat multi-message conversation -> response + POST /api/chat multi-message conversation -> response (vision-aware) + POST /api/vision multipart image diagnosis endpoint (Garden Buddy + Buddy Check contract: image/image2/image3 files + + prompt/system_prompt/backend/model fields) Both /api/generate and /api/chat honour the `stream` flag in the request body (Ollama default is True). When true, responses are emitted as NDJSON chunks; when false, a single JSON object is returned. +Vision support: when /api/chat messages contain an `images` list (Ollama +vision format), each base64 blob is written to a temp file under +CLAUDE_PROXY_IMG_DIR and the file path is appended to the prompt so that +claude's Read tool can load the image as a vision input. + Environment variables: CLAUDE_BIN path to claude CLI (default: "claude") CLAUDE_PROXY_CONCURRENCY max concurrent claude subprocesses (default: 3) CLAUDE_PROXY_MODEL name advertised in /api/tags (default: "claude-code") CLAUDE_PROXY_TIMEOUT per-request timeout in seconds (default: 300) - CLAUDE_CODE_OAUTH_TOKEN long-lived auth token, inherited by claude subprocess + CLAUDE_PROXY_IMG_DIR temp dir for vision images (default: /tmp/claude_proxy_imgs) + CLAUDE_CODE_OAUTH_TOKEN long-lived auth token; used by the claude + subprocess AND as Bearer auth for direct + /v1/messages calls on the /api/vision route + (anthropic-beta: oauth-2025-04-20) + CLAUDE_PROXY_VISION_TOKEN shared secret for /api/vision Bearer auth + (matches gardenbuddy.au gbc_proxy_token); + if unset, /api/vision auth is not enforced + OLLAMA_URL local Ollama base URL for the ollama backend + (default: http://127.0.0.1:11434) + +Version: 0.2.0 +Updated: 2026-06-12 """ from __future__ import annotations import asyncio +import base64 import json import logging import os +import tempfile import time from datetime import datetime, timezone +from pathlib import Path from typing import Any, AsyncIterator +import httpx +from anthropic import AsyncAnthropic from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -45,6 +70,19 @@ CLAUDE_BIN = os.environ.get("CLAUDE_BIN", "claude") CONCURRENCY = int(os.environ.get("CLAUDE_PROXY_CONCURRENCY", "3")) DEFAULT_MODEL = os.environ.get("CLAUDE_PROXY_MODEL", "claude-code") TIMEOUT_SECONDS = int(os.environ.get("CLAUDE_PROXY_TIMEOUT", "300")) +IMG_DIR = Path(os.environ.get("CLAUDE_PROXY_IMG_DIR", "/tmp/claude_proxy_imgs")) +OAUTH_TOKEN = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN", "") +VISION_TOKEN = os.environ.get("CLAUDE_PROXY_VISION_TOKEN", "") +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434").rstrip("/") +VISION_DEFAULT_MODEL = "claude-haiku-4-5" + +# Claude Haiku 4.5 list pricing (USD per million tokens). Used only to fill the +# cost_usd tracking field. Actual marginal cost is $0 because the OAuth token +# bills against the Claude subscription, not pay-per-token. +HAIKU_IN_PER_MTOK = 1.00 +HAIKU_OUT_PER_MTOK = 5.00 +HAIKU_CACHE_READ_PER_MTOK = 0.10 +HAIKU_CACHE_WRITE_PER_MTOK = 1.25 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") log = logging.getLogger("claude-proxy") @@ -52,6 +90,8 @@ log = logging.getLogger("claude-proxy") app = FastAPI(title="claude-code-proxy") _semaphore = asyncio.Semaphore(CONCURRENCY) +IMG_DIR.mkdir(parents=True, exist_ok=True) + # --- Helpers ---------------------------------------------------------------- @@ -59,12 +99,55 @@ def _now_iso() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") -async def _run_claude(prompt: str) -> str: - """Run `claude -p ` and return stdout as a string.""" +def _save_images(images: list[str]) -> list[Path]: + """Write base64-encoded image blobs to temp files; return their paths.""" + paths: list[Path] = [] + for b64 in images: + # Strip data-URI prefix if present (data:image/png;base64,...) + if "," in b64: + b64 = b64.split(",", 1)[1] + try: + data = base64.b64decode(b64) + except Exception: + log.warning("skipping malformed base64 image blob") + continue + # Detect format from magic bytes + ext = ".jpg" + if data[:8] == b"\x89PNG\r\n\x1a\n": + ext = ".png" + elif data[:4] in (b"RIFF", b"WEBP"): + ext = ".webp" + elif data[:4] == b"GIF8": + ext = ".gif" + tmp = tempfile.NamedTemporaryFile( + suffix=ext, prefix="img_", dir=IMG_DIR, delete=False + ) + tmp.write(data) + tmp.close() + paths.append(Path(tmp.name)) + return paths + + +async def _run_claude(prompt: str, image_paths: list[Path] | None = None) -> str: + """Run `claude -p ` and return stdout as a string. + + When image_paths is provided the prompt already contains [image: path] + references and we pass --allowedTools to permit Read on those specific files. + """ async with _semaphore: - log.info("claude -p invoked (prompt %d chars)", len(prompt)) + log.info( + "claude -p invoked (prompt %d chars, images %d) | preview: %s", + len(prompt), + len(image_paths) if image_paths else 0, + prompt[:120].replace("\n", " "), + ) + + cmd = [CLAUDE_BIN, "-p", prompt] + if image_paths: + cmd += ["--allowedTools", "Read"] + proc = await asyncio.create_subprocess_exec( - CLAUDE_BIN, "-p", prompt, + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=os.environ.copy(), @@ -77,29 +160,60 @@ async def _run_claude(prompt: str) -> str: proc.kill() await proc.wait() raise RuntimeError(f"claude -p timed out after {TIMEOUT_SECONDS}s") + finally: + # Always clean up temp image files. + if image_paths: + for p in image_paths: + try: + p.unlink(missing_ok=True) + except Exception: + pass if proc.returncode != 0: err = stderr.decode("utf-8", errors="replace")[:1000] raise RuntimeError(f"claude -p exited {proc.returncode}: {err}") - return stdout.decode("utf-8", errors="replace") + result = stdout.decode("utf-8", errors="replace") + log.info("claude -p response: %d chars | preview: %s", len(result), result[:120].replace("\n", " ")) + return result -def _build_prompt_from_messages(messages: list[dict]) -> str: - """Flatten OpenAI/Ollama-style messages into a single prompt string.""" +def _build_prompt_from_messages(messages: list[dict]) -> tuple[str, list[Path]]: + """Flatten Ollama-style messages into a prompt string and extract any images. + + Returns (prompt_string, list_of_temp_image_paths). + Image references are appended to the user turn that contained them so + claude's Read tool knows to load them as vision inputs. + """ system_parts = [m["content"] for m in messages if m.get("role") == "system"] convo: list[str] = [] + all_image_paths: list[Path] = [] + for m in messages: role = m.get("role", "user") if role == "system": continue prefix = "User" if role == "user" else "Assistant" - convo.append(f"{prefix}: {m.get('content', '')}") + content = m.get("content", "") + + # Handle vision: save images and embed file references in the turn. + images_in_msg = m.get("images", []) + if images_in_msg: + paths = _save_images(images_in_msg) + all_image_paths.extend(paths) + refs = "\n".join(f"[image: {p}]" for p in paths) + content = f"{content}\n{refs}" if content else refs + + convo.append(f"{prefix}: {content}") + convo.append("Assistant:") body = "\n\n".join(convo) if system_parts: - return "[System]\n" + "\n\n".join(system_parts) + "\n\n" + body - return body + prompt = "[System]\n" + "\n\n".join(system_parts) + "\n\n" + body + else: + prompt = body + + return prompt, all_image_paths # --- Streaming generators --------------------------------------------------- @@ -153,7 +267,7 @@ async def root() -> dict: @app.get("/api/version") async def version() -> dict: - return {"version": "0.1.0-claude-proxy"} + return {"version": "0.2.0-claude-proxy"} @app.get("/api/tags") @@ -234,11 +348,11 @@ async def chat(req: Request) -> Any: messages = body.get("messages", []) stream = bool(body.get("stream", True)) - prompt = _build_prompt_from_messages(messages) + prompt, image_paths = _build_prompt_from_messages(messages) started = time.time() try: - text = await _run_claude(prompt) + text = await _run_claude(prompt, image_paths or None) except Exception as e: log.exception("claude invocation failed") return JSONResponse({"error": str(e)}, status_code=500) @@ -256,3 +370,185 @@ async def chat(req: Request) -> Any: "done_reason": "stop", "total_duration": int((time.time() - started) * 1e9), } + + +# --- Vision (Garden Buddy Buddy Check) --------------------------------------- + +def _media_type(data: bytes) -> str: + """Detect image media type from magic bytes (default JPEG).""" + if data[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + if data[:4] == b"GIF8": + return "image/gif" + return "image/jpeg" + + +def _haiku_cost_usd(usage: Any) -> float: + """API-equivalent Haiku 4.5 cost. Informational only (subscription-billed).""" + return round( + getattr(usage, "input_tokens", 0) * HAIKU_IN_PER_MTOK / 1e6 + + getattr(usage, "output_tokens", 0) * HAIKU_OUT_PER_MTOK / 1e6 + + (getattr(usage, "cache_read_input_tokens", 0) or 0) * HAIKU_CACHE_READ_PER_MTOK / 1e6 + + (getattr(usage, "cache_creation_input_tokens", 0) or 0) * HAIKU_CACHE_WRITE_PER_MTOK / 1e6, + 6, + ) + + +async def _ollama_model_available(model: str) -> bool: + try: + async with httpx.AsyncClient(timeout=5) as cx: + r = await cx.get(f"{OLLAMA_URL}/api/tags") + r.raise_for_status() + names = [m.get("name", "") for m in r.json().get("models", [])] + return any(n == model or n.split(":")[0] == model.split(":")[0] for n in names) + except Exception: + return False + + +async def _vision_via_anthropic( + images: list[bytes], system_prompt: str, user_prompt: str, model: str +) -> dict: + """Direct /v1/messages call using the Claude Code OAuth token.""" + content: list[dict] = [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": _media_type(img), + "data": base64.standard_b64encode(img).decode("ascii"), + }, + } + for img in images + ] + content.append({"type": "text", "text": user_prompt}) + + client = AsyncAnthropic( + auth_token=OAUTH_TOKEN, + default_headers={"anthropic-beta": "oauth-2025-04-20"}, + ) + started = time.time() + try: + msg = await client.messages.create( + model=model, + max_tokens=4096, + system=system_prompt or "", + messages=[{"role": "user", "content": content}], + ) + finally: + await client.close() + + text = "".join(b.text for b in msg.content if b.type == "text") + return { + "content": text, + "backend": "anthropic", + "model": msg.model, + "duration_ms": int((time.time() - started) * 1000), + "prompt_eval_count": msg.usage.input_tokens, + "eval_count": msg.usage.output_tokens, + "cache_read_tokens": msg.usage.cache_read_input_tokens or 0, + "cache_creation_tokens": msg.usage.cache_creation_input_tokens or 0, + "cost_usd": _haiku_cost_usd(msg.usage), + } + + +async def _vision_via_ollama( + images: list[bytes], system_prompt: str, user_prompt: str, model: str +) -> dict: + """Local Ollama vision call (only used when the model is installed).""" + payload = { + "model": model, + "stream": False, + "messages": [ + {"role": "system", "content": system_prompt or ""}, + { + "role": "user", + "content": user_prompt, + "images": [base64.standard_b64encode(i).decode("ascii") for i in images], + }, + ], + } + started = time.time() + async with httpx.AsyncClient(timeout=TIMEOUT_SECONDS) as cx: + r = await cx.post(f"{OLLAMA_URL}/api/chat", json=payload) + r.raise_for_status() + data = r.json() + return { + "content": data.get("message", {}).get("content", ""), + "backend": "ollama", + "model": data.get("model", model), + "duration_ms": int((time.time() - started) * 1000), + "prompt_eval_count": data.get("prompt_eval_count", 0), + "eval_count": data.get("eval_count", 0), + "cache_read_tokens": 0, + "cache_creation_tokens": 0, + "cost_usd": 0.0, + } + + +@app.post("/api/vision") +async def vision(req: Request) -> Any: + """Photo diagnosis endpoint matching the gardenbuddy-buddy-check contract. + + Request: multipart/form-data with file fields image, image2, image3 and + text fields prompt, system_prompt, backend (anthropic|ollama), + model. Bearer auth required when CLAUDE_PROXY_VISION_TOKEN set. + Response: 200 JSON {content, backend, model, duration_ms, + prompt_eval_count, eval_count, cache_read_tokens, + cache_creation_tokens, cost_usd}. + """ + if VISION_TOKEN: + auth = req.headers.get("authorization", "") + if auth != f"Bearer {VISION_TOKEN}": + return JSONResponse({"error": "unauthorized"}, status_code=401) + + form = await req.form() + user_prompt = str(form.get("prompt", "") or "") + system_prompt = str(form.get("system_prompt", "") or "") + backend = str(form.get("backend", "anthropic") or "anthropic").lower() + model = str(form.get("model", "") or "") + + images: list[bytes] = [] + for key, value in form.multi_items(): + if key.startswith("image") and hasattr(value, "read"): + data = await value.read() + if data: + images.append(data) + + if not images: + return JSONResponse({"error": "at least one image file is required"}, status_code=400) + if len(images) > 3: + return JSONResponse({"error": "up to 3 images allowed"}, status_code=400) + if not user_prompt: + return JSONResponse({"error": "prompt is required"}, status_code=400) + + # Ollama is only honoured when the requested model is actually installed; + # otherwise we fall back to Anthropic Haiku so a diagnosis never 404s on a + # missing local model. (qwen2.5vl was retired from this GPU on 2026-04-19.) + if backend == "ollama": + ollama_model = model or "qwen2.5vl:7b" + if await _ollama_model_available(ollama_model): + try: + async with _semaphore: + return await _vision_via_ollama(images, system_prompt, user_prompt, ollama_model) + except Exception: + log.exception("ollama vision failed, falling back to anthropic") + else: + log.warning("ollama model %s not installed; falling back to anthropic", ollama_model) + model = "" # discard the ollama model name for the fallback + + anthropic_model = model if model.startswith("claude") else VISION_DEFAULT_MODEL + try: + async with _semaphore: + result = await _vision_via_anthropic(images, system_prompt, user_prompt, anthropic_model) + except Exception as e: + log.exception("anthropic vision call failed") + return JSONResponse({"error": f"vision backend error: {e}"}, status_code=502) + + log.info( + "vision ok: backend=%s model=%s images=%d in=%d out=%d %dms", + result["backend"], result["model"], len(images), + result["prompt_eval_count"], result["eval_count"], result["duration_ms"], + ) + return result