Claude Vision & Multimodal AI

"""Send a picture to Claude and get structured facts back."""
import base64
import json
import math
import re
 
import cv2
 
MODEL = "claude-sonnet-5-5"
LONG_EDGE = 1568                      # bigger pictures are shrunk by the service anyway: shrink first, it is faster and safer
 
PROMPT = """Look at this football match photo.
Answer ONLY with JSON, no other text, in exactly this shape:
{"players_visible": <integer>, "ball_visible": <true or false>, "ball_location": "<short words, e.g. near the left goal>",
 "goal_scored_this_moment": <true or false>, "confidence": "low" | "medium" | "high", "notes": "<one short sentence>"}
If you are unsure, say so in confidence and notes. Do not guess names. Do not identify anyone."""
 
 
def prepare_image(frame, long_edge=LONG_EDGE, quality=85):
    """OpenCV frame -> (base64 text, media type, width, height) ready for the API."""
    h, w = frame.shape[:2]
    scale = min(1.0, long_edge / max(h, w))
    if scale < 1.0:
        frame = cv2.resize(frame, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_AREA)
    ok, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, quality])
    if not ok:
        raise ValueError("could not encode the frame")
    h, w = frame.shape[:2]
    return base64.standard_b64encode(buf.tobytes()).decode(), "image/jpeg", w, h
 
 
def image_tokens(w, h, high_res=True):
    """Rough cost of one picture: one token per 28x28 patch (from Anthropic's vision docs).
    Newer models allow a larger picture; older ones cap at 1568 tokens."""
    long_edge, max_tokens = (2576, 4784) if high_res else (1568, 1568)
    scale = min(1.0, long_edge / max(w, h))
    w, h = w * scale, h * scale
    return min(max_tokens, math.ceil(w / 28) * math.ceil(h / 28))
 
 
def ask_about_frame(frame, client, model=MODEL, prompt=PROMPT, max_tokens=300):
    data, media_type, w, h = prepare_image(frame)
    msg = client.messages.create(
        model=model,
        max_tokens=max_tokens,
        messages=[{
            "role": "user",
            "content": [
                {"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}},   # picture first,
                {"type": "text", "text": prompt},                                                           # then the question
            ],
        }],
    )
    return "".join(b.text for b in msg.content if b.type == "text")
 
 
def parse_answer(text):
    """Claude is asked for JSON, but may wrap it in markdown code fences or add a sentence. Be tolerant, then strict."""
    m = re.search(r"\{.*\}", text, re.S)
    if not m:
        raise ValueError("no JSON found in the answer")
    data = json.loads(m.group(0))
    need = {"players_visible": int, "ball_visible": bool, "goal_scored_this_moment": bool, "confidence": str}
    for key, typ in need.items():
        if key not in data or not isinstance(data[key], typ) or (typ is int and isinstance(data[key], bool)):
            raise ValueError(f"missing or wrong type: {key}")
    if data["confidence"] not in ("low", "medium", "high"):
        raise ValueError("confidence must be low, medium or high")
    if data["players_visible"] < 0 or data["players_visible"] > 50:
        raise ValueError("players_visible out of range")
    return data

"""Claude versus your detector: where do they agree? Only trust a disagreement after you have looked at the picture."""
 
 
def compare(frame_id, yolo_ball, claude_answer):
    """yolo_ball: dict with x,y or None. claude_answer: parsed JSON from parse_answer()."""
    y, c = yolo_ball is not None, claude_answer["ball_visible"]
    if y and c:
        verdict = "agree: ball seen by both"
    elif not y and not c:
        verdict = "agree: no ball"
    elif y and not c:
        verdict = "CHECK: detector sees a ball, Claude does not (false alarm, or tiny ball?)"
    else:
        verdict = "CHECK: Claude sees a ball, detector does not (missed ball?)"
    return {"frame": frame_id, "detector_ball": y, "claude_ball": c,
            "claude_confidence": claude_answer["confidence"], "verdict": verdict}
 
 
def should_ask_claude(frame_index, detector_ball, every=150, last_asked=-10**9, min_gap=30):
    """Claude is slow and costs money, so ask rarely: on a schedule, or when the detector is unsure/silent."""
    if frame_index - last_asked < min_gap:
        return False
    if frame_index % every == 0:
        return True
    return detector_ball is None and frame_index % 30 == 0

import cv2
from vision import ask_about_frame, parse_answer
from vision import MODEL
from commentary import get_client        # from lesson 37
 
frame = cv2.imread("frame.jpg")           # any frame from your own video
text = ask_about_frame(frame, get_client())
print(text)
print(parse_answer(text))

import json
import numpy as np
from compare import compare, should_ask_claude
from detector import find_ball
from fake_client import FakeClient
from scene import make_clip
from vision import ask_about_frame, image_tokens, parse_answer, prepare_image
 
bad = 0
def check(name, ok, extra=""):
    global bad; bad += not ok; print("PASS" if ok else "FAIL", name, extra)
 
frames, truth = make_clip(n=130)
f = frames[60]
b64, mt, w, h = prepare_image(f)
check("image prepared as jpeg base64", mt == "image/jpeg" and len(b64) > 100 and (w, h) == (640, 360), (w, h, len(b64)))
big = np.zeros((2000, 4000, 3), np.uint8)
_, _, w2, h2 = prepare_image(big)
check("big picture shrunk to 1568 on the long edge", max(w2, h2) == 1568, (w2, h2))
 
check("tokens 640x360 = 23*13 = 299", image_tokens(640, 360) == 299, image_tokens(640, 360))
check("tokens capped at 1568 (older tier)", image_tokens(4000, 3000, high_res=False) <= 1568)
check("tokens capped at 4784 (newer tier)", image_tokens(8000, 8000, high_res=True) <= 4784)
 
good = '{"players_visible": 4, "ball_visible": true, "ball_location": "near the goal", "goal_scored_this_moment": false, "confidence": "high", "notes": "ok"}'
fc = FakeClient(good)
ans = parse_answer(ask_about_frame(f, fc))
call = fc.calls[0]; content = call["messages"][0]["content"]
check("image block comes before the text", content[0]["type"] == "image" and content[1]["type"] == "text")
check("source is base64 jpeg", content[0]["source"]["type"] == "base64" and content[0]["source"]["media_type"] == "image/jpeg")
check("parsed answer", ans["players_visible"] == 4 and ans["ball_visible"] is True)
 
tick = chr(96) * 3                                   # three backticks, written this way so this page stays tidy
fenced = "Sure! " + tick + "json\n" + good + "\n" + tick
check("fenced JSON still parses", parse_answer(fenced)["confidence"] == "high")
for name, text in [("no json", "I see a lovely match."), ("bad type", good.replace("4,", '"four",', 1)),
                   ("bad confidence", good.replace("high", "certain")), ("too many players", good.replace(": 4,", ": 400,")),
                   ("boolean as int", good.replace(": 4,", ": true,")), ("missing key", '{"players_visible": 3}')]:
    try:
        parse_answer(text); check(f"rejects {name}", False)
    except (ValueError, json.JSONDecodeError):
        check(f"rejects {name}", True)
 
ball = find_ball(f)
check("compare agree", compare(60, ball, ans)["verdict"].startswith("agree"))
check("compare flags missed ball", compare(60, None, ans)["verdict"].startswith("CHECK"))
check("compare flags false alarm", compare(60, ball, {**ans, "ball_visible": False})["verdict"].startswith("CHECK"))
 
last, sent = -10**9, 0
for i in range(600):
    if should_ask_claude(i, {"x": 1}, last_asked=last):
        last, sent = i, sent + 1
check("with a ball always seen, only the schedule fires (every 150 frames)", sent == 4, sent)
print("failures:", bad)

from events import EventBus
from goal_detector import GoalDetector
from vision import ask_about_frame, parse_answer
from commentary import get_client
 
client = get_client()
latest = {"frame": None}                     # the match loop keeps this up to date
 
def second_opinion(zone, t):
    answer = parse_answer(ask_about_frame(latest["frame"], client))
    verdict = "agrees" if answer["goal_scored_this_moment"] else "DISAGREES"
    print(f"goal at {t:.1f} s: Claude {verdict} (confidence {answer['confidence']})")
 
bus = EventBus()
bus.on("goal", second_opinion)