Field Trip 3: The Final Deployment Test

"""Acceptance test: did the product meet the success criteria in the lesson 47 brief, at the venue, honestly?
Uses debrief.py (lesson 40) to match what the system announced with what the scribe saw."""
import glob
import math
import os
import statistics
 
from debrief import load_log, match_goals, split_events
 
 
def wilson(k, n, z=1.96):
    """95 percent interval for a proportion k out of n. Honest about small samples."""
    if n == 0:
        return (0.0, 1.0)
    p = k / n
    d = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / d
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return (max(0.0, centre - half), min(1.0, centre + half))
 
 
def measure(log_path, minutes, tolerance=3.0):
    """Turn the event log of the test into the numbers a brief usually asks for."""
    detected, truth = split_events(load_log(log_path))
    pairs, fa, missed = match_goals(detected, truth, tolerance)
    tp, fp, fn = len(pairs), len(fa), len(missed)
    delays = [d - g for d, g in pairs]
    return {
        "real_events": tp + fn, "tp": tp, "fp": fp, "fn": fn, "minutes": minutes,
        "recall": tp / (tp + fn) if tp + fn else None,
        "precision": tp / (tp + fp) if tp + fp else None,
        "false_per_10min": fp / minutes * 10 if minutes > 0 else None,
        "median_delay_s": statistics.median(delays) if delays else None,
        "counts": {"recall": (tp, tp + fn), "precision": (tp, tp + fp)},      # k out of n, for the error bars
    }
 
 
MIN_EVENTS = 5
 
 
def judge(criterion, value, counts=None):
    """One criterion -> {'status': PASS | LIKELY | FAIL | NOT MEASURED, 'note': str}.
    PASS = the whole error bar is on the right side. LIKELY = the number passes but the error bar crosses the target."""
    if value is None:
        return {"status": "NOT MEASURED", "note": "no value: the test did not produce this number"}
    op, target = criterion["op"], criterion["target"]
    passes = {">=": value >= target, ">": value > target, "<=": value <= target, "<": value < target, "==": value == target}[op]
    if not passes:
        return {"status": "FAIL", "note": f"measured {value:.3g}, needed {op} {target}"}
    if counts:
        k, n = counts
        lo, hi = wilson(k, n)
        clear = (lo >= target) if op in (">=", ">") else (hi <= target)
        if n < MIN_EVENTS:
            return {"status": "LIKELY", "note": f"passes, but only {n} events: far too few to be sure (95 percent range {lo:.2f} to {hi:.2f})"}
        if not clear:
            return {"status": "LIKELY", "note": f"passes ({k} of {n}), but the 95 percent range {lo:.2f} to {hi:.2f} crosses the target"}
        return {"status": "PASS", "note": f"{k} of {n}, 95 percent range {lo:.2f} to {hi:.2f}"}
    return {"status": "PASS", "note": f"measured {value:.3g}, needed {op} {target} (one test, no error bar)"}
 
 
def evaluate(brief, bindings, measured):
    """bindings: {criterion metric text: key in measured}. A criterion with no binding is NOT MEASURED."""
    out = []
    for c in brief["success"]:
        key = bindings.get(c["metric"])
        value = measured.get(key) if key else None
        v = judge(c, value, measured.get("counts", {}).get(key) if key else None)
        out.append({"metric": c["metric"], "op": c["op"], "target": c["target"], "unit": c["unit"], "value": value, **v})
    return out
 
 
def decide(verdicts):
    s = {v["status"] for v in verdicts}
    if not verdicts or s & {"FAIL", "NOT MEASURED"}:
        return "NOT YET"
    return "ACCEPT WITH CAUTION" if "LIKELY" in s else "ACCEPT"
 
 
REQUIRED = {"event log": ["event_log.csv"], "raw video": ["*.mp4", "*.mov", "*.mkv"], "consent record": ["consent*"],
            "venue permission": ["permission*", "venue*"], "notes": ["notes*", "*.md", "*.txt"]}
 
 
def check_artifacts(folder):
    """The evidence folder: without it the numbers are just claims."""
    missing = []
    for name, patterns in REQUIRED.items():
        found = [p for pat in patterns for p in glob.glob(os.path.join(folder, pat)) if os.path.getsize(p) > 0]
        if not found:
            missing.append(name)
    return missing
 
 
def report_markdown(brief, verdicts, decision, run, missing):
    lines = [f"# Acceptance report: {brief['name']}", "",
             f"**Decision: {decision}**", "",
             f"Venue: {run['venue']}. Date: {run['date']}. Length of test: {run['minutes']} minutes. Camera: {run['camera']}.", "",
             "| Criterion | Target | Measured | Result | Note |", "|---|---|---|---|---|"]
    for v in verdicts:
        val = "none" if v["value"] is None else f"{v['value']:.3g}"
        lines.append(f"| {v['metric']} | {v['op']} {v['target']} {v['unit']} | {val} | {v['status']} | {v['note']} |")
    lines += ["", "## Evidence", ""]
    lines.append("All evidence is in the folder." if not missing else "Missing from the evidence folder: " + ", ".join(missing) + ".")
    lines += ["", "## What we would do next", "", run.get("next", "(the team writes this)")]
    return "\n".join(lines)

"""Write pretend test logs. 'messy' has a missed goal, a false alarm and a duplicate; 'clean' has no mistakes."""
import csv
 
 
def write(path, rows):
    with open(path, "w", newline="") as f:
        w = csv.writer(f); w.writerow(["wall_time", "match_seconds", "kind", "text"]); w.writerows(rows)
 
 
def messy(path="event_log.csv"):
    write(path, [["19:03:30", 140.0, "goal", "north goal"], ["19:03:29", 139.2, "note", "GOAL red 7"],
                 ["19:07:05", 355.0, "goal", "north goal"],
                 ["19:11:00", 590.0, "note", "GOAL blue 3"],
                 ["19:20:05", 1135.0, "goal", "north goal"], ["19:20:03", 1133.5, "note", "GOAL red 9"],
                 ["19:20:09", 1139.0, "goal", "north goal"]])
    return path
 
 
def clean(path="event_log_clean.csv", goals=10, delay=1.0, spacing=120.0):
    rows = []
    for i in range(goals):
        t = 60 + i * spacing
        rows += [["x", t, "note", f"GOAL {i}"], ["x", t + delay, "goal", "north goal"]]
    write(path, rows)
    return path

{
  "name": "Goal Announcer",
  "success": [
    {"metric": "goal recall (real goals announced)", "op": ">=", "target": 0.9, "unit": "fraction"},
    {"metric": "false announcements per 10 minutes", "op": "<=", "target": 1, "unit": "count"},
    {"metric": "delay between goal and announcement", "op": "<=", "target": 3, "unit": "seconds"}
  ]
}

import json
from acceptance import measure, evaluate, decide
from make_demo_logs import messy, clean
 
brief = json.load(open("example_brief.json"))
bindings = {"goal recall (real goals announced)": "recall",
             "false announcements per 10 minutes": "false_per_10min",
             "delay between goal and announcement": "median_delay_s"}
 
for label, path, minutes in [("messy", messy(), 21), ("clean, 10 goals", clean(goals=10), 25), ("clean, 40 goals", clean("big.csv", goals=40), 90)]:
    verdicts = evaluate(brief, bindings, measure(path, minutes))
    print(label, "->", decide(verdicts))
    for v in verdicts:
        print("   ", v["status"].ljust(12), v["metric"], "|", v["note"])

from acceptance import check_artifacts, report_markdown
 
missing = check_artifacts("evidence")          # a folder with your log, raw video, consent, permission and notes
run = {"venue": "Your venue", "date": "2026-12-05", "minutes": 21, "camera": "phone on tripod, 1080p",
        "next": "Write what you would fix first."}
verdicts = evaluate(brief, bindings, measure("event_log.csv", 21))      # your own test log and its length in minutes
text = report_markdown(brief, verdicts, decide(verdicts), run, missing)
open("acceptance_report.md", "w").write(text)
print(text)

import json, os, shutil
from acceptance import *
from make_demo_logs import messy, clean
 
n = 0
def ok(name, cond):
    global n
    print(("PASS " if cond else "FAIL ") + name); n += bool(cond)
    if not cond: raise SystemExit(1)
 
brief = json.load(open("example_brief.json"))
B = {"goal recall (real goals announced)": "recall", "false announcements per 10 minutes": "false_per_10min",
     "delay between goal and announcement": "median_delay_s"}
 
lo, hi = wilson(10, 10); ok("wilson 10 of 10 is not certain", 0.7 < lo < 0.75 and hi == 1.0)
lo, hi = wilson(40, 40); ok("wilson 40 of 40 clears 0.9", lo > 0.9)
ok("wilson 0 events is 0 to 1", wilson(0, 0) == (0.0, 1.0))
lo, hi = wilson(0, 10); ok("wilson 0 of 10 starts at 0", lo == 0.0 and hi < 0.35)
 
m = measure(messy(), minutes=21)
ok("messy: 2 right, 2 false, 1 missed", (m["tp"], m["fp"], m["fn"]) == (2, 2, 1))
ok("messy: recall two thirds", abs(m["recall"] - 2 / 3) < 1e-9 and abs(m["precision"] - 0.5) < 1e-9)
ok("messy: false alarms per 10 min", abs(m["false_per_10min"] - 2 / 21 * 10) < 1e-9)
ok("messy: median delay 1.15 s", abs(m["median_delay_s"] - 1.15) < 1e-9)
v = evaluate(brief, B, m)
ok("messy: recall FAILS", v[0]["status"] == "FAIL")
ok("messy: false alarms pass (one test)", v[1]["status"] == "PASS" and "no error bar" in v[1]["note"])
ok("messy: delay passes", v[2]["status"] == "PASS")
ok("messy: NOT YET", decide(v) == "NOT YET")
 
m = measure(clean(goals=10), minutes=25)
v = evaluate(brief, B, m)
ok("clean 10 of 10: recall only LIKELY", v[0]["status"] == "LIKELY" and "crosses the target" in v[0]["note"])
ok("clean: zero false alarms pass", v[1]["status"] == "PASS" and m["false_per_10min"] == 0)
ok("clean 10 events: ACCEPT WITH CAUTION", decide(v) == "ACCEPT WITH CAUTION")
m = measure(clean("event_log_big.csv", goals=40), minutes=90)
ok("clean 40 of 40: ACCEPT", decide(evaluate(brief, B, m)) == "ACCEPT")
m = measure(clean("event_log_small.csv", goals=3), minutes=10)
v = evaluate(brief, B, m); ok("3 events: LIKELY and says too few", v[0]["status"] == "LIKELY" and "only 3 events" in v[0]["note"])
 
ok("unbound criterion is NOT MEASURED", evaluate(brief, {}, m)[0]["status"] == "NOT MEASURED" and decide(evaluate(brief, {}, m)) == "NOT YET")
ok("no real events gives no recall, not a crash", measure(clean("e0.csv", goals=0), minutes=10)["recall"] is None)
ok("None value is NOT MEASURED", judge({"op": ">=", "target": 1}, None)["status"] == "NOT MEASURED")
ok("operators work", judge({"op": "<", "target": 5}, 5)["status"] == "FAIL" and judge({"op": ">", "target": 5}, 6)["status"] == "PASS")
ok("empty verdict list is NOT YET", decide([]) == "NOT YET")
ok("delay is one test and only judged by point value", judge({"op": "<=", "target": 3}, 4.2)["status"] == "FAIL")
 
os.makedirs("ev", exist_ok=True)
ok("empty evidence folder lists all missing", len(check_artifacts("ev")) == 5)
for f, t in [("event_log.csv", "x"), ("match.mp4", "x"), ("consent_signed.pdf", "x"), ("permission_letter.pdf", "x"), ("notes.md", "x")]:
    open(f"ev/{f}", "w").write(t)
ok("full evidence folder is complete", check_artifacts("ev") == [])
open("ev/notes.md", "w").write(""); os.remove("ev/consent_signed.pdf")
ok("empty notes and no consent are missing", set(check_artifacts("ev")) == {"consent record", "notes"})
 
run = {"venue": "St Xavier ground", "date": "2026-12-05", "minutes": 21, "camera": "phone on tripod, 1080p", "next": "Fix the duplicate and the dark corner."}
md = report_markdown(brief, evaluate(brief, B, measure(messy(), 21)), "NOT YET", run, ["consent record"])
ok("report has decision, table and missing evidence", "**Decision: NOT YET**" in md and "| goal recall" in md and "Missing from the evidence folder: consent record." in md)
ok("report has no stray pipes in numbers", all(l.count("|") == 6 for l in md.splitlines() if l.startswith("| goal") or l.startswith("| false") or l.startswith("| delay")))
 
for p in ("event_log.csv", "event_log_clean.csv", "event_log_big.csv", "event_log_small.csv", "e0.csv"): os.path.exists(p) and os.remove(p)
shutil.rmtree("ev")
print(f"\n{n} checks passed")