Add CarMax task verifiers + judge rubrics

sites/carmax/verify/verify_0.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--0: find any 2022 Honda Civic; report full title, price, mileage.
+
+Deterministic-first: nav to a 2022 Honda Civic detail | answer has make/model + price +
+mileage of the (unique) 2022 Civic in inventory | LLM-anchored fallback.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, navigated_re, final_answer, last_shot,
+                        contains_all, price_mentioned, resolve_db, db_query,
+                        llm_text_match, Judge, parse_args)
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--0', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    gt = db_query(init, "SELECT year,make,model,trim,price,mileage FROM vehicles "
+                        "WHERE year=2022 AND make='Honda' AND model='Civic'") if init else []
+    yr, mk, md, tr, price, mi = gt[0] if gt else (2022, "Honda", "Civic", "EX", 15300, 58626)
+    # Title/price/mileage are all shown on the inventory card, so a search/results page
+    # OR the detail page both count as real on-site viewing (anti knowledge-shortcut).
+    j.check("nav_civic_on_site",
+            navigated_re(t, r"(?i)/(vehicle|cars).*civic"),
+            f"expected a /cars?...Civic search or /vehicle/...honda-civic page; urls={[s.get('url') for s in t.get('steps', [])]}")
+    j.check("answer_make_model", contains_all(fa, ["Honda", "Civic"]), f"final={fa!r}")
+    j.check("answer_price", price_mentioned(fa, int(price)), f"expected ${int(price):,} in {fa!r}")
+    j.check("answer_mileage", price_mentioned(fa, int(mi)), f"expected {int(mi):,} mi in {fa!r}")
+    ok, ev = llm_text_match(fa, f"{yr} {mk} {md} {tr}, price ${int(price):,}, {int(mi):,} miles",
+                            "Report the full title, price and mileage of a 2022 Honda Civic.")
+    j.check("answer_consistent", ok, ev, llm=True)
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_1.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--1: find a Toyota Tacoma TRD Off-Road; report store, mileage, price.
+
+Deterministic-first: nav to the Tacoma TRD Off-Road detail | answer has store + mileage + price.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, navigated_re, final_answer,
+                        contains_any, price_mentioned, resolve_db, db_query,
+                        llm_text_match, Judge, parse_args)
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--1', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    gt = db_query(init, "SELECT v.mileage, v.price, s.name, s.city FROM vehicles v "
+                        "JOIN stores s ON s.id=v.store_id WHERE v.make='Toyota' AND v.model='Tacoma' "
+                        "AND v.trim LIKE '%TRD Off-Road%'") if init else []
+    mi, price, store, city = gt[0] if gt else (91787, 15000, "CarMax Seattle Lynnwood", "Lynnwood")
+    # store/mileage/price are shown on the inventory card, so a search/results page OR
+    # the detail page both count as real on-site viewing (anti knowledge-shortcut).
+    j.check("nav_tacoma_on_site", navigated_re(t, r"(?i)/(vehicle|cars).*tacoma"),
+            f"expected a /cars?...Tacoma search or /vehicle/...tacoma page; urls={[s.get('url') for s in t.get('steps', [])]}")
+    j.check("answer_store", contains_any(fa, [store, city]), f"expected {city!r}; final={fa!r}")
+    j.check("answer_price", price_mentioned(fa, int(price)), f"expected ${int(price):,}")
+    j.check("answer_mileage", price_mentioned(fa, int(mi)), f"expected {int(mi):,} mi")
+    ok, ev = llm_text_match(fa, f"store {store} ({city}), {int(mi):,} miles, price ${int(price):,}",
+                            "Report the Toyota Tacoma TRD Off-Road's store location, mileage and price.")
+    j.check("answer_consistent", ok, ev, llm=True)
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_10.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--10: sign in as alice, reserve any 2022 Toyota Camry for 7 days
+with appointment date 2026-05-20, then confirm it is listed as active.
+
+Deterministic-first: nav login + reserve + reservations page | DB after-state: alice has an
+ACTIVE reservation for a 2022 Toyota Camry appt 2026-05-20 that was NOT in the initial seed.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, final_answer, contains_any, resolve_db,
+                        reservations_for, Judge, parse_args)
+
+EMAIL = "alice.j@test.com"
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--10', a.no_llm)
+    t = load_run(a.run_dir)
+    after = resolve_db(a.after_db, a.container, "instance")
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    aft = reservations_for(after, EMAIL, "active") or []
+    ini = reservations_for(init, EMAIL, "active") or []
+    def is_target(r):  # (year, make, model, appointment_date, status)
+        return r[0] == 2022 and r[1] == "Toyota" and r[2] == "Camry" and str(r[3]) == "2026-05-20"
+    new_camry = [r for r in aft if is_target(r) and r not in ini]
+    j.check("nav_login", navigated_to(t, "/login"), "expected /login")
+    j.check("nav_reserve", navigated_to(t, "/reserve"), "expected a /reserve flow")
+    j.check("nav_reservations", navigated_to(t, "/account/reservations"), "expected reservations page")
+    j.check("db_active_2022_camry_reservation_2026_05_20", bool(new_camry),
+            f"after_active={aft} initial_active={ini}")
+    fa = final_answer(t)
+    j.check("answer_confirms_reservation", contains_any(fa, ["Camry", "reserved", "reservation", "active"]),
+            f"expected the answer to confirm the active 2022 Camry reservation; final={fa!r}")
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_11.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--11: sign in as bob.k, schedule an at-home test drive for any
+2022 Ford F-150 on 2026-05-22 at 2:00 PM with note 'Please call gate buzzer 4B',
+then confirm it shows on the test drives page.
+
+Deterministic-first: nav login + test-drive + test-drives page | DB after-state: bob.k has an
+at_home test drive for a 2022 Ford F-150 on 2026-05-22 2:00 PM with the note, not in seed.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, final_answer, contains_any, resolve_db,
+                        test_drives_for, norm, Judge, parse_args)
+
+EMAIL = "bob.k@test.com"
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--11', a.no_llm)
+    t = load_run(a.run_dir)
+    after = resolve_db(a.after_db, a.container, "instance")
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    aft = test_drives_for(after, EMAIL) or []
+    ini = test_drives_for(init, EMAIL) or []
+    # row: (year, make, model, location_type, scheduled_date, scheduled_time, notes, status)
+    def is_target(r):
+        return (r[0] == 2022 and r[1] == "Ford" and r[2] == "F-150" and r[3] == "at_home"
+                and str(r[4]) == "2026-05-22" and "2:00" in (r[5] or "")
+                and "gate buzzer 4b" in norm(r[6]))
+    new_td = [r for r in aft if is_target(r) and r not in ini]
+    j.check("nav_login", navigated_to(t, "/login"), "expected /login")
+    j.check("nav_test_drive", navigated_to(t, "/test-drive"), "expected a /test-drive flow")
+    j.check("nav_test_drives_page", navigated_to(t, "/account/test-drives"), "expected test-drives page")
+    j.check("db_at_home_f150_testdrive_with_note", bool(new_td),
+            f"after={aft} initial={ini}")
+    fa = final_answer(t)
+    j.check("answer_confirms_testdrive", contains_any(fa, ["F-150", "test drive", "test-drive", "scheduled"]),
+            f"expected the answer to confirm the F-150 at-home test drive; final={fa!r}")
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_12.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--12: find the FAQ answer to 'How long is my appraisal offer good
+for?' and report the number of days it is valid.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, final_answer, price_mentioned, contains_any,
+                        llm_text_match, Judge, parse_args)
+
+# CarMax appraisal offers are valid 7 days (matches reserve/offer logic: created + 7d).
+DAYS = 7
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--12', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    j.check("nav_faq", navigated_to(t, "/faq"), "expected a /faq page")
+    j.check("answer_days", price_mentioned(fa, DAYS) or contains_any(fa, ["7 days", "seven days"]),
+            f"expected {DAYS} days; final={fa!r}")
+    ok, ev = llm_text_match(fa, f"the appraisal offer is valid for {DAYS} days",
+                            "How many days is a CarMax appraisal offer good for?")
+    j.check("answer_consistent", ok, ev, llm=True)
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_13.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--13: sign in as alice (who has two saved cars), remove the saved
+vehicle with the HIGHER mileage, then report the year/make/model/store of the remaining one.
+
+Deterministic-first: nav login + saved | DB after-state: the higher-mileage saved car is
+gone and the lower-mileage one remains | answer names the remaining car.
+
+Note (review): the task text says the two saved cars are "from different makes", but in the
+seed both are Honda (2020 Civic 69k mi, 2021 CR-V 57.8k mi). The remove-higher-mileage step
+is still deterministic; this verifier checks the actual data, not the (incorrect) wording.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, final_answer, contains_all, resolve_db,
+                        saved_vehicles_for, Judge, parse_args)
+
+EMAIL = "alice.j@test.com"
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--13', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    after = resolve_db(a.after_db, a.container, "instance")
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    ini = saved_vehicles_for(init, EMAIL) or []   # rows: (year, make, model, trim, mileage) sorted by mileage
+    aft = saved_vehicles_for(after, EMAIL) or []
+    higher = max(ini, key=lambda r: r[4]) if ini else None   # should be removed
+    lower = min(ini, key=lambda r: r[4]) if ini else None     # should remain
+    j.check("nav_login", navigated_to(t, "/login"), "expected /login")
+    # Saved cars are shown on both /saved and the /account page, so either is a valid path.
+    j.check("nav_saved_or_account", navigated_to(t, "/saved") or navigated_to(t, "/account"),
+            f"expected /saved or /account; urls={[s.get('url') for s in t.get('steps', [])]}")
+    j.check("db_one_saved_remains", len(aft) == max(0, len(ini) - 1),
+            f"initial_saved={len(ini)} after_saved={len(aft)}")
+    j.check("db_higher_mileage_removed", bool(higher) and higher not in aft,
+            f"higher-mileage car {higher} should be removed; after={aft}")
+    j.check("db_lower_mileage_remains", bool(lower) and lower in aft,
+            f"lower-mileage car {lower} should remain; after={aft}")
+    if lower:
+        j.check("answer_names_remaining", contains_all(fa, [str(lower[0]), lower[1], lower[2]]),
+                f"expected remaining {lower[0]} {lower[1]} {lower[2]}; final={fa!r}")
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_14.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--14: sign in as carol.l, identify her active appraisal, then buy a
+2022 Honda CR-V applying that appraisal as a trade-in, CarMax Auto Finance, 60-month term,
+$3,000 down, 6.49% APR, no MaxCare; place the order and report the order number and total.
+
+Deterministic-first: DB after-state: carol has a NEW order for a 2022 Honda CR-V with the
+trade-in value applied, no MaxCare, 60-mo/6.49%/$3000-down; her appraisal is redeemed |
+answer contains the order number and total.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, final_answer, contains_any, price_mentioned,
+                        resolve_db, orders_for, appraisals_for, Judge, parse_args)
+
+EMAIL = "carol.l@test.com"
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--14', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    after = resolve_db(a.after_db, a.container, "instance")
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    ao = orders_for(after, EMAIL) or []
+    io = orders_for(init, EMAIL) or []
+    new_orders = [o for o in ao if o["order_number"] not in {x["order_number"] for x in io}]
+    crv = [o for o in new_orders if o["make"] == "Honda" and o["model"] == "CR-V" and o["year"] == 2022]
+    j.check("nav_checkout", navigated_to(t, "/checkout") or navigated_to(t, "/vehicle"),
+            "expected a /checkout flow")
+    j.check("db_new_crv_order", bool(crv), f"new orders={[o['order_number'] for o in new_orders]}")
+    if crv:
+        o = crv[-1]
+        j.check("order_trade_in_applied", (o["trade_in_value"] or 0) > 0,
+                f"trade_in_value={o['trade_in_value']}")
+        j.check("order_no_maxcare", (o["maxcare_plan"] or "") == "", f"maxcare_plan={o['maxcare_plan']!r}")
+        j.check("order_terms", int(o["payment_term_months"] or 0) == 60 and abs((o["payment_apr"] or 0) - 6.49) < 0.01
+                and abs((o["down_payment"] or 0) - 3000) < 1,
+                f"term={o['payment_term_months']} apr={o['payment_apr']} down={o['down_payment']}")
+        j.check("answer_order_number", contains_any(fa, [o["order_number"]]),
+                f"expected {o['order_number']}; final={fa!r}")
+        j.check("answer_total", price_mentioned(fa, int(round(o["total"])), tol=2),
+                f"expected total ${o['total']:,.2f}")
+    # appraisal should be redeemed after the trade-in
+    act_after = appraisals_for(after, EMAIL, "active") or []
+    j.check("appraisal_redeemed", len(act_after) == 0,
+            f"carol should have 0 active appraisals after trade-in; got {act_after}")
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_15.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--15: find the cheapest 2023 vehicle in stock, open its detail, then
+visit the store that has it; report (a) year/make/model/price, (b) store name & city,
+(c) whether that store offers home delivery.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, navigated_re, final_answer, contains_all,
+                        contains_any, price_mentioned, resolve_db, db_query, llm_text_match,
+                        Judge, parse_args)
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--15', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    gt = db_query(init, "SELECT v.year, v.make, v.model, v.price, s.name, s.city, s.has_home_delivery "
+                        "FROM vehicles v JOIN stores s ON s.id=v.store_id WHERE v.year=2023 "
+                        "ORDER BY v.price ASC LIMIT 1") if init else []
+    yr, mk, md, price, store, city, hd = gt[0] if gt else (2023, "Hyundai", "Elantra", 14400, "CarMax Seattle Lynnwood", "Lynnwood", 1)
+    hd_word = "yes" if hd else "no"
+    j.check("nav_vehicle_detail", navigated_re(t, r"/vehicle/"), "expected a vehicle detail page")
+    j.check("nav_store", navigated_to(t, "/store"), "expected a /store detail page")
+    j.check("answer_vehicle", contains_all(fa, [str(yr), mk, md]) and price_mentioned(fa, int(price)),
+            f"expected {yr} {mk} {md} ${int(price):,}; final={fa!r}")
+    j.check("answer_store", contains_any(fa, [store, city]), f"expected store {city!r}")
+    j.check("answer_home_delivery", contains_any(fa, [hd_word, "home delivery"]),
+            f"expected home delivery = {hd_word}")
+    ok, ev = llm_text_match(fa, f"{yr} {mk} {md} ${int(price):,}; store {store} in {city}; "
+                                f"home delivery: {hd_word}",
+                            "Cheapest 2023 vehicle, its store name/city, and whether the store offers home delivery.")
+    j.check("answer_consistent", ok, ev, llm=True)
+    j.emit()
+
+if __name__ == "__main__":
+    main()

sites/carmax/verify/verify_16.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""Verifier for CarMax--16: open the article 'Getting Pre-Qualified: Shop with Personalized
+Financing Terms' and, per the article, state the key difference between pre-qualification
+and pre-approval (one sentence).
+
+Open-ended answer → LLM-anchored on the ACTUAL article body (read from the DB at verify
+time), so the grader checks the answer against the site's own text, never model knowledge.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from verify_lib import (load_run, navigated_to, navigated_re, final_answer, contains_any,
+                        resolve_db, db_query, llm_text_match, Judge, parse_args)
+
+def main():
+    a = parse_args()
+    j = Judge('CarMax--16', a.no_llm)
+    t = load_run(a.run_dir); fa = final_answer(t)
+    init = resolve_db(a.initial_db, a.container, "instance_seed")
+    row = db_query(init, "SELECT slug, body FROM articles WHERE title LIKE 'Getting Pre-Qualified%'") if init else []
+    slug, body = (row[0][0], row[0][1]) if row else ("", "")
+    j.check("nav_article", (slug and navigated_to(t, slug)) or navigated_re(t, r"/articles/"),
+            "expected the Getting Pre-Qualified article page")
+    j.check("answer_nonempty", len(fa) > 0, f"final={fa!r}")
+    ok, ev = llm_text_match(
+        fa,
+        f"The key difference as stated in this article:\n{body[:1500]}",
+        "What is the key difference between pre-qualification and pre-approval at CarMax, "
+        "according to the article? (Judge the agent's one-sentence answer against the article text above.)")
+    j.check("answer_matches_article", ok, ev, llm=True)
+    j.emit()
+
+if __name__ == "__main__":
+    main()