From bcd1b9d6196a4425a67b8791c542f5c744adcd99 Mon Sep 17 00:00:00 2001 From: DEM1TASSE Date: Fri, 3 Jul 2026 00:41:16 +0000 Subject: [PATCH] Add CarMax task verifiers and judge rubrics Per-task deterministic verifiers (verify_.py) + shared verify_lib.py under sites/carmax/verify/, plus verifier_path + judge_rubric on each task in tasks.jsonl. Each verifier checks navigation (anti knowledge-shortcut), DB after-state for stateful tasks, and the answer against frozen ground truth. Validated against the react agent. Co-Authored-By: Claude Opus 4.8 (1M context) --- sites/carmax/tasks.jsonl | 20 ++ sites/carmax/verify/verify_0.py | 35 ++++ sites/carmax/verify/verify_1.py | 34 ++++ sites/carmax/verify/verify_10.py | 37 ++++ sites/carmax/verify/verify_11.py | 41 ++++ sites/carmax/verify/verify_12.py | 26 +++ sites/carmax/verify/verify_13.py | 45 +++++ sites/carmax/verify/verify_14.py | 49 +++++ sites/carmax/verify/verify_15.py | 36 ++++ sites/carmax/verify/verify_16.py | 33 ++++ sites/carmax/verify/verify_17.py | 40 ++++ sites/carmax/verify/verify_18.py | 34 ++++ sites/carmax/verify/verify_19.py | 36 ++++ sites/carmax/verify/verify_2.py | 30 +++ sites/carmax/verify/verify_3.py | 44 +++++ sites/carmax/verify/verify_4.py | 31 +++ sites/carmax/verify/verify_5.py | 36 ++++ sites/carmax/verify/verify_6.py | 38 ++++ sites/carmax/verify/verify_7.py | 39 ++++ sites/carmax/verify/verify_8.py | 31 +++ sites/carmax/verify/verify_9.py | 42 +++++ sites/carmax/verify/verify_lib.py | 301 ++++++++++++++++++++++++++++++ 22 files changed, 1058 insertions(+) create mode 100644 sites/carmax/tasks.jsonl create mode 100644 sites/carmax/verify/verify_0.py create mode 100644 sites/carmax/verify/verify_1.py create mode 100644 sites/carmax/verify/verify_10.py create mode 100644 sites/carmax/verify/verify_11.py create mode 100644 sites/carmax/verify/verify_12.py create mode 100644 sites/carmax/verify/verify_13.py create mode 100644 sites/carmax/verify/verify_14.py create mode 100644 sites/carmax/verify/verify_15.py create mode 100644 sites/carmax/verify/verify_16.py create mode 100644 sites/carmax/verify/verify_17.py create mode 100644 sites/carmax/verify/verify_18.py create mode 100644 sites/carmax/verify/verify_19.py create mode 100644 sites/carmax/verify/verify_2.py create mode 100644 sites/carmax/verify/verify_3.py create mode 100644 sites/carmax/verify/verify_4.py create mode 100644 sites/carmax/verify/verify_5.py create mode 100644 sites/carmax/verify/verify_6.py create mode 100644 sites/carmax/verify/verify_7.py create mode 100644 sites/carmax/verify/verify_8.py create mode 100644 sites/carmax/verify/verify_9.py create mode 100644 sites/carmax/verify/verify_lib.py diff --git a/sites/carmax/tasks.jsonl b/sites/carmax/tasks.jsonl new file mode 100644 index 00000000..3d685d65 --- /dev/null +++ b/sites/carmax/tasks.jsonl @@ -0,0 +1,20 @@ +{"web_name": "CarMax", "id": "CarMax--0", "ques": "Find any 2022 Honda Civic in the inventory and report its full title, price, and mileage.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_0.py", "judge_rubric": "FACT CHECKPOINTS: (1) trajectory MUST navigate to a 2022 Honda Civic entry page (URL like /vehicle/...2022-honda-civic...). (2) The answer MUST report that it is a Honda Civic AND its price ($15,300) AND its mileage (58,626 mi). FAIL if: no visit to a 2022 Civic detail page; answer empty; wrong/absent price or mileage."} +{"web_name": "CarMax", "id": "CarMax--1", "ques": "Search for a Toyota Tacoma TRD Off-Road in the inventory and report its store location, mileage, and asking price.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_1.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate to the Toyota Tacoma TRD Off-Road detail page. (2) Answer MUST report its store location (Lynnwood, WA / CarMax Seattle Lynnwood), mileage (91,787 mi) and asking price ($15,000). FAIL if: no visit to that vehicle; any of the three fields wrong or missing."} +{"web_name": "CarMax", "id": "CarMax--2", "ques": "Filter the inventory for AWD SUVs under $25,000 sorted by lowest price. Report the year, make, model, trim and price of the cheapest one.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_2.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST use the inventory filters (AWD + SUV + price under $25,000) — trajectory reaches /cars with those constraints. (2) Answer MUST name the cheapest such vehicle: 2019 Kia Sportage LX at $10,500 (year/make/model/trim/price). FAIL if: names a non-cheapest or non-AWD/non-SUV vehicle; wrong price."} +{"web_name": "CarMax", "id": "CarMax--3", "ques": "Search the inventory for a Tesla Model 3 with under 50,000 miles. Then sort the results by lowest mileage and open the detail page of the lowest-mileage one. Report its price, mileage, exterior color, and the store it's at.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_3.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST open a Tesla Model 3 with under 50,000 miles (lowest-mileage one). (2) Answer reports its price, mileage, exterior color and store. NOTE: the current inventory has NO Tesla Model 3 under 50,000 miles, so this task is UNSOLVABLE as written — mark success=false unless the environment is fixed (a sub-50k Tesla Model 3 seeded, or the constraint relaxed). FAIL if the reported car is not a Tesla Model 3 under 50k, or is a different make/model (e.g. a Toyota Tacoma surfaced by loose search)."} +{"web_name": "CarMax", "id": "CarMax--4", "ques": "Open the detail page for any 2022 Honda CR-V in inventory and report its horsepower, combined MPG, exterior color, and the store it is located at.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_4.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST open a 2022 Honda CR-V detail page. (2) Answer MUST report its horsepower (190), combined MPG (29), exterior color (Crystal Black Pearl) and store (Houston Katy). FAIL if: no visit to a 2022 CR-V; any field wrong/missing."} +{"web_name": "CarMax", "id": "CarMax--5", "ques": "On the 2022 Honda Civic research page, list every available trim, then report both the RepairPal reliability rating and the average customer rating shown on that page.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_5.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate to the 2022 Honda Civic research page (/research/honda/civic/2022). (2) Answer MUST list the available trim(s) (EX) AND report the RepairPal reliability rating (3.5) AND the average customer rating (4.5) shown on that page (the page shows the AVERAGE of that model-year's reviews, 4.5 — not the per-vehicle 4.0). FAIL if: not on the research page; missing the RepairPal rating or the customer rating; reports 4.0 instead of the page's 4.5."} +{"web_name": "CarMax", "id": "CarMax--6", "ques": "Add three vehicles to the comparison tool: a 2022 Honda Accord, a 2022 Toyota Camry, and a 2022 Nissan Altima. Then report which of the three has the most horsepower and which has the best combined MPG.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_6.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST add the 2022 Accord, Camry and Altima to the comparison tool and reach /compare showing all three. (2) Answer MUST correctly state the most horsepower = Toyota Camry (203 hp) and best combined MPG = Honda Accord (34 mpg). FAIL if: fewer than 3 cars compared; the HP/MPG winners are attributed wrongly."} +{"web_name": "CarMax", "id": "CarMax--7", "ques": "Get an instant offer to sell a 2018 Toyota Camry LE with 78,500 miles in good condition, ZIP 30303, no reported accidents, one previous owner. Report the dollar offer amount and the expiration date.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_7.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST complete the sell-my-car instant-offer form for a 2018 Toyota Camry LE, 78,500 mi, good condition (an appraisal is created). (2) Answer MUST report the offer amount **$4,850** AND the expiration date **2026-05-21**. FAIL if: no offer generated; offer amount is not $4,850; expiry is not 2026-05-21; either is absent."} +{"web_name": "CarMax", "id": "CarMax--8", "ques": "Go to the CarMax store locator and report (a) how many states have at least one CarMax store and (b) the street address of any CarMax store in California.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_8.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate the store locator. (2) Answer MUST report the number of states with at least one store (12) AND the street address (or city) of a California store (e.g. 6101 Auto Center Dr, Buena Park). FAIL if: wrong state count; no valid CA store address/city."} +{"web_name": "CarMax", "id": "CarMax--9", "ques": "Create a new CarMax account with first name Test, last name Buyer, email new.buyer.benchmark@test.com, phone 4045550199, ZIP 30303, password Welcome2026. After signing in, get pre-qualified with $80,000 annual income, employed full-time, $500 max monthly payment, $2,000 down, 72-month term, good credit. Report the estimated APR shown on the result page.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_9.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST register the new account (new.buyer.benchmark@test.com) AND complete the pre-qualification form (80k income, full-time, $500 max, $2,000 down, 72 months, good credit). (2) Answer MUST report the estimated APR **7.99%** (the value the site computes for good credit / 72-month term). FAIL if: no account created; no pre-qual completed; APR absent or not 7.99%."} +{"web_name": "CarMax", "id": "CarMax--10", "ques": "Sign in as alice.j@test.com with password CarMax!2026. Reserve any 2022 Toyota Camry for 7 days, with the appointment date 2026-05-20. Then go to your reservations page and confirm the reservation is listed as active.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_10.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST sign in as alice and create a reservation for a 2022 Toyota Camry with appointment date 2026-05-20. (2) The reservation MUST appear as ACTIVE on the reservations page (and exist in the DB). FAIL if: not signed in as alice; no 2022 Camry reservation; not active; wrong appointment date."} +{"web_name": "CarMax", "id": "CarMax--11", "ques": "Sign in as bob.k@test.com with password CarMax!2026. Schedule an at-home test drive for any 2022 Ford F-150 on 2026-05-22 at 2:00 PM, leaving a note that says 'Please call gate buzzer 4B'. Confirm the test drive shows on your test drives page.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_11.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST sign in as bob.k and schedule an AT-HOME test drive for a 2022 Ford F-150 on 2026-05-22 at 2:00 PM with the note 'Please call gate buzzer 4B'. (2) The test drive MUST appear on the test-drives page (and exist in the DB with location_type=at_home, that date/time and note). FAIL if: not bob.k; wrong vehicle/date/time; not at-home; note missing."} +{"web_name": "CarMax", "id": "CarMax--12", "ques": "Find the answer in the CarMax FAQ to the question 'How long is my appraisal offer good for?' and report the number of days the offer is valid.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_12.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate to the FAQ. (2) Answer MUST report that a CarMax appraisal offer is valid for 7 days. FAIL if: not on the FAQ; wrong number of days."} +{"web_name": "CarMax", "id": "CarMax--13", "ques": "Sign in as alice.j@test.com with password CarMax!2026. You currently have two saved cars from different makes; remove the saved vehicle with the higher mileage. Then report the year, make, model, and store location of the remaining saved vehicle.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_13.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST sign in as alice and remove the HIGHER-mileage saved car (the 2020 Honda Civic, 69,000 mi), leaving the lower-mileage one (2021 Honda CR-V, 57,824 mi). (2) Answer MUST report the remaining car as the **2021 Honda CR-V** and its store **CarMax Atlanta Southlake, Morrow, GA**. NOTE: the task text says 'different makes' but both saved cars are Honda — judge on the remove-higher-mileage action, not the wording. FAIL if: wrong car removed; answer names the removed 2020 Civic; store wrong/missing."} +{"web_name": "CarMax", "id": "CarMax--14", "ques": "Sign in as carol.l@test.com with password CarMax!2026. You have one active sell-my-car appraisal offer; identify which vehicle it is for and the offer amount. Then start checkout on any 2022 Honda CR-V, apply that appraisal as a trade-in, choose CarMax Auto Finance with a 60-month term, $3,000 down, 6.49% APR, no MaxCare, and place the order. Report the final order number and total amount.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_14.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST sign in as carol.l and place an order for a 2022 Honda CR-V, applying her active appraisal (2019 Nissan Altima, $14,750) as a trade-in, CarMax Auto Finance, 60-month term, $3,000 down, 6.49% APR, no MaxCare. (2) The order MUST exist in the DB with the trade-in applied, no MaxCare, those finance terms; her appraisal becomes redeemed. (3) Answer MUST report a valid order number (format CMX-YYYY-NNNNNN) AND the total amount **$5,431.00** (subtotal $18,800 minus the $14,750 trade-in plus tax/fees — NOT $18,800). FAIL if: no order; MaxCare included; wrong terms; trade-in not applied; total is not ~$5,431 (e.g. reports the $18,800 subtotal); order number absent."} +{"web_name": "CarMax", "id": "CarMax--15", "ques": "Find the cheapest 2023 vehicle currently in stock. Open its detail page, then visit the store that has it. Report (a) the vehicle's year, make, model, and price, (b) the store name and city, and (c) whether that store offers home delivery.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_15.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST find the cheapest 2023 vehicle (2023 Hyundai Elantra SEL, $14,400), open its detail, and visit its store. (2) Answer MUST report (a) the vehicle year/make/model/price, (b) the store name/city (CarMax Seattle Lynnwood, Lynnwood), (c) whether that store offers home delivery (yes). FAIL if: not the cheapest 2023; store or home-delivery info wrong/missing."} +{"web_name": "CarMax", "id": "CarMax--16", "ques": "Open the article titled 'Getting Pre-Qualified: Shop with Personalized Financing Terms'. According to the article, what is the key difference between pre-qualification and pre-approval at CarMax? Answer in one sentence.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_16.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST open the article 'Getting Pre-Qualified: Shop with Personalized Financing Terms'. (2) Answer MUST capture the article's stated key difference: a **pre-qualification uses a SOFT credit inquiry and does NOT impact your credit score** (and lets you shop with personalized terms), whereas committing to final terms requires a credit application that is a **HARD inquiry**. Judge the answer against the article body, not outside knowledge. NOTE: the article frames the contrast as pre-qualification vs the final credit application (hard inquiry) — it does not use the word 'pre-approval'; accept an answer that captures the soft-vs-hard-inquiry / no-credit-impact distinction. FAIL if: article not opened; answer misses the soft-inquiry / no-credit-impact point or contradicts the article."} +{"web_name": "CarMax", "id": "CarMax--17", "ques": "Sign in as dan.m@test.com with password CarMax!2026. Find your account's order history and report (a) the order number, (b) the vehicle year/make/model, (c) the total amount, (d) whether MaxCare coverage was included, and (e) the scheduled pickup date.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_17.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST sign in as dan.m and open order history. (2) Answer MUST report (a) order number **CMX-2026-000001**, (b) vehicle **2023 Toyota Tacoma TRD Sport**, (c) total **$26,686** as shown on the page (the page rounds the exact $26,685.50 to $26,686 — accept either), (d) MaxCare **included (Gold)**, (e) pickup date **2026-05-17**. FAIL if: not dan.m; any of the five fields wrong or missing."} +{"web_name": "CarMax", "id": "CarMax--18", "ques": "Visit the CarMax used car value page for the 2020 Honda Accord. Report (a) the CarMax average price across current inventory, (b) the price range (lowest to highest), and (c) the number of 2020 Honda Accords currently in stock.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_18.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate to the 2020 Honda Accord used-car-value page. (2) Answer MUST report (a) the average price ($13,000), (b) the price range low-to-high ($13,000 to $13,000 — both in stock are identically priced), (c) the count in stock (2). FAIL if: not on the value page; wrong average/range/count."} +{"web_name": "CarMax", "id": "CarMax--19", "ques": "Open the MaxCare extended service plans page. Compare the Silver, Gold, and Platinum tiers. Report (a) the one-time price of each tier, (b) the price difference between Gold and Silver, and (c) the maximum coverage period (months / miles) of the Platinum plan.", "web": "http://localhost:40015/", "upstream_url": "https://www.carmax.com/", "verifier_path": "sites/carmax/verify/verify_19.py", "judge_rubric": "FACT CHECKPOINTS: (1) MUST navigate to the MaxCare service-plans page. (2) Answer MUST report the one-time price of each tier (Silver $1,495, Gold $1,895, Platinum $2,395), the Gold-minus-Silver difference ($400), and the Platinum maximum coverage (60 months / 100,000 miles). FAIL if: not on the MaxCare page; any price/difference/coverage figure wrong or missing."} diff --git a/sites/carmax/verify/verify_0.py b/sites/carmax/verify/verify_0.py new file mode 100644 index 00000000..59db25b5 --- /dev/null +++ b/sites/carmax/verify/verify_0.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--0: find any 2022 Honda Civic; report full title, price, mileage. + +Deterministic-first: nav to a 2022 Honda Civic detail | answer has make/model + price + +mileage of the (unique) 2022 Civic in inventory | LLM-anchored fallback. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, navigated_re, final_answer, last_shot, + contains_all, price_mentioned, resolve_db, db_query, + llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--0', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + gt = db_query(init, "SELECT year,make,model,trim,price,mileage FROM vehicles " + "WHERE year=2022 AND make='Honda' AND model='Civic'") if init else [] + yr, mk, md, tr, price, mi = gt[0] if gt else (2022, "Honda", "Civic", "EX", 15300, 58626) + # Title/price/mileage are all shown on the inventory card, so a search/results page + # OR the detail page both count as real on-site viewing (anti knowledge-shortcut). + j.check("nav_civic_on_site", + navigated_re(t, r"(?i)/(vehicle|cars).*civic"), + f"expected a /cars?...Civic search or /vehicle/...honda-civic page; urls={[s.get('url') for s in t.get('steps', [])]}") + j.check("answer_make_model", contains_all(fa, ["Honda", "Civic"]), f"final={fa!r}") + j.check("answer_price", price_mentioned(fa, int(price)), f"expected ${int(price):,} in {fa!r}") + j.check("answer_mileage", price_mentioned(fa, int(mi)), f"expected {int(mi):,} mi in {fa!r}") + ok, ev = llm_text_match(fa, f"{yr} {mk} {md} {tr}, price ${int(price):,}, {int(mi):,} miles", + "Report the full title, price and mileage of a 2022 Honda Civic.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_1.py b/sites/carmax/verify/verify_1.py new file mode 100644 index 00000000..64a20e08 --- /dev/null +++ b/sites/carmax/verify/verify_1.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--1: find a Toyota Tacoma TRD Off-Road; report store, mileage, price. + +Deterministic-first: nav to the Tacoma TRD Off-Road detail | answer has store + mileage + price. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, navigated_re, final_answer, + contains_any, price_mentioned, resolve_db, db_query, + llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--1', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + gt = db_query(init, "SELECT v.mileage, v.price, s.name, s.city FROM vehicles v " + "JOIN stores s ON s.id=v.store_id WHERE v.make='Toyota' AND v.model='Tacoma' " + "AND v.trim LIKE '%TRD Off-Road%'") if init else [] + mi, price, store, city = gt[0] if gt else (91787, 15000, "CarMax Seattle Lynnwood", "Lynnwood") + # store/mileage/price are shown on the inventory card, so a search/results page OR + # the detail page both count as real on-site viewing (anti knowledge-shortcut). + j.check("nav_tacoma_on_site", navigated_re(t, r"(?i)/(vehicle|cars).*tacoma"), + f"expected a /cars?...Tacoma search or /vehicle/...tacoma page; urls={[s.get('url') for s in t.get('steps', [])]}") + j.check("answer_store", contains_any(fa, [store, city]), f"expected {city!r}; final={fa!r}") + j.check("answer_price", price_mentioned(fa, int(price)), f"expected ${int(price):,}") + j.check("answer_mileage", price_mentioned(fa, int(mi)), f"expected {int(mi):,} mi") + ok, ev = llm_text_match(fa, f"store {store} ({city}), {int(mi):,} miles, price ${int(price):,}", + "Report the Toyota Tacoma TRD Off-Road's store location, mileage and price.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_10.py b/sites/carmax/verify/verify_10.py new file mode 100644 index 00000000..8904381b --- /dev/null +++ b/sites/carmax/verify/verify_10.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--10: sign in as alice, reserve any 2022 Toyota Camry for 7 days +with appointment date 2026-05-20, then confirm it is listed as active. + +Deterministic-first: nav login + reserve + reservations page | DB after-state: alice has an +ACTIVE reservation for a 2022 Toyota Camry appt 2026-05-20 that was NOT in the initial seed. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, resolve_db, + reservations_for, Judge, parse_args) + +EMAIL = "alice.j@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--10', a.no_llm) + t = load_run(a.run_dir) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + aft = reservations_for(after, EMAIL, "active") or [] + ini = reservations_for(init, EMAIL, "active") or [] + def is_target(r): # (year, make, model, appointment_date, status) + return r[0] == 2022 and r[1] == "Toyota" and r[2] == "Camry" and str(r[3]) == "2026-05-20" + new_camry = [r for r in aft if is_target(r) and r not in ini] + j.check("nav_login", navigated_to(t, "/login"), "expected /login") + j.check("nav_reserve", navigated_to(t, "/reserve"), "expected a /reserve flow") + j.check("nav_reservations", navigated_to(t, "/account/reservations"), "expected reservations page") + j.check("db_active_2022_camry_reservation_2026_05_20", bool(new_camry), + f"after_active={aft} initial_active={ini}") + fa = final_answer(t) + j.check("answer_confirms_reservation", contains_any(fa, ["Camry", "reserved", "reservation", "active"]), + f"expected the answer to confirm the active 2022 Camry reservation; final={fa!r}") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_11.py b/sites/carmax/verify/verify_11.py new file mode 100644 index 00000000..e3adc403 --- /dev/null +++ b/sites/carmax/verify/verify_11.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--11: sign in as bob.k, schedule an at-home test drive for any +2022 Ford F-150 on 2026-05-22 at 2:00 PM with note 'Please call gate buzzer 4B', +then confirm it shows on the test drives page. + +Deterministic-first: nav login + test-drive + test-drives page | DB after-state: bob.k has an +at_home test drive for a 2022 Ford F-150 on 2026-05-22 2:00 PM with the note, not in seed. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, resolve_db, + test_drives_for, norm, Judge, parse_args) + +EMAIL = "bob.k@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--11', a.no_llm) + t = load_run(a.run_dir) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + aft = test_drives_for(after, EMAIL) or [] + ini = test_drives_for(init, EMAIL) or [] + # row: (year, make, model, location_type, scheduled_date, scheduled_time, notes, status) + def is_target(r): + return (r[0] == 2022 and r[1] == "Ford" and r[2] == "F-150" and r[3] == "at_home" + and str(r[4]) == "2026-05-22" and "2:00" in (r[5] or "") + and "gate buzzer 4b" in norm(r[6])) + new_td = [r for r in aft if is_target(r) and r not in ini] + j.check("nav_login", navigated_to(t, "/login"), "expected /login") + j.check("nav_test_drive", navigated_to(t, "/test-drive"), "expected a /test-drive flow") + j.check("nav_test_drives_page", navigated_to(t, "/account/test-drives"), "expected test-drives page") + j.check("db_at_home_f150_testdrive_with_note", bool(new_td), + f"after={aft} initial={ini}") + fa = final_answer(t) + j.check("answer_confirms_testdrive", contains_any(fa, ["F-150", "test drive", "test-drive", "scheduled"]), + f"expected the answer to confirm the F-150 at-home test drive; final={fa!r}") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_12.py b/sites/carmax/verify/verify_12.py new file mode 100644 index 00000000..0e144d1f --- /dev/null +++ b/sites/carmax/verify/verify_12.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--12: find the FAQ answer to 'How long is my appraisal offer good +for?' and report the number of days it is valid. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, price_mentioned, contains_any, + llm_text_match, Judge, parse_args) + +# CarMax appraisal offers are valid 7 days (matches reserve/offer logic: created + 7d). +DAYS = 7 + +def main(): + a = parse_args() + j = Judge('CarMax--12', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + j.check("nav_faq", navigated_to(t, "/faq"), "expected a /faq page") + j.check("answer_days", price_mentioned(fa, DAYS) or contains_any(fa, ["7 days", "seven days"]), + f"expected {DAYS} days; final={fa!r}") + ok, ev = llm_text_match(fa, f"the appraisal offer is valid for {DAYS} days", + "How many days is a CarMax appraisal offer good for?") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_13.py b/sites/carmax/verify/verify_13.py new file mode 100644 index 00000000..730a1bf8 --- /dev/null +++ b/sites/carmax/verify/verify_13.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--13: sign in as alice (who has two saved cars), remove the saved +vehicle with the HIGHER mileage, then report the year/make/model/store of the remaining one. + +Deterministic-first: nav login + saved | DB after-state: the higher-mileage saved car is +gone and the lower-mileage one remains | answer names the remaining car. + +Note (review): the task text says the two saved cars are "from different makes", but in the +seed both are Honda (2020 Civic 69k mi, 2021 CR-V 57.8k mi). The remove-higher-mileage step +is still deterministic; this verifier checks the actual data, not the (incorrect) wording. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_all, resolve_db, + saved_vehicles_for, Judge, parse_args) + +EMAIL = "alice.j@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--13', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + ini = saved_vehicles_for(init, EMAIL) or [] # rows: (year, make, model, trim, mileage) sorted by mileage + aft = saved_vehicles_for(after, EMAIL) or [] + higher = max(ini, key=lambda r: r[4]) if ini else None # should be removed + lower = min(ini, key=lambda r: r[4]) if ini else None # should remain + j.check("nav_login", navigated_to(t, "/login"), "expected /login") + # Saved cars are shown on both /saved and the /account page, so either is a valid path. + j.check("nav_saved_or_account", navigated_to(t, "/saved") or navigated_to(t, "/account"), + f"expected /saved or /account; urls={[s.get('url') for s in t.get('steps', [])]}") + j.check("db_one_saved_remains", len(aft) == max(0, len(ini) - 1), + f"initial_saved={len(ini)} after_saved={len(aft)}") + j.check("db_higher_mileage_removed", bool(higher) and higher not in aft, + f"higher-mileage car {higher} should be removed; after={aft}") + j.check("db_lower_mileage_remains", bool(lower) and lower in aft, + f"lower-mileage car {lower} should remain; after={aft}") + if lower: + j.check("answer_names_remaining", contains_all(fa, [str(lower[0]), lower[1], lower[2]]), + f"expected remaining {lower[0]} {lower[1]} {lower[2]}; final={fa!r}") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_14.py b/sites/carmax/verify/verify_14.py new file mode 100644 index 00000000..2d13c10c --- /dev/null +++ b/sites/carmax/verify/verify_14.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--14: sign in as carol.l, identify her active appraisal, then buy a +2022 Honda CR-V applying that appraisal as a trade-in, CarMax Auto Finance, 60-month term, +$3,000 down, 6.49% APR, no MaxCare; place the order and report the order number and total. + +Deterministic-first: DB after-state: carol has a NEW order for a 2022 Honda CR-V with the +trade-in value applied, no MaxCare, 60-mo/6.49%/$3000-down; her appraisal is redeemed | +answer contains the order number and total. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, price_mentioned, + resolve_db, orders_for, appraisals_for, Judge, parse_args) + +EMAIL = "carol.l@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--14', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + ao = orders_for(after, EMAIL) or [] + io = orders_for(init, EMAIL) or [] + new_orders = [o for o in ao if o["order_number"] not in {x["order_number"] for x in io}] + crv = [o for o in new_orders if o["make"] == "Honda" and o["model"] == "CR-V" and o["year"] == 2022] + j.check("nav_checkout", navigated_to(t, "/checkout") or navigated_to(t, "/vehicle"), + "expected a /checkout flow") + j.check("db_new_crv_order", bool(crv), f"new orders={[o['order_number'] for o in new_orders]}") + if crv: + o = crv[-1] + j.check("order_trade_in_applied", (o["trade_in_value"] or 0) > 0, + f"trade_in_value={o['trade_in_value']}") + j.check("order_no_maxcare", (o["maxcare_plan"] or "") == "", f"maxcare_plan={o['maxcare_plan']!r}") + j.check("order_terms", int(o["payment_term_months"] or 0) == 60 and abs((o["payment_apr"] or 0) - 6.49) < 0.01 + and abs((o["down_payment"] or 0) - 3000) < 1, + f"term={o['payment_term_months']} apr={o['payment_apr']} down={o['down_payment']}") + j.check("answer_order_number", contains_any(fa, [o["order_number"]]), + f"expected {o['order_number']}; final={fa!r}") + j.check("answer_total", price_mentioned(fa, int(round(o["total"])), tol=2), + f"expected total ${o['total']:,.2f}") + # appraisal should be redeemed after the trade-in + act_after = appraisals_for(after, EMAIL, "active") or [] + j.check("appraisal_redeemed", len(act_after) == 0, + f"carol should have 0 active appraisals after trade-in; got {act_after}") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_15.py b/sites/carmax/verify/verify_15.py new file mode 100644 index 00000000..53b8a48a --- /dev/null +++ b/sites/carmax/verify/verify_15.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--15: find the cheapest 2023 vehicle in stock, open its detail, then +visit the store that has it; report (a) year/make/model/price, (b) store name & city, +(c) whether that store offers home delivery. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, navigated_re, final_answer, contains_all, + contains_any, price_mentioned, resolve_db, db_query, llm_text_match, + Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--15', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + gt = db_query(init, "SELECT v.year, v.make, v.model, v.price, s.name, s.city, s.has_home_delivery " + "FROM vehicles v JOIN stores s ON s.id=v.store_id WHERE v.year=2023 " + "ORDER BY v.price ASC LIMIT 1") if init else [] + yr, mk, md, price, store, city, hd = gt[0] if gt else (2023, "Hyundai", "Elantra", 14400, "CarMax Seattle Lynnwood", "Lynnwood", 1) + hd_word = "yes" if hd else "no" + j.check("nav_vehicle_detail", navigated_re(t, r"/vehicle/"), "expected a vehicle detail page") + j.check("nav_store", navigated_to(t, "/store"), "expected a /store detail page") + j.check("answer_vehicle", contains_all(fa, [str(yr), mk, md]) and price_mentioned(fa, int(price)), + f"expected {yr} {mk} {md} ${int(price):,}; final={fa!r}") + j.check("answer_store", contains_any(fa, [store, city]), f"expected store {city!r}") + j.check("answer_home_delivery", contains_any(fa, [hd_word, "home delivery"]), + f"expected home delivery = {hd_word}") + ok, ev = llm_text_match(fa, f"{yr} {mk} {md} ${int(price):,}; store {store} in {city}; " + f"home delivery: {hd_word}", + "Cheapest 2023 vehicle, its store name/city, and whether the store offers home delivery.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_16.py b/sites/carmax/verify/verify_16.py new file mode 100644 index 00000000..822d7ab2 --- /dev/null +++ b/sites/carmax/verify/verify_16.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--16: open the article 'Getting Pre-Qualified: Shop with Personalized +Financing Terms' and, per the article, state the key difference between pre-qualification +and pre-approval (one sentence). + +Open-ended answer → LLM-anchored on the ACTUAL article body (read from the DB at verify +time), so the grader checks the answer against the site's own text, never model knowledge. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, navigated_re, final_answer, contains_any, + resolve_db, db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--16', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + row = db_query(init, "SELECT slug, body FROM articles WHERE title LIKE 'Getting Pre-Qualified%'") if init else [] + slug, body = (row[0][0], row[0][1]) if row else ("", "") + j.check("nav_article", (slug and navigated_to(t, slug)) or navigated_re(t, r"/articles/"), + "expected the Getting Pre-Qualified article page") + j.check("answer_nonempty", len(fa) > 0, f"final={fa!r}") + ok, ev = llm_text_match( + fa, + f"The key difference as stated in this article:\n{body[:1500]}", + "What is the key difference between pre-qualification and pre-approval at CarMax, " + "according to the article? (Judge the agent's one-sentence answer against the article text above.)") + j.check("answer_matches_article", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_17.py b/sites/carmax/verify/verify_17.py new file mode 100644 index 00000000..46a12055 --- /dev/null +++ b/sites/carmax/verify/verify_17.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--17: sign in as dan.m and report from order history (a) order number, +(b) vehicle year/make/model, (c) total amount, (d) whether MaxCare was included, +(e) the scheduled pickup date. (Read-only: the order is in the seed.) +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_all, contains_any, + price_mentioned, resolve_db, orders_for, llm_text_match, Judge, parse_args) + +EMAIL = "dan.m@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--17', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + orders = orders_for(init, EMAIL) or [] + j.check("nav_login", navigated_to(t, "/login"), "expected /login") + j.check("nav_orders", navigated_to(t, "/account/orders") or navigated_to(t, "/order"), + "expected the order-history / order page") + j.check("db_has_order", bool(orders), f"dan.m orders={[o['order_number'] for o in orders]}") + if orders: + o = orders[0] + has_maxcare = bool(o["maxcare_plan"]) + mc_words = ["maxcare", o["maxcare_plan"], "yes", "included"] if has_maxcare else ["no maxcare", "not included", "no"] + j.check("answer_order_number", contains_any(fa, [o["order_number"]]), f"expected {o['order_number']}; final={fa!r}") + j.check("answer_vehicle", contains_all(fa, [str(o["year"]), o["make"], o["model"]]), + f"expected {o['year']} {o['make']} {o['model']}") + j.check("answer_total", price_mentioned(fa, int(round(o["total"])), tol=2), f"expected ${o['total']:,.2f}") + j.check("answer_maxcare", contains_any(fa, [w for w in mc_words if w]), + f"maxcare included={has_maxcare} ({o['maxcare_plan']!r})") + j.check("answer_pickup_date", contains_any(fa, [str(o["pickup_date"])]), f"expected pickup {o['pickup_date']}") + # The five deterministic field checks above fully and precisely verify this + # read-only task; an extra LLM-anchored check only adds false negatives (e.g. it + # rejected a correct answer that rounded $26,685.50 to $26,686), so it is omitted. + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_18.py b/sites/carmax/verify/verify_18.py new file mode 100644 index 00000000..7cc8b00f --- /dev/null +++ b/sites/carmax/verify/verify_18.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--18: on the used-car value page for the 2020 Honda Accord, report +(a) the CarMax average price, (b) the price range (lowest to highest), (c) the number of +2020 Honda Accords currently in stock. + +Note (review): both 2020 Accords are priced identically ($13,000), so the 'range' is +degenerate ($13,000–$13,000). The verifier checks the true aggregates from the DB. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, price_mentioned, resolve_db, + db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--18', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + row = db_query(init, "SELECT COUNT(*), MIN(price), MAX(price), ROUND(AVG(price)) FROM vehicles " + "WHERE year=2020 AND make='Honda' AND model='Accord'") if init else [] + cnt, lo, hi, avg = row[0] if row else (2, 13000, 13000, 13000) + j.check("nav_value_page", navigated_to(t, "/value/honda/accord/2020") + or navigated_to(t, "/value/honda/accord"), "expected the 2020 Honda Accord value page") + j.check("answer_avg", price_mentioned(fa, int(avg)), f"expected avg ${int(avg):,}; final={fa!r}") + j.check("answer_range", price_mentioned(fa, int(lo)) and price_mentioned(fa, int(hi)), + f"expected range ${int(lo):,}-${int(hi):,}") + j.check("answer_count", price_mentioned(fa, int(cnt)), f"expected count {cnt} in stock") + ok, ev = llm_text_match(fa, f"average ${int(avg):,}; range ${int(lo):,} to ${int(hi):,}; {cnt} in stock", + "2020 Honda Accord: average price, price range (low-high), and count in stock.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_19.py b/sites/carmax/verify/verify_19.py new file mode 100644 index 00000000..3bf55a1e --- /dev/null +++ b/sites/carmax/verify/verify_19.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--19: on the MaxCare extended-service-plans page, compare the Silver, +Gold and Platinum tiers; report (a) the one-time price of each, (b) the price difference +between Gold and Silver, (c) the maximum coverage period (months / miles) of Platinum. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, price_mentioned, contains_any, + llm_text_match, Judge, parse_args) + +# Frozen from the MaxCare page (see review): Silver $1,495 (36mo/50k), Gold $1,895 (48mo/75k), +# Platinum $2,395 (60mo/100k). Gold - Silver = $400. +SILVER, GOLD, PLAT = 1495, 1895, 2395 +DIFF = GOLD - SILVER # 400 +PLAT_MONTHS, PLAT_MILES = 60, 100000 + +def main(): + a = parse_args() + j = Judge('CarMax--19', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + j.check("nav_maxcare", navigated_to(t, "maxcare-service-plans") or navigated_to(t, "maxcare"), + "expected the MaxCare plans page") + j.check("answer_tier_prices", all(price_mentioned(fa, p) for p in (SILVER, GOLD, PLAT)), + f"expected ${SILVER:,}/${GOLD:,}/${PLAT:,}; final={fa!r}") + j.check("answer_gold_minus_silver", price_mentioned(fa, DIFF), f"expected diff ${DIFF}") + j.check("answer_platinum_coverage", + price_mentioned(fa, PLAT_MONTHS) and (price_mentioned(fa, PLAT_MILES) or contains_any(fa, ["100k", "100,000"])), + f"expected Platinum {PLAT_MONTHS} months / {PLAT_MILES:,} miles") + ok, ev = llm_text_match(fa, f"Silver ${SILVER:,}, Gold ${GOLD:,}, Platinum ${PLAT:,}; " + f"Gold-Silver = ${DIFF}; Platinum covers {PLAT_MONTHS} months / {PLAT_MILES:,} miles", + "MaxCare tier prices, Gold-Silver difference, and Platinum max coverage.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_2.py b/sites/carmax/verify/verify_2.py new file mode 100644 index 00000000..02c210ae --- /dev/null +++ b/sites/carmax/verify/verify_2.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--2: filter AWD SUVs under $25,000 sorted by lowest price; +report year/make/model/trim/price of the cheapest. + +Deterministic-first: nav to filtered inventory | answer names the true cheapest AWD SUV. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_all, + price_mentioned, resolve_db, db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--2', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + gt = db_query(init, "SELECT year,make,model,trim,price FROM vehicles WHERE body_style='SUV' " + "AND drive_type='AWD' AND price<25000 ORDER BY price ASC LIMIT 1") if init else [] + yr, mk, md, tr, price = gt[0] if gt else (2019, "Kia", "Sportage", "LX", 10500) + j.check("nav_inventory_filtered", navigated_to(t, "/cars"), "expected an inventory /cars page") + j.check("answer_names_cheapest", contains_all(fa, [str(yr), mk, md]), f"expected {yr} {mk} {md}; final={fa!r}") + j.check("answer_trim", contains_all(fa, [tr]), f"expected trim {tr!r}") + j.check("answer_price", price_mentioned(fa, int(price)), f"expected ${int(price):,}") + ok, ev = llm_text_match(fa, f"{yr} {mk} {md} {tr}, ${int(price):,}", + "The cheapest AWD SUV under $25,000 (year/make/model/trim/price).") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_3.py b/sites/carmax/verify/verify_3.py new file mode 100644 index 00000000..352318e9 --- /dev/null +++ b/sites/carmax/verify/verify_3.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--3: search a Tesla Model 3 with UNDER 50,000 miles, sort by +lowest mileage, open the lowest-mileage one, report price/mileage/exterior color/store. + +*** BLOCKED — TASK CURRENTLY UNSOLVABLE *** +Inventory has no Tesla Model 3 under 50,000 miles (lowest is 52,838), so there is no +correct target to open and no correct answer. This verifier encodes the task's INTENDED +requirement, so it will (correctly) FAIL until the env is fixed — either seed a sub-50k +Tesla Model 3 or relax the task to e.g. <60,000 (see review finding A1). Once fixed, the +checks below become satisfiable unchanged. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_re, final_answer, resolve_db, db_query, + contains_any, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--3', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + elig = db_query(init, "SELECT year,trim,mileage,price,exterior_color," + "(SELECT city FROM stores WHERE id=store_id) " + "FROM vehicles WHERE make='Tesla' AND model='Model 3' AND mileage<50000 " + "ORDER BY mileage ASC LIMIT 1") if init else [] + # (1) Precondition: a sub-50k Tesla Model 3 must EXIST to be openable. Fails today. + j.check("env_has_tesla_model3_under_50k", bool(elig), + "no Tesla Model 3 under 50,000 mi in inventory — task is unsolvable until seeded/relaxed") + # (2) Agent must have opened a Tesla Model 3 detail page. + j.check("nav_tesla_model3_detail", navigated_re(t, r"/vehicle/.*tesla-model-3"), + "expected a Tesla Model 3 /vehicle/ page") + # (3) Answer must report the eligible car's details (only checkable once one exists). + if elig: + yr, tr, mi, price, color, city = elig[0] + ok, ev = llm_text_match(fa, f"{yr} Tesla Model 3 {tr}, ${int(price):,}, {int(mi):,} mi, " + f"{color}, store city {city}", + "Report the lowest-mileage sub-50k Tesla Model 3's price/mileage/color/store.") + j.check("answer_consistent", ok, ev, llm=True) + else: + j.check("answer_consistent", False, "no eligible vehicle to anchor the answer against") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_4.py b/sites/carmax/verify/verify_4.py new file mode 100644 index 00000000..872fe888 --- /dev/null +++ b/sites/carmax/verify/verify_4.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--4: open any 2022 Honda CR-V detail; report horsepower, +combined MPG, exterior color, and its store. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_re, navigated_to, final_answer, contains_any, + price_mentioned, resolve_db, db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--4', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + gt = db_query(init, "SELECT v.horsepower, v.mpg_combined, v.exterior_color, s.name, s.city " + "FROM vehicles v JOIN stores s ON s.id=v.store_id " + "WHERE v.year=2022 AND v.make='Honda' AND v.model='CR-V'") if init else [] + hp, mpg, color, store, city = gt[0] if gt else (190, 29, "Crystal Black Pearl", "CarMax Houston Katy", "Katy") + j.check("nav_2022_crv_detail", navigated_re(t, r"/vehicle/.*2022-honda-cr-v") or navigated_to(t, "honda-cr-v"), + "expected a 2022 Honda CR-V /vehicle/ page") + j.check("answer_hp", price_mentioned(fa, int(hp)), f"expected {hp} hp; final={fa!r}") + j.check("answer_mpg", price_mentioned(fa, int(mpg)), f"expected {mpg} mpg") + j.check("answer_color", contains_any(fa, [color]), f"expected color {color!r}") + j.check("answer_store", contains_any(fa, [store, city]), f"expected store {city!r}") + ok, ev = llm_text_match(fa, f"{hp} hp, {mpg} mpg combined, {color}, store {store} ({city})", + "Report a 2022 Honda CR-V's horsepower, combined MPG, exterior color, store.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_5.py b/sites/carmax/verify/verify_5.py new file mode 100644 index 00000000..5cdc3858 --- /dev/null +++ b/sites/carmax/verify/verify_5.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--5: on the 2022 Honda Civic research page, list every available +trim, then report the RepairPal reliability rating and the average customer rating. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, price_mentioned, + resolve_db, db_query, last_shot, llm_text_match, llm_screenshot_shows, + Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--5', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + rows = db_query(init, "SELECT DISTINCT trim, repairpal_rating, customer_rating FROM vehicles " + "WHERE year=2022 AND make='Honda' AND model='Civic'") if init else [] + trims = sorted({r[0] for r in rows}) or ["EX"] + repairpal = rows[0][1] if rows else 3.5 + # The research page shows the AVERAGE of this make/model/year's reviews when any + # exist, else the vehicle's customer_rating (mirrors research_model_year in app.py). + rev = db_query(init, "SELECT ROUND(AVG(rating), 1) FROM reviews " + "WHERE make_slug='honda' AND model_slug='civic' AND year=2022") if init else [] + cust = (rev[0][0] if rev and rev[0][0] is not None else (rows[0][2] if rows else 4.0)) + j.check("nav_research_civic_2022", navigated_to(t, "/research/honda/civic/2022"), + "expected /research/honda/civic/2022") + j.check("answer_lists_a_trim", contains_any(fa, trims), f"expected a trim from {trims}; final={fa!r}") + j.check("answer_repairpal", contains_any(fa, [str(repairpal)]), f"expected RepairPal {repairpal}") + j.check("answer_customer_rating", contains_any(fa, [str(cust)]), f"expected customer rating {cust}") + ok, ev = llm_text_match(fa, f"trims: {trims}; RepairPal reliability {repairpal}; avg customer rating {cust}", + "List the 2022 Honda Civic trims and report the RepairPal rating and avg customer rating.") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_6.py b/sites/carmax/verify/verify_6.py new file mode 100644 index 00000000..c0631d7a --- /dev/null +++ b/sites/carmax/verify/verify_6.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--6: add a 2022 Accord, Camry, Altima to the comparison tool; +report which has the most horsepower and which the best combined MPG. + +Deterministic-first: nav to /compare (and to the 3 vehicles) | answer attributes most-HP +and best-MPG correctly. Compare is session-scoped, so the DB isn't a reliable per-user +signal here; the check is nav + LLM-anchored on the frozen HP/MPG ground truth. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, resolve_db, db_query, + llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--6', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + specs = {} + for mk, md in [("Honda", "Accord"), ("Toyota", "Camry"), ("Nissan", "Altima")]: + r = db_query(init, "SELECT horsepower, mpg_combined FROM vehicles WHERE year=2022 AND make=? AND model=?", + (mk, md)) if init else [] + specs[md] = r[0] if r else None + # Ground truth from data: most HP and best MPG. + valid = {k: v for k, v in specs.items() if v} + most_hp = max(valid, key=lambda k: valid[k][0]) if valid else "Camry" + best_mpg = max(valid, key=lambda k: valid[k][1]) if valid else "Accord" + j.check("nav_compare", navigated_to(t, "/compare"), "expected the /compare page") + j.check("answer_names_all_three", all(x in fa for x in ["Accord", "Camry", "Altima"]), + f"final={fa!r}") + ok, ev = llm_text_match(fa, f"most horsepower: {most_hp} ({valid.get(most_hp,['?'])[0]} hp); " + f"best combined MPG: {best_mpg} ({valid.get(best_mpg,['?','?'])[1]} mpg)", + "Among 2022 Accord/Camry/Altima, which has the most HP and which the best combined MPG?") + j.check("answer_hp_mpg_correct", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_7.py b/sites/carmax/verify/verify_7.py new file mode 100644 index 00000000..a1911d82 --- /dev/null +++ b/sites/carmax/verify/verify_7.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--7: get an instant offer to sell a 2018 Toyota Camry LE, 78,500 mi, +good condition, ZIP 30303, no accidents, one owner; report the offer amount and expiry. + +Deterministic-first: nav to sell-my-car | DB after-state: an appraisal for that 2018 Camry +exists in the after DB and NOT in the initial seed (proves the agent created it) | +answer contains the offer amount + expiry that the env computed for it. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, navigated_re, final_answer, price_mentioned, + contains_any, resolve_db, db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--7', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + q = ("SELECT offer_amount, offer_valid_until FROM appraisals WHERE year=2018 AND make='Toyota' " + "AND model='Camry' AND mileage=78500") + aft = db_query(after, q) if after else [] + ini = db_query(init, q) if init else [] + j.check("nav_sell_my_car", navigated_to(t, "/sell-my-car"), "expected /sell-my-car flow") + j.check("db_appraisal_created", len(aft) > len(ini), + f"after={len(aft)} appraisal(s) for the 2018 Camry vs initial={len(ini)} (agent must create one)") + if aft: + offer, expires = aft[-1] + j.check("answer_offer_amount", price_mentioned(fa, int(offer)), f"expected offer ${int(offer):,}; final={fa!r}") + j.check("answer_expiry", contains_any(fa, [str(expires)]), f"expected expiry {expires}") + ok, ev = llm_text_match(fa, f"offer ${int(offer):,}, valid until {expires}", + "Report the instant-offer dollar amount and expiration date.") + j.check("answer_consistent", ok, ev, llm=True) + else: + j.check("answer_offer_amount", False, "no appraisal row to anchor the offer against") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_8.py b/sites/carmax/verify/verify_8.py new file mode 100644 index 00000000..10647838 --- /dev/null +++ b/sites/carmax/verify/verify_8.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--8: store locator; report (a) how many states have >=1 store and +(b) the street address of any CA store. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, price_mentioned, + resolve_db, db_query, llm_text_match, Judge, parse_args) + +def main(): + a = parse_args() + j = Judge('CarMax--8', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + init = resolve_db(a.initial_db, a.container, "instance_seed") + n_states = db_query(init, "SELECT COUNT(DISTINCT state) FROM stores")[0][0] if init else 12 + ca = db_query(init, "SELECT name, street, city FROM stores WHERE state='CA'") if init else [] + ca_streets = [r[1] for r in ca] or ["6101 Auto Center Dr"] + ca_cities = [r[2] for r in ca] or ["Buena Park"] + j.check("nav_stores", navigated_to(t, "/stores") or navigated_to(t, "/store"), + "expected the store locator") + j.check("answer_state_count", price_mentioned(fa, int(n_states)), f"expected {n_states} states; final={fa!r}") + j.check("answer_ca_address", contains_any(fa, ca_streets + ca_cities), + f"expected a CA store street/city from {ca_streets} / {ca_cities}") + ok, ev = llm_text_match(fa, f"{n_states} states with a store; a CA store street such as " + f"{ca_streets[0]} in {ca_cities[0]}", + "How many states have a CarMax store, and the street address of a CA store?") + j.check("answer_consistent", ok, ev, llm=True) + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_9.py b/sites/carmax/verify/verify_9.py new file mode 100644 index 00000000..0f099a54 --- /dev/null +++ b/sites/carmax/verify/verify_9.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verifier for CarMax--9: register a new account (Test Buyer / new.buyer.benchmark@test.com, +ZIP 30303), then get pre-qualified (80k income, full-time, $500 max monthly, $2,000 down, +72-month term, good credit); report the estimated APR. + +Deterministic-first: nav register + pre-qual | DB after-state: the new user exists (not in +initial seed) AND has a pre-qual APR set | answer contains that computed APR. +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from verify_lib import (load_run, navigated_to, final_answer, contains_any, resolve_db, + db_query, user_exists, user_prequal, llm_text_match, Judge, parse_args) + +EMAIL = "new.buyer.benchmark@test.com" + +def main(): + a = parse_args() + j = Judge('CarMax--9', a.no_llm) + t = load_run(a.run_dir); fa = final_answer(t) + after = resolve_db(a.after_db, a.container, "instance") + init = resolve_db(a.initial_db, a.container, "instance_seed") + j.check("nav_register", navigated_to(t, "/register"), "expected /register") + j.check("nav_prequal", navigated_to(t, "/pre-qual"), "expected /pre-qual flow") + j.check("db_user_created", user_exists(after, EMAIL) and not user_exists(init, EMAIL), + f"user {EMAIL} present after={user_exists(after, EMAIL)} initial={user_exists(init, EMAIL)}") + pq = user_prequal(after, EMAIL) + apr = pq[1] if pq else None + j.check("db_prequal_apr_set", bool(apr), f"pre_qual row/apr for {EMAIL}: {pq}") + if apr: + # answer must contain the APR the env computed (robust: read from DB, don't hardcode) + apr_str = f"{apr:.2f}".rstrip("0").rstrip(".") + j.check("answer_apr", contains_any(fa, [str(apr), apr_str, f"{apr:.2f}"]), + f"expected APR {apr}; final={fa!r}") + ok, ev = llm_text_match(fa, f"estimated APR {apr}%", + "Report the estimated APR shown on the pre-qualification result.") + j.check("answer_consistent", ok, ev, llm=True) + else: + j.check("answer_apr", False, "no pre-qual APR to anchor against") + j.emit() + +if __name__ == "__main__": + main() diff --git a/sites/carmax/verify/verify_lib.py b/sites/carmax/verify/verify_lib.py new file mode 100644 index 00000000..6fc043ff --- /dev/null +++ b/sites/carmax/verify/verify_lib.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""verify_lib.py — shared deterministic + LLM utilities for CarMax task verification. + +Philosophy: DETERMINISTIC FIRST (mirrors sites/merriam_webster/verify/verify_lib.py). + 1. Trajectory navigation check (anti knowledge-shortcut): the agent MUST have opened + the relevant on-site page; a correct answer with no matching navigation is a + memory-recall shortcut = FAIL. (CarMax inventory is fictional, so this is strong.) + 2. Answer check: exact / token-containment / numeric against frozen ground truth. + 3. DB after-state check (stateful tasks): query the SQLite instance DB directly — the + strongest signal (saved_vehicles / reservations / test_drives / orders / appraisals). + 4. LLM utilities (text match, screenshot-contains) ONLY where exact matching is brittle, + ALWAYS anchored on ground truth; the model verifies *presence*, never supplies knowledge. + +Input signature (per task): + --run_dir DIR agent trajectory dir: trajectory.json + screenshots/step_NNN.png + --initial_db PATH initial-state SQLite DB (default: fetched instance_seed from container) + --after_db PATH after-state SQLite DB (default: fetched live instance DB from container) + --container NAME docker container to fetch DBs from (default: $WH_CONTAINER or wh-review) + --no_llm skip LLM-based checks (deterministic-only) +Output: JSON {task_id, pass, reason, evidence[]} to stdout; exit 0 on PASS, 1 on FAIL. +""" +import base64, json, os, re, sqlite3, subprocess, sys, tempfile, urllib.request +from pathlib import Path +from dataclasses import dataclass +import simpleArgParser as sap + +SITE = "carmax" + +# ---------------------------------------------------------------- trajectory +def load_run(run_dir): + d = Path(run_dir) + traj = json.loads((d / "trajectory.json").read_text()) + traj["_run_dir"] = d + traj["_shots"] = {p.name: p for p in sorted((d / "screenshots").glob("step_*.png"))} + return traj + +def step_urls(traj): + return [s.get("url", "") for s in traj.get("steps", [])] + +def navigated_to(traj, substr, times=1): + return sum(1 for u in step_urls(traj) if substr in u) >= times + +def navigated_any(traj, substrs): + return any(navigated_to(traj, s) for s in substrs) + +def navigated_re(traj, pattern): + rx = re.compile(pattern) + return any(rx.search(u or "") for u in step_urls(traj)) + +def final_answer(traj): + return (traj.get("final_answer") or "").strip() + +def _shot(traj, name): + if not name: + return None + p = traj["_shots"].get(Path(name).name) + return p if (p and p.exists()) else None + +def shot_after_url(traj, substr): + for s in traj.get("steps", []): + if substr in s.get("url", ""): + p = _shot(traj, s.get("screenshot_after")) + if p: + return p + return None + +def last_shot(traj): + for s in reversed(traj.get("steps", [])): + p = _shot(traj, s.get("screenshot_after")) or _shot(traj, s.get("screenshot_before")) + if p: + return p + shots = sorted(traj["_shots"].values()) + return shots[-1] if shots else None + +# ---------------------------------------------------------------- deterministic answer match +def norm(s): + return re.sub(r"\s+", " ", (s or "").strip()).casefold() + +def answer_equals(final, expected): + return norm(final) == norm(expected) + +def contains_all(final, tokens): + f = norm(final) + return all(norm(t) in f for t in tokens) + +def contains_any(final, tokens): + f = norm(final) + return any(norm(t) in f for t in tokens) + +def extract_prices(text): + """Return list of dollar amounts as ints, e.g. '$10,500' -> 10500.""" + return [int(m.replace(",", "")) for m in re.findall(r"\$?\s?([\d]{1,3}(?:,\d{3})+|\d{4,6})", text or "")] + +def price_mentioned(final, amount, tol=0): + """True if `amount` (int dollars) appears in final answer (comma or plain).""" + f = norm(final) + cands = {str(amount), f"{amount:,}"} + if tol: + return any(abs(p - amount) <= tol for p in extract_prices(final)) + return any(c in f for c in cands) + +def extract_ints(text): + return [int(m.replace(",", "")) for m in re.findall(r"\b(\d{1,3}(?:,\d{3})+|\d+)\b", text or "")] + +# ---------------------------------------------------------------- DB state +def fetch_db(container, kind): + """kind: 'instance' (after) or 'instance_seed' (initial). docker cp -> temp file.""" + src = f"{container}:/opt/WebSyn/{SITE}/{kind}/{SITE}.db" + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + r = subprocess.run(["docker", "cp", src, path], capture_output=True, text=True) + if r.returncode != 0: + try: + os.unlink(path) + except OSError: + pass + raise RuntimeError(f"docker cp {src} failed: {r.stderr.strip()}") + return path + +def resolve_db(arg, container, kind): + if arg: + return arg + try: + return fetch_db(container, kind) + except Exception: + return None # caller treats None as "unavailable" and FAILs that check + +def db_query(db_path, sql, params=()): + con = sqlite3.connect(db_path) + try: + return con.execute(sql, params).fetchall() + finally: + con.close() + +# --- CarMax-specific state helpers (return None if db unavailable) ----------- +def saved_vehicles_for(db_path, email): + if not db_path: + return None + rows = db_query(db_path, + "SELECT v.year, v.make, v.model, v.trim, v.mileage FROM saved_vehicles s " + "JOIN users u ON u.id=s.user_id JOIN vehicles v ON v.id=s.vehicle_id " + "WHERE u.email=? ORDER BY v.mileage", (email,)) + return [tuple(r) for r in rows] + +def reservations_for(db_path, email, status=None): + if not db_path: + return None + sql = ("SELECT v.year, v.make, v.model, r.appointment_date, r.status FROM reservations r " + "JOIN users u ON u.id=r.user_id JOIN vehicles v ON v.id=r.vehicle_id WHERE u.email=?") + p = [email] + if status: + sql += " AND r.status=?"; p.append(status) + return [tuple(r) for r in db_query(db_path, sql, tuple(p))] + +def test_drives_for(db_path, email): + if not db_path: + return None + return [tuple(r) for r in db_query(db_path, + "SELECT v.year, v.make, v.model, t.location_type, t.scheduled_date, t.scheduled_time, " + "t.notes, t.status FROM test_drives t JOIN users u ON u.id=t.user_id " + "JOIN vehicles v ON v.id=t.vehicle_id WHERE u.email=?", (email,))] + +def orders_for(db_path, email): + if not db_path: + return None + return [dict(zip( + ["order_number", "total", "maxcare_plan", "pickup_or_delivery", "pickup_date", + "trade_in_value", "payment_apr", "payment_term_months", "down_payment", "status", + "year", "make", "model"], + r)) for r in db_query(db_path, + "SELECT o.order_number, o.total, o.maxcare_plan, o.pickup_or_delivery, o.pickup_date, " + "o.trade_in_value, o.payment_apr, o.payment_term_months, o.down_payment, o.status, " + "v.year, v.make, v.model FROM orders o JOIN users u ON u.id=o.user_id " + "JOIN vehicles v ON v.id=o.vehicle_id WHERE u.email=?", (email,))] + +def appraisals_for(db_path, email, status=None): + if not db_path: + return None + sql = ("SELECT year, make, model, offer_amount, status FROM appraisals a " + "JOIN users u ON u.id=a.user_id WHERE u.email=?") + p = [email] + if status: + sql += " AND a.status=?"; p.append(status) + return [tuple(r) for r in db_query(db_path, sql, tuple(p))] + +def user_exists(db_path, email): + if not db_path: + return None + return bool(db_query(db_path, "SELECT 1 FROM users WHERE email=?", (email,))) + +def user_prequal(db_path, email): + """Return (pre_qual_active, pre_qual_apr) for a user, or None.""" + if not db_path: + return None + r = db_query(db_path, "SELECT pre_qual_active, pre_qual_apr FROM users WHERE email=?", (email,)) + return tuple(r[0]) if r else None + +# ---------------------------------------------------------------- shared LLM utilities (anchored) +_NO_LLM = False + +def _llm_config(): + return (os.environ.get("OPENAI_API_KEY", ""), + os.environ.get("OPENAI_BASE_URL", ""), + os.environ.get("JUDGE_MODEL", "")) + +def _chat(messages, max_tokens=1024): + if _NO_LLM: + return None + key, base, model = _llm_config() + if not (key and base and model): + return None + payload = {"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": 1.0} + # Accept OPENAI_BASE_URL as either the base (".../api", like agent.py/eval_judge) or a + # full chat-completions URL (".../chat/completions", the merriam_webster convention). + b = base.rstrip("/") + endpoint = b if b.endswith("/chat/completions") else b + "/chat/completions" + req = urllib.request.Request(endpoint, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {key}"}) + try: + data = json.loads(urllib.request.urlopen(req, timeout=180).read()) + return data["choices"][0]["message"]["content"] + except Exception: + return None + +def _verdict(out): + if not out: + return False, "" + s = out.strip() + return s.upper().startswith("PASS"), s + +def llm_text_match(agent_answer, ground_truth, question): + if _NO_LLM: + return False, "[skipped: --no_llm]" + out = _chat([{"role": "user", "content": + f"You are a STRICT binary grader.\nQuestion: {question}\n" + f"Ground-truth answer (ANCHOR — judge against THIS, never use your own knowledge): {ground_truth}\n" + f"Agent's answer: {agent_answer}\n" + f"Decide PASS or FAIL ignoring case/punctuation/word order/surrounding prose. " + f"PASS only if the agent's answer is consistent with the ground truth AND actually answers the question. " + f"Line 1: PASS or FAIL. Line 2: one-sentence reason."}]) + return _verdict(out) + +def llm_screenshot_shows(shot_path, must_show, question=""): + if _NO_LLM: + return False, "[skipped: --no_llm]" + b64 = base64.b64encode(Path(shot_path).read_bytes()).decode() + out = _chat([{"role": "user", "content": [ + {"type": "text", "text": + f"You are a STRICT binary grader. Only what is VISIBLY rendered in this screenshot counts.\n" + f"Question the page should answer: {question}\n" + f"Expected content to verify PRESENCE of: {must_show}\n" + f"PASS only if the expected content (or a semantically equivalent on-screen value) is visibly shown. " + f"Do NOT use prior knowledge — judge only the rendered pixels.\n" + f"Line 1: PASS or FAIL. Line 2: quote the visible evidence."}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}]) + return _verdict(out) + +# ---------------------------------------------------------------- judge harness + CLI +class Judge: + def __init__(self, task_id, no_llm=False): + global _NO_LLM + _NO_LLM = bool(no_llm) + self.task_id = task_id + self.no_llm = no_llm + self.ok = True + self.reason = "" + self.evidence = [] + + def check(self, name, cond, evidence="", llm=False): + if llm and self.no_llm: + self.evidence.append(f"[SKIP] {name} (--no_llm)") + return True + if cond: + self.evidence.append(f"[PASS] {name}: {evidence}") + else: + self.ok = False + if not self.reason: + self.reason = name + self.evidence.append(f"[FAIL] {name}: {evidence}") + return bool(cond) + + def emit(self): + print(json.dumps({"task_id": self.task_id, "pass": self.ok, + "reason": self.reason, "evidence": self.evidence}, indent=2)) + sys.exit(0 if self.ok else 1) + +def parse_args(): + @dataclass + class VerifyArgs: + run_dir: str = "" + initial_db: str = "" + after_db: str = "" + container: str = os.environ.get("WH_CONTAINER", "wh-review") + no_llm: bool = False + + def post_process(self): + if not self.run_dir: + raise SystemExit("--run_dir is required") + return sap.parse_args(VerifyArgs)