From c6a70461bb9c987532002c317d524ae297cce113 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:48:07 +0800 Subject: [PATCH 1/2] fix(workspace): read original research against real benchmark fixtures Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- apps/presentation/dashboard/package.json | 1 + .../smoke/research-analysis-read-smoke.mjs | 156 ++ ...nch-grid-context-off-20261103-history.json | 459 ++++++ ...ench-grid-context-on-20261103-history.json | 1423 +++++++++++++++++ .../goal-research-analysis.tsx | 111 +- .../goal-research-history.tsx | 4 +- .../goal-research-timeline.tsx | 10 +- docs/reference/trajectory-assessment.md | 7 + examples/personal-workspace-browser-smoke.mjs | 2 + .../research-history-backend.py | 14 +- .../research-history.md | 56 +- .../research-history.mjs | 11 +- .../research-real-benchmark.mjs | 92 ++ tests/presentation/test_explore_history.py | 2 + 14 files changed, 2299 insertions(+), 49 deletions(-) create mode 100644 apps/presentation/dashboard/smoke/research-analysis-read-smoke.mjs create mode 100644 apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-off-20261103-history.json create mode 100644 apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-on-20261103-history.json create mode 100644 examples/personal-workspace-browser/research-real-benchmark.mjs diff --git a/apps/presentation/dashboard/package.json b/apps/presentation/dashboard/package.json index 285cc938a5..6c8f2b23ee 100644 --- a/apps/presentation/dashboard/package.json +++ b/apps/presentation/dashboard/package.json @@ -73,6 +73,7 @@ "smoke:chat-upgrade": "LOOPX_PLAYWRIGHT_PACKAGE=\"$PWD/node_modules/playwright\" node ../../../examples/chat-bundle-upgrade-browser-smoke.mjs", "test:conversation-returns": "node --experimental-strip-types src/data/conversation-returns.test.mjs", "smoke:conversation-history": "vite build --ssr smoke/conversation-history-smoke.ts --outDir node_modules/.cache/loopx-conversation-history --emptyOutDir && node node_modules/.cache/loopx-conversation-history/conversation-history-smoke.js", + "smoke:research-analysis-read": "vite build --ssr smoke/research-analysis-read-smoke.mjs --outDir node_modules/.cache/loopx-research-analysis-read --emptyOutDir && node node_modules/.cache/loopx-research-analysis-read/research-analysis-read-smoke.js", "smoke:task-session-observation": "vite build --ssr smoke/task-session-observation-smoke.ts --outDir node_modules/.cache/loopx-task-session-observation --emptyOutDir && node node_modules/.cache/loopx-task-session-observation/task-session-observation-smoke.js", "smoke:conversation-history-order": "LOOPX_PLAYWRIGHT_PACKAGE=\"$PWD/node_modules/playwright\" node smoke/conversation-history-order-browser-smoke.mjs", "smoke:conversation-returns-packaged": "LOOPX_PLAYWRIGHT_PACKAGE=\"$PWD/node_modules/playwright\" node smoke/conversation-return-browser-smoke.mjs", diff --git a/apps/presentation/dashboard/smoke/research-analysis-read-smoke.mjs b/apps/presentation/dashboard/smoke/research-analysis-read-smoke.mjs new file mode 100644 index 0000000000..66e9854990 --- /dev/null +++ b/apps/presentation/dashboard/smoke/research-analysis-read-smoke.mjs @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import {spawn} from "node:child_process"; +import {once} from "node:events"; +import {mkdtemp, mkdir, readFile, writeFile, rm} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {resolve} from "node:path"; +import {createInterface} from "node:readline"; +import {resolveTestPython} from "../../../../scripts/test-python.mjs"; +import {readAnalysisSnapshot} from "../src/features/personal-workspace/goal-research-analysis.tsx"; + +// A language-specific cache miss must not hide the same registered pack's +// older, unspecified-language result or turn a read into model execution. +const scope = {goal_id: "synthetic-research", case_id: "case", run_id: "run", revision: "fixed", window: "fixed"}; +const config = {model: "fixture", effort: "max", service_tier: "fast"}; +const pack = {pack_id: "a".repeat(64), analysis_key: "b".repeat(64), scope, config, + method_version: "research-episodes-v2", result: null, job: null}; +const original = {...pack, analysis_key: "c".repeat(64), + job: {status: "succeeded", usage: {input_tokens: 10, output_tokens: 3}}, + result: {analysis_key: "c".repeat(64), scope, method_version: pack.method_version, + metrics: [{metric: "benefit_test", yes: 0, no: 1, unknown: 2, eligible: 3}], + limitations: ["No causal effect has been established."]}}; +const response = p => ({ok: true, catalog: {packs: [p], default_pack_id: pack.pack_id}, pack: p}); +const savedFetch = globalThis.fetch, savedWindow = globalThis.window; +globalThis.window = {location: {href: "http://127.0.0.1:12345/chat/"}}; +let calls = [], localized, saved, failOriginal = false; +globalThis.fetch = async (input, init) => { + assert.ok(!init.method || init.method === "GET", "Saved analysis discovery must not dispatch any write"); + const url = new URL(input, window.location.href); + calls.push(url); + if (url.searchParams.has("output_language")) return Response.json(response(localized)); + assert.equal(url.searchParams.get("pack_id"), pack.pack_id, "A changed default cannot redirect the original read"); + return failOriginal ? Response.json({error: "Original source unavailable"}, {status: 503}) : Response.json(response(saved)); +}; +const read = (zh = true, source, signal, current) => readAnalysisSnapshot(scope.goal_id, null, zh, source, signal, current); +const reset = language => { + calls = []; failOriginal = false; + localized = {...structuredClone(pack), config: {...config, output_language: language}}; + saved = structuredClone(original); +}; +try { + for (const [zh, language] of [[true, "zh"], [false, "en"]]) { + reset(language); + const value = await read(zh); + assert.equal(calls.length, 2); + assert.equal(value.pack.config.output_language, language, "Execution retains the requested language and key"); + assert.equal(value.pack.result, null, "Original output does not populate the language-specific result"); + assert.deepEqual(value.originalPack, original, "Original judgments, negative/unknown counts, cost and identity remain unchanged"); + assert.equal(value.originalPack.result.output_language, undefined, "The reader never invents a translation"); + } + reset("zh"); + localized.result = {...original.result, analysis_key: localized.analysis_key, output_language: "zh"}; + let value = await read(); + assert.equal(calls.length, 1, "A current-language result needs no original lookup"); + assert.equal(value.originalPack, null); + + reset("zh"); + localized.result = {...original.result, output_language: "zh"}; + await assert.rejects(read(), /读回不一致/); + assert.equal(calls.length, 1, "A mismatched current result is rejected before any original lookup"); + + reset("en"); + saved.config.output_language = "zh"; + saved.result.output_language = "zh"; + value = await read(false); + assert.equal(value.originalPack, null, "An explicit other-language artifact is not an unspecified-language original"); + + for (const corrupt of [ + p => {p.pack_id = "d".repeat(64);}, + p => {p.scope.run_id = "other-run";}, + p => {p.method_version = "other-method"; p.result.method_version = "other-method";}, + p => {p.result.analysis_key = "d".repeat(64);}, + p => {p.result.method_version = "other-method";}, + p => {p.result.scope = {...scope, revision: "other-revision"};}, + p => {p.result.output_language = "en";}, + ]) { + reset("zh"); corrupt(saved); + value = await read(); + assert.equal(value.originalPack, null, "Mismatched saved evidence must never enter the reader"); + assert.ok(value.originalReadError, "A rejected original remains distinguishable from an absent result"); + assert.equal(value.pack.analysis_key, pack.analysis_key); + } + reset("zh"); failOriginal = true; + value = await read(); + assert.ok(value.originalReadError); + assert.equal(value.pack.result, null, "An original read failure does not erase the current-language state"); + failOriginal = false; + assert.deepEqual((await read()).originalPack, original, "The ordinary read retries recovery without execution"); + + for (const options of [{current: () => false}, {signal: AbortSignal.abort()}]) { + reset("zh"); + value = await read(true, undefined, options.signal, options.current); + assert.equal(calls.length, 1, "A stale or aborted read cannot start an original lookup"); + assert.equal(value.originalPack, null); + } + reset("zh"); + await read(true, `http://127.0.0.1:23456/status.json?source_namespace=${"e".repeat(64)}`); + assert.ok(calls.every(url => url.origin === "http://127.0.0.1:23456" && url.searchParams.get("source_namespace") === "e".repeat(64)), + "Both observations retain the selected physical source namespace"); + + // Qualify the production consumer against the actual pack/job/HTTP owner. + // The sole startup output is a synthetic process fixture, never a model. + const repoRoot = resolve(process.cwd(), "../../.."); + const directory = await mkdtemp(resolve(tmpdir(), "loopx-original-analysis-read-")); + let child, lines; + try { + const capture = JSON.parse(await readFile(resolve(repoRoot, "apps/presentation/dashboard/src/data/fixtures/research-native-readback.json"), "utf8")); + const analysis = capture.analyses[0]; + analysis.config = {model: "fixture", effort: "max", service_tier: "fast", research_synthesis: false}; + analysis.outputs = {original: {assessments: [{episode_id: analysis.input.episodes[0].id, metric: "benefit_test", + verdict: "no", reason: "The recorded observation does not establish task utility.", + citations: [{source_id: "facts", first_line: 1, last_line: 1}]}], limitations: ["Synthetic reader fixture only."]}}; + const path = resolve(directory, "capture.json"), temporary = resolve(directory, "tmp"); + await mkdir(temporary); await writeFile(path, JSON.stringify(capture)); + child = spawn(resolveTestPython({repoRoot}), ["-u", "examples/personal-workspace-browser/research-history-backend.py", "--fixture", path], + {cwd: repoRoot, env: {...process.env, PYTHONPATH: repoRoot, TMPDIR: temporary, TMP: temporary, TEMP: temporary}, stdio: ["ignore", "pipe", "pipe"]}); + let diagnostic = ""; + child.stderr.on("data", chunk => {diagnostic = (diagnostic + chunk).slice(-4000);}); + lines = createInterface({input: child.stdout}); + const ready = await new Promise((accept, reject) => { + const timer = setTimeout(() => reject(new Error(`Reader startup timed out: ${diagnostic}`)), 25000); + child.once("error", e => {clearTimeout(timer); reject(e);}); + child.once("exit", () => {clearTimeout(timer); reject(new Error(`Reader exited: ${diagnostic}`));}); + lines.once("line", line => {clearTimeout(timer); try {accept(JSON.parse(line));} catch (e) {reject(e);}}); + }); + const origin = `http://127.0.0.1:${ready.port}`; + const requests = []; + globalThis.fetch = (input, init) => { + assert.ok(!init?.method || init.method === "GET"); + requests.push(String(input)); + return savedFetch(new URL(input, origin), init); + }; + const actual = await readAnalysisSnapshot(capture.goal_id, null, true); + assert.equal(requests.length, 2); + assert.equal(actual.pack.result, null); + assert.equal(actual.originalPack.result.metrics.find(row => row.metric === "benefit_test").no, 1); + assert.equal(actual.originalPack.result.output_language, undefined); + const p = actual.originalPack; + const citation = await (await savedFetch(`${origin}/api/chat/research-analysis/citation?${new URLSearchParams({ + goal_id: capture.goal_id, pack_id: p.pack_id, analysis_key: p.analysis_key, + source_id: "facts", first_line: "1", last_line: "1"})}`)).json(); + assert.equal(citation.text, analysis.input.sources[0].text.split("\n")[0]); + const readback = await (await savedFetch(`${origin}/fixture/readback`)).json(); + assert.equal(readback.mutation_attempts, 0); + assert.equal(readback.source_unchanged, true); + } finally { + lines?.close(); + if (child && child.exitCode === null && child.signalCode === null) { + const exited = once(child, "exit"); child.kill("SIGTERM"); await exited; + } + await rm(directory, {recursive: true, force: true}); + } + console.log("research original analysis read: passed"); +} finally { + globalThis.fetch = savedFetch; + if (savedWindow === undefined) delete globalThis.window; else globalThis.window = savedWindow; +} diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-off-20261103-history.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-off-20261103-history.json new file mode 100644 index 0000000000..f75ad2a8f5 --- /dev/null +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-off-20261103-history.json @@ -0,0 +1,459 @@ +{ + "description": "Complete captured Explore display state from a recent real Grid Turing Robot run. Original wording, recording times, identities, measurements and references are unchanged.", + "source_sha256": "d91047d06aa7ef43249bc79e70d8ba53d37a88240c124dcd17a75034ea524375", + "source_bytes": 20575, + "goal_id": "benchmark-goal", + "display_name": "Grid · context off · 2026-11-03", + "provenance": { + "case_id": "grid_turing_robot", + "source_run_id": "grid-sol-xhigh-explore-stride-context-off-20261103-18h-r7", + "selection": "Both terminal R7 context arms, selected by recency and complete saved Explore log availability, including negative and unknown observations; no event or score-based sampling.", + "capture_kind": "terminal_collected_display_state", + "exported_at": "2026-11-03T17:24:14.634397+00:00", + "claim_scope": "diagnostic_only", + "score_countable": false, + "runner_status": "runner_invalid", + "limitations": [ + "The complete saved Explore log is not the complete solver trajectory or a coverage claim for every attempt.", + "Recorded confirmed labels are source-authored. Negative and unknown results remain in their original prose.", + "No explicit parent or relationship edges were recorded; shared wording or chronological adjacency does not establish causality.", + "The two arms have the same logical Goal ID in different runtimes. Load them as independent sources.", + "These runs include operational interventions and are not a qualified causal comparison. Local and native measurements retain separate scopes." + ], + "redactions": [], + "boundary": "Display-state events only. No execution/authority store, credentials, raw session transcript, hidden evaluator, workspace or model weights are included." + }, + "source_counts": { + "event_count": 11, + "event_kinds": { + "node": 2, + "finding": 9 + }, + "statuses": { + "resolved": 2, + "confirmed": 9 + }, + "first_recorded_at": "2026-11-03T06:04:18+00:00", + "last_recorded_at": "2026-11-03T14:55:52Z", + "edge_count": 0, + "parent_records": 0 + }, + "events": [ + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "2b7d84b345c01fbf", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T06:04:18+00:00", + "result_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "run_id": "explore-result:f541d3f6517716e37d8847940e8e246107f4de2eaf479f487c713665bd2b418c", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Grid Turing Robot local generated suite; no native generalization claim.", + "title": "Can backward color/state packing complete generated routes with substantially fewer resources than periodic packing?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "17d2453a9052a440", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v2-local40" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:3caa8b62e16999df468ac0caca68b792da1bcd87e03e63ca76454634815d70a3", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v2-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1938 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T06:04:18+00:00", + "result_id": "finding_f541d3f6517716e37d8847940e8e246107f4de2eaf479f487c713665bd2b418c", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592 Applicability: Grid Turing Robot local generated suite; no native generalization claim. Observation: All 40 local cases complete; total score 1938, maximum 6.874 s and 21000 KiB. Five-case score falls from 358 to 216. Interpretation: Lookahead that assigns both state and color is the implementation route. Shortest paths preserve T feasibility. Native fixed-case quality and useful further improvements remain to be evaluated in the existing optimization Todo.", + "tags": [ + "writeback-result" + ], + "title": "Can backward color/state packing complete generated routes with substantially fewer resources than periodic packing?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "13b9e044274a6087", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T06:22:37+00:00", + "result_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "run_id": "explore-result:7695b7f81a2352cf357c7d3c433797b10b282ac6ce81845f92e148e9cd9db8aa", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Grid Turing Robot local generated suite; no native generalization claim.", + "title": "Can backward color/state packing complete generated routes with substantially fewer resources than periodic packing?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "8679228da660d572", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v3-local40" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:391ca3ab068d2ff5559b81f3ff267a626ee1be77c4ba87dd8b2eff1bcb156f82", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v3-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1883 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T06:22:37+00:00", + "result_id": "finding_7695b7f81a2352cf357c7d3c433797b10b282ac6ce81845f92e148e9cd9db8aa", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592 Applicability: Grid Turing Robot local generated suite; no native generalization claim. Observation: Packed_v3 completes all 40 local cases with total 1883 versus 1938 for v2; 31 improve, none regress. Max 16.0328 s and 22356 KiB. Interpretation: Additional shortest-route and backward-placement alternatives improve the same local suite by 2.84% within the runtime limit. Retain v3 locally and the separately native-validated v2 recovery source. Native score 2224 belongs to v2; v3 native evaluation is pending cooldown.", + "tags": [ + "writeback-result" + ], + "title": "Can backward color/state packing complete generated routes with substantially fewer resources than periodic packing?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "29b57a2ff80b4a67", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v6-local40", + "research/ALLOCATION_REFINEMENT.md" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:5f17f0d630204a2f7ed51a2c84a8fb0a5eda922f7d6b7fc009afcf668951f554", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v6-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1856 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T08:24:32Z", + "result_id": "finding_4c5955f2c47f", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Same generated input bytes and official tester under the 30-second external limit: total 1856 versus 1866 for v5, nine improvements, zero regressions. The added color-popularity phase follows complete v5 and shares the 24-second internal deadline. Five existing holdout and three fresh large-board cases also pass; max across all 48 is 23.5196 seconds and 23520 KiB. This supports retaining v6 locally. Native v5 is 2139, but v6 has no native result; its next comparison is cooldown-limited. See research/ALLOCATION_REFINEMENT.md and research/results/packed_v6/report.json.", + "title": "V6 completes the original 40 local cases for 1856 with adaptive color and state allocation" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "8d75a6d45003bc32", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v7-local40", + "research/GROUPED_STATES.md" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:5d1ba686381bd6a4eb88ef695dab9b28faccf6772c849a40d44d58886ab36300", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v7-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1850 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T09:32:00Z", + "result_id": "finding_38bb42448035", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Same input bytes, official tester and 30-second external limit. V7 scores 1850 versus 1856 for v6 on the original 40 cases, six improvements and no regressions. Five existing holdout cases improve 100 to 98, and three stress cases pass unchanged. Maximum across all 48 is 26.1593 seconds and 22852 KiB. The full v6 search precedes paired-direction packing; its final phase has a 26-second total internal soft cutoff. Shared-pool and one-versus-three probes give no additional gains in their tested cohorts. Native v6 is 2131; v7 has no native result and awaits eligibility. See research/GROUPED_STATES.md.", + "title": "V7 completes the same 40 local cases for 1850 using paired-direction states" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "9d9348028e3d3850", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v8-local40", + "research/THREE_GROUPS.md" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:5b23e1cc59ee40a8bd20ac9ad51b187a0481350f14407b033055b05ae8f6ad61", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v8-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1844 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T10:35:44Z", + "result_id": "finding_59c93c429d7c", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Same input bytes, official tester and 30-second external protocol. V8 scores 1844 versus 1850 for v7, six improvements and no regressions on original40. Five reused short cases remain98 and three reused stress cases remain251. All48 pass; max26.1678 seconds and23008 KiB. The complete v7 search precedes six three-group state partitions, within the same26-second total internal soft cutoff. Joint color/state choice adds no gain in two nine-case probes. Direction-color groups match one already-achieved gain in the initial cohort and add none on six additional v8 cases, so remain unselected. Native v7 is separately source-matched at2124 with50/50 passes; v8 native score is unknown. See research/THREE_GROUPS.md.", + "title": "V8 completes the same 40 local cases for 1844 using three direction-state groups" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "e2c09dfc1b064778", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v9-local40", + "research/CONDITIONAL_COLORS.md" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:d9841a57d345daef50f9cb804b6f84ed73e8cf3215ee0dd781662854d9e4ef2d", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v9-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1840 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T11:40:51Z", + "result_id": "finding_50c700341a47", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Same input bytes, official tester and 30-second external protocol. V9 scores 1840 versus 1844 for v8, four improvements and no regressions on original40. Reused holdout5 stays98 and stress3 stays251. All48 pass; max26.1552 seconds and22856 KiB. Conditional color preference runs after the complete v8 portfolio within the same26-second total internal soft cutoff. Counts reflect existing output triples whose next state supports the next movement after the preceding cell visit. Allocation-transfer and static permitted-direction probes add no gain on their nine-case cohorts and remain unselected. Native v8 is separately source-matched at2115 with50/50 passes; v9 native feedback is unknown. See research/CONDITIONAL_COLORS.md.", + "title": "V9 completes the same 40 local cases for 1840 with next-direction-conditioned repaint colors" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "d934478a9e2c7a12", + "event_kind": "finding", + "evidence_refs": [ + "evidence:packed-v10-local40", + "research/MERGE_REPAIR.md" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:5a47930a4263daa10a1c1f0a2873ee59f5a94cd9cfcf0f88f9ba87f115ac1178", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "evidence:packed-v10-local40", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1834 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T12:52:45Z", + "result_id": "finding_747a75eff5fa", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Same input bytes, official tester and 30-second external protocol. V10 scores 1834 versus 1840 for v9, six improvements and no regressions on original40. Reused holdout5 stays98 and stress3 improves251 to250. All48 pass; max26.1982 seconds and23560 KiB. A bounded merging repair phase follows the complete v9 portfolio within the same26-second total soft cutoff. Surviving mapped rules are unreserved hints; each actual input pair is committed once. Straight deletion adds no gain on nine cases. Preferring incumbent input hints adds no resource gain beyond the known base repair in its nine-case follow-up; its one gain matches the earlier standalone result on an identical input program. Native v9 is separately source-matched at2113 with50/50 passes; v10 native feedback is unknown. See research/MERGE_REPAIR.md.", + "title": "V10 completes the same 40 local cases for 1834 using symbol merging with partial-table repair" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "351bc93b9a55d336", + "event_kind": "finding", + "evidence_refs": [ + "research/PHASE_BUDGET.md", + "research/results/packed_v10/report_all_cases.json", + "research/results/packed_v11/report_all_cases.json", + "research/results/packed_v11/report.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:b338497fa9523c1003508995ad0f200d300469fecbf5825a89874a64e3815402", + "comparison_scope": "grid-turing:local48:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/results/packed_v11/report_all_cases.json", + "higher_is_better": false, + "input_revision": "sha256:f6496759b49b5987dbc22a1cd7a485d14c279afae60d9940073ab181e1d6d399", + "metric": "total_score", + "unit": "score", + "value": 2181 + }, + { + "candidate_ref": "sha256:b338497fa9523c1003508995ad0f200d300469fecbf5825a89874a64e3815402", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/results/packed_v11/report.json", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1834 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T13:36:45Z", + "result_id": "finding_fba351edeccc", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Two fixed-budget alternatives pass pilot12: reserve4 scores618 and reserve8 scores617 versus619 for v10. Select reserve8 under the predeclared maximum72 solver runs. Its production source matches the pilot AST after removing tracing. All48 production cases pass for2181 versus2182. Core40 ties1834 with improvements on10 and24 and regressions on17 and27; short5 ties98; stress3 improves250 to249 on1021. Maximum26.1603 seconds and25016KiB under the same26-second soft cutoff and30-second process limit. Adopt v11 by the declared lower-aggregate criterion and retain all adverse cases. Four-second allocation has no full48 result. Cohorts are reused, not fresh. Native-supported v9 remains2113 with50/50 passes; v10 and v11 have no native result. See research/PHASE_BUDGET.md.", + "title": "V11 passes48 local cases for2181 by reserving eight seconds for merge repair" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "2e0f169a424b2988", + "event_kind": "finding", + "evidence_refs": [ + "research/CONDITIONAL_TAIL.md", + "research/results/conditional_tail_pilot/full_validation.json", + "research/results/conditional_tail_2_full/report_all_cases.json", + "research/results/conditional_tail_2_full/report.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:d235a01b40581d68cba564206a83a4c2bd1895fd5c50c199c867643b992f93d8", + "comparison_scope": "grid-turing:local48:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/results/conditional_tail_2_full/report_all_cases.json", + "higher_is_better": false, + "input_revision": "sha256:f6496759b49b5987dbc22a1cd7a485d14c279afae60d9940073ab181e1d6d399", + "metric": "total_score", + "unit": "score", + "value": 2176 + }, + { + "candidate_ref": "sha256:d235a01b40581d68cba564206a83a4c2bd1895fd5c50c199c867643b992f93d8", + "comparison_scope": "grid-turing:local40:30s-v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/results/conditional_tail_2_full/report.json", + "higher_is_better": false, + "input_revision": "sha256:4a379082f19059e6e8aeda9f7a3ee70c7f356dc2dd5353ac4977df9f9a76a592", + "metric": "total_score", + "unit": "score", + "value": 1830 + } + ], + "node_id": "question_50bda2a8b490f6141421dd06905838ab3a6d497d1bfb8d8df655df376a7e9cb5", + "recorded_at": "2026-11-03T14:55:52Z", + "result_id": "finding_b36af3cc3c07", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Pilot26/27/28 passes12 runs with totals316/315/314. Select two extra seconds after complete v10 prefix. Production d235a01b40581d68cba564206a83a4c2bd1895fd5c50c199c867643b992f93d8 passes48 for2176 versus historical v10 2182: core1830 versus1834, short98, stress248 versus250; six improvements and no regressions. Same input bytes, official tester and30-second external protocol; cohorts reused. Full max28.1595s,23344KiB; separate pilot max29.5348s preserves narrow-margin evidence. All18 helpers and35 original solve-prefix statements match v10 by AST; timed outcomes need not match. Actual root seed1021 passes87 in28.1411s. V9 native best remains2113; v12 submission e4fcfc225623 is pending as of the latest query. No native-quality or completion claim.", + "title": "V12 completes48 local cases for2176 with appended conditional refinement" + } + ] +} diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-on-20261103-history.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-on-20261103-history.json new file mode 100644 index 0000000000..dc64720394 --- /dev/null +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-on-20261103-history.json @@ -0,0 +1,1423 @@ +{ + "description": "Complete captured Explore display state from a recent real Grid Turing Robot run. Original wording, recording times, identities, measurements and references are unchanged.", + "source_sha256": "292f493ee5d88e0f99546776ee2a46958e19517ceae192c10288276f270f404d", + "source_bytes": 80992, + "goal_id": "benchmark-goal", + "display_name": "Grid · context on · 2026-11-03", + "provenance": { + "case_id": "grid_turing_robot", + "source_run_id": "grid-sol-xhigh-explore-stride-context-on-20261103-18h-r7", + "selection": "Both terminal R7 context arms, selected by recency and complete saved Explore log availability, including negative and unknown observations; no event or score-based sampling.", + "capture_kind": "terminal_collected_display_state", + "exported_at": "2026-11-03T17:24:14.633032+00:00", + "claim_scope": "diagnostic_only", + "score_countable": false, + "runner_status": "runner_invalid", + "limitations": [ + "The complete saved Explore log is not the complete solver trajectory or a coverage claim for every attempt.", + "Recorded confirmed labels are source-authored. Negative and unknown results remain in their original prose.", + "No explicit parent or relationship edges were recorded; shared wording or chronological adjacency does not establish causality.", + "The two arms have the same logical Goal ID in different runtimes. Load them as independent sources.", + "These runs include operational interventions and are not a qualified causal comparison. Local and native measurements retain separate scopes." + ], + "redactions": [], + "boundary": "Display-state events only. No execution/authority store, credentials, raw session transcript, hidden evaluator, workspace or model weights are included." + }, + "source_counts": { + "event_count": 33, + "event_kinds": { + "node": 8, + "finding": 25 + }, + "statuses": { + "resolved": 8, + "confirmed": 25 + }, + "first_recorded_at": "2026-11-03T06:02:35+00:00", + "last_recorded_at": "2026-11-03T15:06:44+00:00", + "edge_count": 0, + "parent_records": 0 + }, + "events": [ + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "981043665539ac69", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T06:02:35+00:00", + "result_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "run_id": "explore-result:24b520c4ca70ee762188e278db7a56e31218c44587d7fa95f859a275c372634a", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only.", + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "20abfa64b53d4fc9", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/validated_candidate.json", + "research/reports/summary.json", + "research/reports/resource_check_9.json", + "research/candidates/validated_candidate.py", + "research/reports/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:88a44107699d8d74b0ccc0b65a6a8231cb04f85776139c5583fef125c30f1630", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/validated_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1445 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T06:02:35+00:00", + "result_id": "finding_24b520c4ca70ee762188e278db7a56e31218c44587d7fa95f859a275c372634a", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: 27/27 complete; total score 1445; maximum measured case runtime 8.42 seconds. On matched seeds 0..9, total score falls from baseline 1437089 to 487. Interpretation: The tester confirms complete ordered visitation for this corpus. Backward construction fixes each repaint to the next visit color, and exact rule reuse preserves the trajectory. Self-paint allocation improves the earlier compiler total from 663 to 487 on seeds 0..9. This supports retaining the approach locally; native generalization is unknown. Next: use the existing P1 for native feedback and compare automaton merging or use of spare steps.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "1b89310dacd2c25c", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/palette_candidate.json", + "research/reports/palette_summary.json", + "research/candidates/palette_candidate.py" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:73df12d647255ab3ca4ae30e51e15ba85f26ac3798254e0b0bdacf2018ffdb1c", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/palette_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1441 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T06:23:41Z", + "result_id": "finding_da1fc9e51231", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Exact palette source 73df12d64725 completes all 27 local cases; seeds 8, 42 and 99 improve, with no per-case regression. Input and evaluator hashes match the prior local comparison. Native performance of this revision remains unmeasured.", + "title": "Palette allocation lowers the matched 27-case total from 1445 to 1441 with complete coverage" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "a3877a8d94156119", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T06:25:08+00:00", + "result_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "run_id": "explore-result:88a7052ee44c013724373243f690729946d4eae513a94cec0b4d891f66000e5f", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved.", + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "2f1eb278977cc88f", + "event_kind": "finding", + "evidence_refs": [ + "research/native/e5f28fa80668/result.json", + "research/native/e5f28fa80668/manifest.json", + "research/native/e5f28fa80668/solution.py", + "research/selection.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:88a44107699d8d74b0ccc0b65a6a8231cb04f85776139c5583fef125c30f1630", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/e5f28fa80668/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2447 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T06:25:08+00:00", + "result_id": "finding_88a7052ee44c013724373243f690729946d4eae513a94cec0b4d891f66000e5f", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Submission e5f28fa80668 (agent-1), source 88a44107699d, passed 50/50 native cases with total score 2447; no reported errors or timeouts. Interpretation: This confirms the first complete native baseline for the preserved source. Public per-case scores are unavailable. Later palette allocation changes are a separate local candidate and are not certified by this result. Keep the native-supported snapshot and evaluate a stronger candidate only after accepted-submission cooldown; continue useful independent optimization meanwhile.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "8a9d756549ebb49a", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/pool_candidate.json", + "research/reports/pool_summary.json", + "research/candidates/pool_candidate.py", + "research/reports/pool_resource_9.json", + "research/turns/wake-1793687227579770176-171fa2199391/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:6f7193a4ccf48d1c229e8c409d3749040029789d982af5582373eebbf3d98151", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/pool_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1284 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T06:57:56+00:00", + "result_id": "finding_1da0fa50555fcfa9592e43ca1491d002f01d0fab3f5c5714edeb7382394ecdaf", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: All27 official local checks complete every destination; total score1284 versus1441, no per-case regressions. Max runtime20.09s; seed9 uses20584KiB RSS. Interpretation: The pool compiler preserves the complete shortest-path trajectory while choosing a free state row using the preceding step as lookahead. Fresh input pairs never overwrite rules; exact reuse preserves outputs. Unused terminal states cannot affect steps before the last target. This supports adopting the stronger local source. Joint congruence and mixed-direction synthesis showed no gain in their tested scopes. Native performance of this source is unmeasured; keep the prior native-supported snapshot and test only after cooldown.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "b65a827c8f871ffe", + "event_kind": "finding", + "evidence_refs": [ + "research/native/4b07ae24c8b6/result.json", + "research/native/4b07ae24c8b6/manifest.json", + "research/native/4b07ae24c8b6/solution.py" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:09bafdbd4eef456552ea295ada05dd3314b1f9cbd139258fef7c5cd110c6e29f", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/4b07ae24c8b6/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2196 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T07:11:17Z", + "result_id": "finding_c54dd0af7fb0", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Accepted submission4b07ae24c8b6, roundagent-2, evaluates exact source09bafdbd4eef. The same fixed50 native protocol reports2196 with50passed, no errors or timeouts; improvement251 over source88a441. Public per-case numeric scores remain unavailable. Further flexible-pool work is independent local evidence and is not certified by this result. Preserve both immutable snapshots and continue independent optimization; next native admission cannot precede2026-11-03T08:09:19.349185Z.", + "title": "State-pool candidate improves native score from2447 to2196, with50/50 complete" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "8156975a33adfa3e", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/spill_candidate.json", + "research/reports/spill_summary.json", + "research/candidates/spill_candidate.py", + "research/reports/pool_order_comparison.json", + "research/turns/wake-1793689222063002377-d8e40ca978e6/execution_observation.json", + "research/reports/extended_pool_candidate.json", + "research/reports/spill_resource_9.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:c09268de7d9064cb7fb452d1fc14390a684387a12afb39b0f536d16ab92ab994", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/spill_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1247 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T07:24:50+00:00", + "result_id": "finding_0c6a5d49f7f51e9be173a430f15eda815dc54f1a27b7ca4c4a88e84ded60fa1f", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: All27 official local checks finish every target; matched total1272 improves to1247 without case regressions, max25.13s. Root seed9 repeats93 versus91, all targets complete, sampled26824KiB RSS. The separate native2196 result belongs to the prior source. Interpretation: Spilling permits mixed direction state rows while retaining unique input pairs and the backward repaint invariant. All original native-supported functions remain unchanged; the extra pass uses remaining time. An earlier shared-dispatch variant scored1249 but regressed seed9, so the isolated version was selected after full matched validation. Early-only pool seeding also regressed a case. Preserve the native2196 snapshot and test two-step predecessor reuse next; current local gains do not certify native performance. Deadline timing can affect the retained objective; the resource repeat demonstrates this limitation. The matched aggregate is evidence for this run, not a guarantee of identical later scores.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "62da3163a4538b24", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/twostep_candidate.json", + "research/reports/twostep_summary.json", + "research/candidates/twostep_candidate.py", + "research/reports/twostep_probe_summary.json", + "research/reports/twostep_structure.json", + "research/turns/wake-1793690784526726205-f5e20fe281ee/execution_observation.json", + "research/reports/twostep_resource_9.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:636ce7010db85e3b3cd0ff901d5d47b826992af894b71c523a984dd0e3e02fc9", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/twostep_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1209 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T07:42:18+00:00", + "result_id": "finding_ee0ec8532edee9f01851fc1b7298c20c465c250bfde7dfcc888bbb8d22f83e34", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: All27 official checks complete every destination; matched total1247 improves to1209 with no case regressions, max25.16s. Root seed9 repeats87 and uses26872KiB RSS. New source636ce7 is unmeasured natively; prior native2196 belongs to source09baf. Interpretation: Two-step lookahead prefers rows that can reuse both preceding transitions, accounting for a preceding return to the current cell. Preferred-group pruning omits only rows dominated by the first rank component. Unique input pairs and reverse repaint semantics preserve every planned move. Search ordering also changed, so the38-point gain is for the full portfolio rather than an isolated causal effect. Q1..3 spilling gave no reduction in10 saved programs and was not integrated. Preserve native2196 and local snapshots, then test depth3 reuse chains against their lookup cost within the same deadline.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "7b5aab3bafe83f8c", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/depth3_tail_candidate.json", + "research/reports/depth3_tail_summary.json", + "research/candidates/depth3_tail_candidate.py", + "research/reports/depth3_probe_summary.json", + "research/reports/depth3_cost.json", + "research/turns/wake-1793691842472373293-c1cc1314e755/execution_observation.json", + "research/reports/depth3_resource_12.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0da994b023624712758b31ac5a60c76ba3c80a75067bb48335214117eed8df45", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/depth3_tail_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1207 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T08:00:01+00:00", + "result_id": "finding_7d23d0970a40e4cc168e43b24f477654289c51213159522c664a7b26f6588429", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: All27 official checks complete every target; matched total1209 improves to1207 without case regression, max25.11s. Actual root seed12 repeats44,26900KiB RSS. Source0da994 is unmeasured natively; native2196 certifies source09baf. Interpretation: Depth3 reuse tracks temporary repaint colors for repeated positions and includes the proposed rule, with at most8 loose alternatives per step. A same-baseline saved10 comparison improves413 to411 while the two-step control stays413. Fixed15-attempt batches on3 seeds show17..27percent extra compiler time, so only a remaining-time final pass was added. The complete27 check confirms the modest integrated gain. Preserve earlier snapshots; alternate shortest-route weights can now test changed compiler conditions, while native evaluation still requires cooldown eligibility.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "f53a948a9de674bc", + "event_kind": "finding", + "evidence_refs": [ + "research/native/d66799bdb01a/result.json", + "research/native/d66799bdb01a/manifest.json", + "research/native/d66799bdb01a/solution.py", + "research/reports/route_load_summary.json", + "research/reports/detour_reuse_probe.json", + "research/reports/pre_native_stress_116.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0da994b023624712758b31ac5a60c76ba3c80a75067bb48335214117eed8df45", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/d66799bdb01a/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2074 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T08:19:13+00:00", + "result_id": "finding_c0e8f7217aaf697b41728e4349d0dfe9ac6ed0a0d5c76e34f8a0da1ff9f3b52e", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Submissiond66799bdb01a, source0da994, passes50/50 native tests with total2074, improving2196 by122; no reported errors or timeouts. Interpretation: The native result confirms improvement for the exact retained standalone source, which also completes the local27 corpus and a fresh large stress case. Alternate shortest-route weights and bounded fewer-turn detours gave no gain on10 saved cases each, so the solver stayed unchanged. Preserve prior snapshots. Further work should test bounded direction-pool allocation changes with measured cost and complete coverage; the native cooldown is now based on this third accepted submission.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "9beca95bc753546a", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/pool_transfer_candidate.json", + "research/reports/pool_transfer_summary.json", + "research/candidates/pool_transfer_candidate.py", + "research/reports/pool_transfer_probe.json", + "research/reports/pool_transfer_resource_6.json", + "research/turns/wake-1793694229908192807-37e9250efd43/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:26bdcb1efefb9f02e67fc643f3b6f42d73b574360b9c6b14675cca95eada0c40", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/pool_transfer_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1203 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T08:40:09+00:00", + "result_id": "finding_ab10912e0fb5251415db28f4843e5ef8adcf7f3565ad8c3bff401b91893151c8", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: Source26bdcb completes all27 local cases, total1203 versus1207, no case regressions, max25.15s. Actual root seed6 repeats32 in10.96s with21716KiB RSS. Native2074 still belongs to source0da994. Interpretation: A fixed-Q transfer moves one preferred state between directions while keeping at least one per used direction. On10 saved0da994 programs,5 extra seconds including route construction improves412 to410. The standalone integration preserves all17 baseline functions and search order, adding at most5 seconds only inside the original25-second deadline. Four local cases improve; both prior tested sources and the native2074 snapshot remain recoverable. This confirms a modest local gain, not native improvement or optimality. Static outputs contain no duplicate exact output triples, but loose successor/direction groups have choices; a future bounded probe can rank final-visit reuse alternatives using predecessor lookahead.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "5cb94f56462e21c2", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/loose_match_candidate.json", + "research/reports/loose_match_summary.json", + "research/candidates/loose_match_candidate.py", + "research/reports/loose_match_probe_summary.json", + "research/reports/loose_match_cost.json", + "research/reports/loose_match_resource_0.json", + "research/turns/wake-1793695347337797714-5a4be89c1edb/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:8ef535a802d792549eb3c86b61d597b7d386dd20ff46c79b1a41bc479dcd78d5", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/loose_match_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1201 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T09:02:33+00:00", + "result_id": "finding_87ab8f10fec4d8ba0baafc0671324f6d478ce7300c6044e1d14bd103fd87ce61", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: Source8ef535 completes all27 local cases, total1201 versus1203, no case regressions, max25.12s. Actual root seed0 repeats44 in22.03s,23276KiB RSS. Native2074 remains bound to0da994. Interpretation: Existing loose final-visit matches are ranked by predecessor reuse depth with temporary color overrides, then original state/color ordering. Known-repaint matching and free-pair ranking are unchanged. Paired saved10 tests give new408/control409 from410; only seed0 is a new-policy-specific gain, while seed9 improves in both. Fixed15-attempt batches on3 seeds with counters disabled cost19..83percent more, so the integration retains all19 baseline functions/main order and adds only an at-most5s tail inside25s. The full27 portfolio improves two cases; deadline effects limit causal attribution. Preserve both local and native snapshots. Static planned routes contain two-step returns; a future paired probe can test alternative free input colors instead of always preferring repaint color.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "d949a16e9736f101", + "event_kind": "finding", + "evidence_refs": [ + "research/native/9d67d71d7de1/result.json", + "research/native/9d67d71d7de1/manifest.json", + "research/native/9d67d71d7de1/solution.py", + "research/reports/return_color_probe_summary.json", + "research/reports/color_choice_probe_summary.json", + "research/reports/return_color_retained_stress_116.json", + "research/turns/wake-1793696680491733282-9c57e348fd63/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:8ef535a802d792549eb3c86b61d597b7d386dd20ff46c79b1a41bc479dcd78d5", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/9d67d71d7de1/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2066 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T09:27:04+00:00", + "result_id": "finding_3280c1873331ea7121d546ba7223fa52acfd2092be1bbcdf8d570c5ba6cbc98b", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Submission9d67d71d7de1, exact source8ef535, passes50/50 native cases at2066, improving2074 by8; no errors/timeouts. Retained root also completes fresh stress116 at88 in25.11s,23652KiB RSS. Interpretation: The native improvement validates the combined preferred-pool transfer and loose-match tails for the exact uploaded solver; it does not isolate each method. Local27 stays1201. Two changed color-policy questions were tested with paired unchanged controls and5extra seconds/method on10 saved programs each. Forced alternate color gives409/control408; considering both colors gives408/control408. All40 checks complete, but neither policy adds value over its control, so root stays8ef535. Preserve scoped negatives and immutable earlier sources. The next discriminating allocation probe compares radius2 integer state-pool transfers with a matched one-transfer control. Another native submission requires server admission after10:13:17.816523Z.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "a213ba69a2ac7402", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/pool_radius_candidate.json", + "research/reports/pool_radius_candidate_summary.json", + "research/reports/pool_radius_probe_summary.json", + "research/reports/pool_radius_baseline_repeat_16.json", + "research/reports/pool_radius_resource_1.json", + "research/turns/wake-1793698188529610126-fea53d9d0844/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:e541114d20fb0fb5be0240bb7162fe4c8a673f68b964c74ff57428ff02c60f2f", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/pool_radius_candidate.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1200 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T09:54:02+00:00", + "result_id": "finding_95226bd05fe4fa4be15c0c559cf15074ae0d04c5c1d72ce99779cbb59f8c71c1", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: Exact sourcee54111 completes27 cases at1200 versus1201, max25.13s. Gains1,9999; seed16 regresses56 to57. Unchanged baseline16 repeat also57. Actual root1 repeats32 in17.22s,22684KiB RSS. Interpretation: The established observation is scoped complete coverage and lower measured aggregate, not distributional dominance or native improvement. Radius2 beats matched radius1 control408 to409 on10 saved programs with5extra seconds each; only2,3 exhaust radius2 coverage. The standalone solver preserves all21 earlier functions/main prefix and adds the tested helper only within remaining25s. One per-case regression remains in the original full27 record. A targeted unchanged baseline repeat reproduces57 on16, demonstrating timing sensitivity without proving its cause or erasing the regression. Adopt the lower aggregate source and preserve native2066/source8ef535 separately. Static mixed-state rows motivate the next bounded preferred-group ordering comparison; benefit is unmeasured. No new native submission before10:13:17.816523Z and server admission.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "5f7fca03c32f45d7", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T10:10:29+00:00", + "result_id": "question_e1ada92c1a146bb505bc80cd1cfde07839a4ffd433f1024b3bb540c87b07bd87", + "run_id": "explore-result:34a69c579a1d09e10848ce567cf3d35898ef07ec9b7c3ee6fa5e567761d4ba62", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Generated seeds0..9; exact e54111 saved incumbents; radius2 count search, policy4, allocations0/1/2;5extra seconds/method including routes, rotating order; tools/bin/tester.", + "title": "Can preferred-state group ordering or spill-pressure ranking improve saved source e54111 programs within five extra seconds?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "1e803159a0f7ee0c", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/pool_order_probe.json", + "research/reports/pool_order_diagnostic.json", + "research/reports/pool_pressure_probe.json", + "research/reports/pool_pressure_probe_summary.json", + "research/reports/pool_order_retained_stress_116.json", + "research/turns/wake-1793699721491935149-bcea70a06800/execution_observation.json", + "research/turns/wake-1793699721491935149-bcea70a06800/saved_program_protocol.json", + "research/candidates/pool_pressure_pressure_early.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:9f03869bafb8c6b375a4212a74adb0e8e0d93944bd5bfe033692a95b089726bc", + "comparison_scope": "grid-turing-robot:saved-e54111-10:extra-5s:radius2:v1", + "evaluation_ref": "sha256:d8fa3467d4c3378742208a74b690a771e14fb75dcd8365220d876a173759122e", + "evidence_ref": "research/reports/pool_pressure_probe.json", + "higher_is_better": false, + "input_revision": "sha256:adf9622dcf32b0c3b1e5b59380ef9251135534698b0984b8d05ab3f2e29c0136", + "metric": "total_case_score", + "unit": "score", + "value": 408 + } + ], + "node_id": "question_e1ada92c1a146bb505bc80cd1cfde07839a4ffd433f1024b3bb540c87b07bd87", + "recorded_at": "2026-11-03T10:10:29+00:00", + "result_id": "finding_34a69c579a1d09e10848ce567cf3d35898ef07ec9b7c3ee6fa5e567761d4ba62", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:adf9622dcf32b0c3b1e5b59380ef9251135534698b0984b8d05ab3f2e29c0136 Applicability: Generated seeds0..9; exact e54111 saved incumbents; radius2 count search, policy4, allocations0/1/2;5extra seconds/method including routes, rotating order; tools/bin/tester. Observation: Canonical/reverse/axis-swap each408; unchanged/late/early spill pressure each408,60 retained outputs complete and no strict fits.40 fixed canonical clone calls agree exactly. Diagnostic q ties are rare on3 sampled cases;12 fitted outputs pass. Interpretation: The bounded answer is no measured gain from either tested family, so no variant is integrated. Order attempts20741/20635/20983; pressure control/late/early18697/18204/18201. Only seeds2,3 exhaust the tested radius2 search. Diagnostic18 fixed calls use BFS only atQfixed and capacityC or2C: atC, cross-group q ties occur2or3 times on seed0 failed prefixes and0 on5,9 despite spill steps. This motivates pressure ahead of row occupancy or reuse, but both placements also tie control. These results do not rule out all orderings, counts, demand models or larger budgets. The measurement is early-pressure helper plus exact parameters on saved incumbents with5extra seconds, not a standalone/native score. Retain local1200/sourcee54111 and native2066/source8ef535 separately. Fresh actual root116 completes88 in25.10s,23592KiB RSS. Prior depth3 gains motivate next depth4-versus-depth3 comparison; fixed-attempt cost and full27 are required before integration. No new native before10:13:17.816523Z plus server admission.", + "tags": [ + "writeback-result" + ], + "title": "Can preferred-state group ordering or spill-pressure ranking improve saved source e54111 programs within five extra seconds?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "75570cb2a4f8949a", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/horizon_portfolio_12.json", + "research/reports/horizon_portfolio_12_summary.json", + "research/reports/horizon_probe_summary.json", + "research/reports/horizon_cost.json", + "research/reports/horizon_portfolio_resource_116.json", + "research/turns/wake-1793700738530828175-cf0b3270ded2/integration_review.json", + "research/turns/wake-1793700738530828175-cf0b3270ded2/local_execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:8dc4cd836e68e7aec432bd3644dd2ca9a2961f41a637e369102a16337b095a09", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/horizon_portfolio_12.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1196 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T10:43:58Z", + "result_id": "finding_77382bf34b6c", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Exact source8dc4cd completes all27 local cases at1196 versus1200, no case regressions in this matched run, max25.158s. Gains5,9,11,999. Saved10 depth4406 versus depth3control407 from408; seed5 is depth4-specific,9shared. Fixed15-attempt batches on0,5,9 cost4.6..14.2percent more fordepth4, a small sample. Both demonstrated gains already consume25s in prior source, so reserve12s primary, twostep until17, depth4 up to5s/until22, pool23, spill24, other passes25. The8s primary sample ties12s at248 on4cases, so retain12s. All22 other baseline functions are AST-identical; solve gains only optional budget argument, copied helper preserves overrides/proposed rule/branchcap8 plus recursion timeout check every256calls. Actual root116 completes88 in25.107s,28124KiB RSS. This is the combined runtime/depth4 portfolio, not isolated horizon attribution or native improvement. Native5accepted3d18078fab52 remains running; native2066 still belongs to8ef535.", + "title": "Reserved-time depth4 portfolio completes27 local cases at1196, improving1200." + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "30b5e3b778b8c98a", + "event_kind": "finding", + "evidence_refs": [ + "research/native/3d18078fab52/result.json", + "research/native/3d18078fab52/manifest.json", + "research/native/3d18078fab52/solution.py", + "research/reports/horizon_portfolio_12_summary.json", + "research/reports/horizon_probe_summary.json", + "research/reports/horizon_cost.json", + "research/reports/horizon_stage_summary.json", + "research/turns/wake-1793700738530828175-cf0b3270ded2/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:8dc4cd836e68e7aec432bd3644dd2ca9a2961f41a637e369102a16337b095a09", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/3d18078fab52/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2055 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T10:52:22+00:00", + "result_id": "finding_56116e83eb0e339563f323e4ce03079c59f4544b7a32efc388015ab39897a9f3", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Submission3d18078fab52/agent5, exact8dc4cd, passes50/50 at2055 versus2066,0errors/timeouts. Full27=1196 vs1200, max25.158s, no case regressions; actual root116 completes88,25.11s,28124KiB RSS. Interpretation: Native improvement11 validates the entire submitted source, including the previously unsubmitted radius2 tail and earlier depth4/runtime scheduling; it does not isolate a component. Saved10 depth4406/control407 from408 gives a depth4-specific gain5 and sharedgain9. Fixed15-attempt cost on3 cases adds4.6..14.2percent, so the new method gets at most5s within25s. The12s/8s primary samples tie248 on4 cases; choose12s. Local27 improves4, recorded separately as finding_77382bf34b6c with stable local scope. Stage traces5/9 show horizon gains62to61/87to86, while primary caps are nonbinding; this corrects an earlier working-brief inference that primary consumed the full25s. Late refinements starve other positive tails in these instrumented samples, motivating a capped-budget comparison. Traces alter timing and are not an isolated ablation. Public per-case numeric scores are unavailable; native153.075s is aggregate. Native5timestamp1793702306.366869 sets next eligibility no earlier11:38:26.366869Z, subject to server admission. Five accepted completed/no pending; keep P1 open.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "d5d812896aa8aa55", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T11:08:25+00:00", + "result_id": "question_a43b5beda8342f41eee3f7c7cac42e62623064caaad65a9c01cf0a2aecbc1855", + "run_id": "explore-result:538350b38cb0a069ce211632e21442256c86e17f4f173224b7aeeaa008b88921", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Seeds0..9, saved8dc4cd incumbents; same seven compilers/routes/parameters,5extra seconds including shared routes; rotating method order and official tester.", + "title": "Can equal or weighted late-stage time budgets improve saved source8dc4cd programs within five extra seconds?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "2dc8aec658bc4995", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/late_budget_probe.json", + "research/reports/late_budget_summary.json", + "research/late_budget.py", + "research/late_budget_probe.py", + "research/turns/wake-1793703393262674400-dd1e63213897/execution_observation.json", + "research/turns/wake-1793703393262674400-dd1e63213897/weighted_manifest.json", + "research/turns/wake-1793703393262674400-dd1e63213897/evaluation_protocol.json", + "research/reports/late_budget_final_root_9.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:2fd9faec9dcb6de30f9c1963ac286bea61ab4069afc56038ac4e3a0ce633e6da", + "comparison_scope": "grid-turing-robot:saved-8dc4cd-10:extra5s:v1", + "evaluation_ref": "sha256:c49f8c965860d524f8842d83b156bce4bd57868c3ed768ed45f56b70e189f8fe", + "evidence_ref": "research/reports/late_budget_probe.json", + "higher_is_better": false, + "input_revision": "sha256:9bd6e7d40e3148c86ee2b1412e0642462101cb17a54080f6d9c6c4c51fd4400e", + "metric": "total_case_score", + "unit": "score", + "value": 406 + } + ], + "node_id": "question_a43b5beda8342f41eee3f7c7cac42e62623064caaad65a9c01cf0a2aecbc1855", + "recorded_at": "2026-11-03T11:08:25+00:00", + "result_id": "finding_538350b38cb0a069ce211632e21442256c86e17f4f173224b7aeeaa008b88921", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:9bd6e7d40e3148c86ee2b1412e0642462101cb17a54080f6d9c6c4c51fd4400e Applicability: Seeds0..9, saved8dc4cd incumbents; same seven compilers/routes/parameters,5extra seconds including shared routes; rotating method order and official tester. Observation: Current/equal/weighted totals406/406/406 from406;30 official outputs complete, no strict gains, max5.000952s including routes. Capped schedules reach transfer/loose/radius on cases5/9; root unchanged. Interpretation: This controlled negative answers only the10 saved-program, extra5s scope. On5/9 current order reaches zero transfer/radius attempts; equal/weighted reach64/70 and29/31 transfer attempts plus83/178 and35/79 radius attempts without fits. Redistributing effort increased coverage without reducing any tested objective, so do not integrate these schedules. All methods use unchanged compilers and incumbent copies; missing attempt counts for other stages remain unknown. The probe is not the actual native schedule and does not exclude benefits on other seeds, intermediate programs or different budgets. Preserve native2055/local1196 source8dc4cd and earlier scoped negative evidence. Initial list/count driver error was corrected before rerunning the complete comparison; rejected helper/stderr retained. Fresh unchanged root9 completes86 in25.109s with28032KiB RSS. Next independent probe compares depth4/5/6 reuse with branch cap8 and matched budgets; only useful gains warrant full27 and root validation. No new native evaluation during cooldown.", + "tags": [ + "writeback-result" + ], + "title": "Can equal or weighted late-stage time budgets improve saved source8dc4cd programs within five extra seconds?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "6ed196f8d3d2e8c5", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/fused_after_pool2.json", + "research/reports/fused_after_pool2_summary.json", + "research/reports/fused_reuse_summary.json", + "research/reports/deeper_horizon_summary.json", + "research/reports/fused_reuse_cost.json", + "research/reports/fused_winning_case.json", + "research/reports/fused_after_pool2_resource_116.json", + "research/turns/wake-1793704199105111701-87aa3169bd04/local_execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0bcb7f79db77504c022c6889b6789d39a9a785eb4117a924594955927cef1c97", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/fused_after_pool2.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1195 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T11:47:29Z", + "result_id": "finding_a0793236d4ed", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Exact source0bcb7f completes27 local cases at1195 versus1196, no case regressions, max25.123s; gain9:86to85. Actual root116 improves88to87 in25.106s/28516KiB RSS. New-rule depths4/5/6 tie406 on10 saved8dc4cd programs; fusednewrule4 plusmatchdepth0/2/3 scores406/405/406,30complete outputs eachfamily.20canonicalfixedcalls identical. Match2fixed15-attempt overhead13.2..33.4percent on3small batches; posthocC47/Q38 onlyalloc1/match2fits85 among9triples,allofficialoutputscomplete. Directearly replacement regresses248to249(sample4); split3s/2s full27=1197 losesold11/999gains. Retaining full canonical5s andpool23, thenfusion atmost2s/until25 restores those gains and adds9gain; one-second phase missed it. All26parentfunctions AST-identical,3functions added,permatchcounters disabled. This verifies combined ranking/placement underlocalconditions, not native improvement or universalmethod superiority. Native6accepted53e974a860fb pending;2055 still belongs to8dc4cd. Keep scoped negative sources and bestnative snapshot.", + "title": "Preserved canonical prefix plus fused-match search completes27 at1195 versus1196." + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "8143724ac6acdcb0", + "event_kind": "finding", + "evidence_refs": [ + "research/native/53e974a860fb/result.json", + "research/native/53e974a860fb/manifest.json", + "research/native/53e974a860fb/solution.py", + "research/reports/fused_after_pool2_summary.json", + "research/reports/fused_after_pool2_resource_116.json", + "research/reports/fused_reuse_summary.json", + "research/reports/jump_search_summary.json", + "research/turns/wake-1793704199105111701-87aa3169bd04/execution_observation.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0bcb7f79db77504c022c6889b6789d39a9a785eb4117a924594955927cef1c97", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/53e974a860fb/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2050 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T11:55:58+00:00", + "result_id": "finding_efee8a3f5857a5434febbcaa6a2b94a244697d498133dc3df8fa6e9272579e52", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Native6 53e974a860fb exact0bcb7f passes50/50 at2050 versus2055,0errors/timeouts. Local27=1195 versus1196,no case regressions; actualroot116 completes87,25.106s,28516KiB RSS. Jumpmargins tie259 on4. Interpretation: Nativegain5 validates fullsubmitted0bcb7f and its fusedranking/placement, not isolated choices. Local27 gain1(9:86to85) recorded separately finding_a0793236d4ed; stress11688to87. Saved newrule5/6 tie406; combined4/match2 scores405/control406,match3flat. Directreplacement249/248 andsplit1197/1196 regressed; preservecanonical5s/pool23 prefix thenfusionuntil25,26parentfunctions unchanged. PosthocC47/Q38 onlyalloc1/match2fits85;1scontrol misses searchorder. Instrumentedtrace9 showscanonical87to86 thenfusion86to85; not literalCLI timing or isolated ablation. Duringnativepending,margins1/2/3 each259 on4currentprograms,12complete,max5.001s;noadoption,otherinputsunknown. Publicpercasescoresnull;153.456s native runtimeaggregate. Sixacceptedcompleted/nonepending; submitted_at1793706275.7548187 setsnextfloor12:44:35.754819Z,subjectserver. Nextallocation1-first within1s mayreclaimtailtime; verifybeforeadoption. KeepGoal/P1open.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "b212f96f3e98e535", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T12:38:00+00:00", + "result_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "run_id": "explore-result:60ed9a3ea61a2ec84404c200d4df86fb07ac518a60c7e2bf858f778cc5c914cb", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only.", + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "0e7f47f4b881e323", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/reverse_route_tail.json", + "research/reports/reverse_route_tail_summary.json", + "research/reports/reverse_route_summary.json", + "research/reports/fused_priority1_1s_summary.json", + "research/reports/reverse_route_repeat42.json", + "research/reports/reverse_route_resource7.json", + "research/reports/reverse_route_resource116.json", + "research/turns/wake-1793707309934361684-b5977dd4c50e/execution_observation.json" + ], + "execution_observation": { + "candidate_sha256": "1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "change_reason": null, + "elapsed_seconds": 25.152105178000056, + "evidence_refs": [ + "research/reports/reverse_route_resource116.json", + "research/reports/reverse_route_resource7.json", + "research/reports/reverse_route_tail.json", + "research/reports/reverse_route_tail_summary.json", + "research/turns/wake-1793707309934361684-b5977dd4c50e/execution_observation.json" + ], + "executed_method_id": "reverse-route-tail-25s", + "exit_code": 0, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "limit_seconds": 30, + "planned_method_id": "reverse-route-tail-25s", + "result_candidate_sha256": "1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "result_ref": "research/reports/reverse_route_tail.json", + "schema_version": "explore_execution_observation_v0", + "search_iterations": null, + "stop_reason": "All27 complete; elapsed=max case walltime,limit=per case30s. Main25s cap or search exhaustion; iteration count unmeasured." + }, + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/reverse_route_tail.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1195 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T12:38:00+00:00", + "result_id": "finding_60ed9a3ea61a2ec84404c200d4df86fb07ac518a60c7e2bf858f778cc5c914cb", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd Applicability: Local generated seeds 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,42,99,123,999,2026,9999,10000; Python 3, 30-second and 512-MB case limits, supported tools/bin/tester only. Observation: Exact1d4 completes27 at1195 tie,max25.152s. Gain7:36to35;loss42:79to80. Oldroot42repeat79 andexactnew42repeat79; originalregression retained. Actualroot7 repeats35/25.102s/24956KiB;116 retains87/25.125s/29280KiB. Native1d4 unknown. Interpretation: The27-case tie and within-candidate42 variation are established, not a corrected1194 aggregate, isolated cause or native gain. Active1d4 keeps all29functions andoldmain prefix, adding reversed shortest construction only when0.7s remains to25s. Matched extra5s forward/reverse on10saved0bcb programs scores405/404, all20outputscomplete,only7gain;30newroutes legal/same shortest length andall differ fromforward. Allocation1-first extra1s reached known9fitat0.629s, totals406/405/406 vscontrol/first2 onolder8dc10,30complete; its whole1s variant ties1195 butloses11687to88,so rejected. Do not conflate datasets/budgets. Preserve nativebest2050 andfirstbestlocal1195 source0bcb separately; use1d4 as validated nextnative candidate. Six prioracceptednativecompleted; no newsubmission before12:44:35.754819Z plusserver. Next independent test combines deepernewrule5/6 withmatch2; previous deepernegative usedmatch0. Scopedquestionanswered; Goal/P1continue.", + "tags": [ + "writeback-result" + ], + "title": "Does reverse trajectory compilation produce complete compact programs on the declared 27-seed corpus?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "cacb4166a1a37f2b", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T12:57:18+00:00", + "result_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "run_id": "explore-result:e1836e8496e129739343b33fc1b55009fc2a141c0f1c9423430bf7d06c455359", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved.", + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "b720eaaaef144b97", + "event_kind": "finding", + "evidence_refs": [ + "research/native/7f156c1e602a/result.json", + "research/native/7f156c1e602a/manifest.json", + "research/native/7f156c1e602a/solution.py", + "research/reports/fused_deeper_summary.json", + "research/reports/fused_weight_mix_summary.json", + "research/reports/native7_final_root42.json", + "research/turns/wake-1793709660328732653-f8c4cc464818/execution_observation.json", + "research/turns/wake-1793709660328732653-f8c4cc464818/next_route_priority_plan.json" + ], + "execution_observation": { + "candidate_sha256": "1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "change_reason": null, + "elapsed_seconds": null, + "evidence_refs": [ + "research/native/7f156c1e602a/manifest.json", + "research/native/7f156c1e602a/result.json", + "research/turns/wake-1793709660328732653-f8c4cc464818/execution_observation.json" + ], + "executed_method_id": "reverse-route-tail-native50", + "exit_code": null, + "input_revision": "grid_turing_robot:native-fixed-50", + "limit_seconds": null, + "planned_method_id": "reverse-route-tail-native50", + "result_candidate_sha256": "1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "result_ref": "research/native/7f156c1e602a/result.json", + "schema_version": "explore_execution_observation_v0", + "search_iterations": null, + "stop_reason": "Native50 completed; percase30s/512MiB. Reported154.227s runtime is aggregate, not elapsed/percase; iterations unmeasured." + }, + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:1d4c83e2b5c42c18e5a7d35c65a9173f8e2fcbc086352b64e77c131c574450e3", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/7f156c1e602a/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2048 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T12:57:18+00:00", + "result_id": "finding_e1836e8496e129739343b33fc1b55009fc2a141c0f1c9423430bf7d06c455359", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Native7 7f156c1e602a exact1d4 scores2048 vs2050,50/50,0errors/timeouts. Full27 remains1195tie with7gain/42loss retained. Actualroot42 repeats79,25.106s,28196KiB. Both depth4/5/6match2 andweightmix control/geo/linear tie404 on10. Interpretation: The complete1d4 submission improves the nativefixed50 aggregate by2; this doesnotisolate whichpath or timing effect causedit. Archive8b23...8209bytes andsource1d4 snapshots agree before/afterupload. Publicpercase scores/messages/weightsnull,metrics{};154.227s reportedruntimeaggregate. Priorfull27 1195tie includes7:36to35 and42:79to80; fresh42=79 demonstratesvariation, doesnoteraseoriginalregression orproducecorrected1194. Deepernewrule4/5/6 WITHmatch2 onstronger1d4saved10 all404,30outputscomplete,max5.0004s; mixedendpoint/geometric/linearweights all404,30complete,max5.0007s,clonecontrol result/stats exact on2/3. Theseareseparate5s savedprogram negatives, notuniversal bans or nativeablations; noadoption. Keepnative2048/source1d4 andold0bcb2050, allreports. Sevenacceptedcompleted/nonepending; server submitted_at1793710010.2521164 setsnextfloor13:46:50.252116Z, notresultarrival. NextphysicalBFS priorityprobe followsnewpath evidence, distinctfromoldnumericstateorderingnegative. Scopedquestionanswered; Goal/P1continue.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "e6e34f87560a76f5", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/joint_route_tail.json", + "research/reports/joint_route_tail_summary.json", + "research/reports/joint_route_resource42.json", + "research/reports/joint_route_resource116.json", + "research/turns/wake-1793710939781155513-79360cad50f9/local_protocol_readback.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:e439ddfb96390ba8f33b0c83ff96ae173817741e129cb89bfbdc61946efe35f6", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/joint_route_tail.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1184 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T13:53:38Z", + "result_id": "finding_d70b55930ec54d66829d15656a0609074746082ec7bacf8a30aef293f59beb1b", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Exacte439 standalone28.5s search retains30 original1d4 functions/mainprefix and adds reuse-first shortest-predecessor joint packing. Whole27 complete1184 versus original1d4 1195: gains4,11,15,16,17,18,42, no regressions; max28.639412s. Saved10 five-second proxy402vs404 uses different budget; seed0 gain doesnotrepeat standalone. Actualroot42=77/28.609s/29956KiB and116=86/28.610s/30324KiB complete; full27 peakRSS unmeasured. Native8 d7371ed25f19 accepted exacte439/archiveb2d7...9762bytes, stillrunning at publication; nativebest2048/source1d4 remains distinct. No isolatedcause or generalization claim. Same input/evaluator bytes and30s/512MB conditions verified; seven case gains are retained as whole-run evidence.", + "title": "Reuse-first joint shortest-path/rule tail completes27 at1184 versus1195." + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "bee4873232600ed0", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T14:02:51+00:00", + "result_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "run_id": "explore-result:d39cca9fe845b3418c9e35076e5cf27d71633e4c6f92f294cea98fc640fa1155", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved.", + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "256f38c74657bd19", + "event_kind": "finding", + "evidence_refs": [ + "research/native/d7371ed25f19/result.json", + "research/native/d7371ed25f19/manifest.json", + "research/native/d7371ed25f19/solution.py", + "research/reports/joint_route_tail_summary.json", + "research/reports/route_design_research.json", + "research/reports/joint_pruning_probe.json", + "research/turns/wake-1793710939781155513-79360cad50f9/execution_observation.json", + "research/reports/joint_pruning_diagnostic.json" + ], + "execution_observation": { + "candidate_sha256": "e439ddfb96390ba8f33b0c83ff96ae173817741e129cb89bfbdc61946efe35f6", + "change_reason": null, + "elapsed_seconds": null, + "evidence_refs": [ + "research/native/d7371ed25f19/manifest.json", + "research/native/d7371ed25f19/result.json", + "research/turns/wake-1793710939781155513-79360cad50f9/execution_observation.json" + ], + "executed_method_id": "reuse-first-joint-route-native50", + "exit_code": null, + "input_revision": "grid_turing_robot:native-fixed-50", + "limit_seconds": null, + "planned_method_id": "reuse-first-joint-route-native50", + "result_candidate_sha256": "e439ddfb96390ba8f33b0c83ff96ae173817741e129cb89bfbdc61946efe35f6", + "result_ref": "research/native/d7371ed25f19/result.json", + "schema_version": "explore_execution_observation_v0", + "search_iterations": null, + "stop_reason": "Native8 completed50/50; percase30s/512MB. Reported176.041s is aggregate; elapsed, iterations and peakRSS unmeasured." + }, + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:e439ddfb96390ba8f33b0c83ff96ae173817741e129cb89bfbdc61946efe35f6", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/d7371ed25f19/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2029 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T14:02:51+00:00", + "result_id": "finding_d39cca9fe845b3418c9e35076e5cf27d71633e4c6f92f294cea98fc640fa1155", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Native8 exact e439:2029 vs2048,50/50; full27:1184 vs1195,no losses. Root42/116 complete at28.61s/<=30324KiB. Joint5s:402vs404; pruning matches six finished outputs/routes and improves3.5s proxy403to402. Separate59cf smoke0=43. Interpretation: Native8 is bound to immutablee439, archiveb2d7...9762bytes;2029 improves2048by19 with noerrors/timeouts. Public percase scores/messages/weights are null;176.041s is reported runtimeaggregate, not elapsed or percase. Full27 gains4,11,15,16,17,18,42, no losses; original1d4 total1195 remains comparator despite favorable42repeats. Physical BFS bank tie had0novelty, so it doesnotreject novel paths. Fixed hybrids created40new legal shortest trajectories but tied404; mixed-credit joint1/2 also tied404. Reuse-first joint2 achieved402 under5s and supported the28.5s standalone tail. Subsequent pruning only removes new-rule candidates lexically dominated by any existing match: six nonbinding exact program/route comparisons plus timeout/capacity checks pass;3.5s old/pruned403/402,20complete outputs. Separate59cf standalone0=43/28.606s; no whole27,1183or native score inferred. Keep evaluatede439/2029 and1d4/2048 fallback. Native accepted_at1793713886.487855 sets floor14:51:26.487855Z plusserver;8completed/nonepending. Scoped native question answered; Goal/P1 remain active. Next full27/resource validation of59cf.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "6ee601501d68ed9a", + "event_kind": "finding", + "evidence_refs": [ + "research/reports/guarded_pruned_full27.json", + "research/reports/guarded_pruned_summary.json", + "research/reports/guarded_pruned_root17.json", + "research/reports/guarded_pruned_root116.json", + "research/reports/seed17_repeats.json", + "research/reports/seed17_profile_summary.json", + "research/reports/deadline_guard_negative.json", + "research/turns/wake-1793714917306718118-7fdcb97ee944/local_protocol_readback.json" + ], + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0a890023601de75df0f92e2feb651998673c19f155b1cab02ffe639d68f8bf57", + "comparison_scope": "grid-turing-robot:local-27:30s-512mb:v1", + "evaluation_ref": "sha256:7fbe2f5df81193f650cfb2b79e1ed1d991225a8980bcb780c76dc56efb866828", + "evidence_ref": "research/reports/guarded_pruned_full27.json", + "higher_is_better": false, + "input_revision": "sha256:dbb983128d11a49c57ff7b3f1149839b97415c3f55f20b4c0406599e8b08bbcd", + "metric": "total_case_score", + "unit": "score", + "value": 1182 + } + ], + "node_id": "question_c76b736308bd1567685f89e5997fd96c605f921ca9dce7d7119b8e6d493b0e84", + "recorded_at": "2026-11-03T14:51:03Z", + "result_id": "finding_d72ca52a5103409803c3353620d4d1a7c2d2c55818cc0306766dd1e67f345668", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Exact0a89 ordinary full27 completes1182 vsE4391184, gains0:44to43 and999:74to73,no losses,max28.633s. Distinct59cf attempt had31s timeout17 after0..16; no full59cf aggregate. Three serial17 repeats and owned CPU/phase capture complete63; original cause unresolved. A29s signal fallback after valid initialsolve preserves the last complete incumbent on optional late overruns;34 searchfunctions unchanged. Synthetic delayed-phase0/17 guarded2/2 completefixtures while unguarded2/2 exceed.6s; this is fallback semantics,not naturalrootcause proof. Actualroot17=63/28.630s/28544KiB,116=86/28.620s/32628KiB. Profile timing is not acceptance; full27 peakRSS unmeasured. Nativebest remains2029/E439;0a89 nativeunmeasured at publication. Input/evaluator bytes verified unchanged, same30s/512MB case budget; whole-method score,not isolatedcause.", + "title": "Dominance-pruned compiler with late-search fallback completes27 at1182 versus1184." + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "4d2e8ca3ae7f33a6", + "event_kind": "node", + "goal_id": "benchmark-goal", + "node_kind": "question", + "recorded_at": "2026-11-03T15:06:44+00:00", + "result_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "run_id": "explore-result:f0b0d89d62c5ad0e1c8175fb52a0ddbb7cc939bfd298443e7ef631bcc79ef380", + "schema_version": "loopx_explore_result_event_v0", + "status": "resolved", + "summary": "The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved.", + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + }, + { + "agent_id": "benchmark-agent", + "boundary": { + "absolute_paths_recorded": false, + "credential_values_recorded": false, + "raw_logs_recorded": false, + "raw_session_transcript_recorded": false, + "raw_task_text_recorded": false, + "raw_trajectory_recorded": false + }, + "event_id": "dc5af692855b7a51", + "event_kind": "finding", + "evidence_refs": [ + "research/native/63cd98946c33/result.json", + "research/native/63cd98946c33/manifest.json", + "research/native/63cd98946c33/solution.py", + "research/reports/guarded_pruned_summary.json", + "research/reports/seed17_profile_summary.json", + "research/reports/deadline_guard_negative.json", + "research/reports/joint_qmajor_guarded27_summary.json", + "research/turns/wake-1793714917306718118-7fdcb97ee944/execution_observation.json" + ], + "execution_observation": { + "candidate_sha256": "0a890023601de75df0f92e2feb651998673c19f155b1cab02ffe639d68f8bf57", + "change_reason": null, + "elapsed_seconds": null, + "evidence_refs": [ + "research/native/63cd98946c33/manifest.json", + "research/native/63cd98946c33/result.json", + "research/turns/wake-1793714917306718118-7fdcb97ee944/execution_observation.json" + ], + "executed_method_id": "pruned-guarded-native50", + "exit_code": null, + "input_revision": "grid_turing_robot:native-fixed-50", + "limit_seconds": null, + "planned_method_id": "pruned-guarded-native50", + "result_candidate_sha256": "0a890023601de75df0f92e2feb651998673c19f155b1cab02ffe639d68f8bf57", + "result_ref": "research/native/63cd98946c33/result.json", + "schema_version": "explore_execution_observation_v0", + "search_iterations": null, + "stop_reason": "Native9 completed50/50;30s/512MB percase.172.456s report is aggregate, not elapsed/percase; iterations and nativepeakRSS unmeasured." + }, + "goal_id": "benchmark-goal", + "measurements": [ + { + "candidate_ref": "sha256:0a890023601de75df0f92e2feb651998673c19f155b1cab02ffe639d68f8bf57", + "comparison_scope": "grid-turing-robot:native-fixed-50:30s-512mb:v1", + "evaluation_ref": "sha256:bcb472528152b802715c535ff4d1efbf4ad71429dc0bc156e383e0982696b1ab", + "evidence_ref": "research/native/63cd98946c33/result.json", + "higher_is_better": false, + "input_revision": "grid_turing_robot:native-fixed-50", + "metric": "total_case_score", + "unit": "score", + "value": 2026 + } + ], + "node_id": "question_1eb23eb6b3c92e03f94c5ac7b092c1dec17740320768fc5cef4c04d4dd522aab", + "recorded_at": "2026-11-03T15:06:44+00:00", + "result_id": "finding_f0b0d89d62c5ad0e1c8175fb52a0ddbb7cc939bfd298443e7ef631bcc79ef380", + "schema_version": "loopx_explore_result_event_v0", + "status": "confirmed", + "summary": "Input revision: grid_turing_robot:native-fixed-50 Applicability: The task-declared native fixed 50-case suite, 30 seconds and 512 MB per case; exact source revision and accepted submission receipt are preserved. Observation: Native9 exact0a89:2026 vs2029,50/50. Whole27:1182 vs1184,no losses; root17/116 <=28.630s/32628KiB. Original59cf timeout retained. Qextra27 ties1181;Q16fit0.69s vs3.07/miss; separate90ab smoke16=55 only. Interpretation: Exact0a89/archive8573...10036bytes completes50 at2026,3 below2029. Native172.456s is reported aggregate runtime; percase scores/messages/weightsnull. New full27 source0a89 scores1182 vs1184,gains0/999,no losses; it is not a repaired59cf aggregate. Initial59cf exceeded31s on17;3 ordinary repeats and owned CPU/phase capture passed63 without reproducing it. Cause stays unresolved; profile time is not a benchmark. A29s alarm after valid initialsolve preserves a complete incumbent during late overruns; forced delayed-phase0/17 guard passes,controltimesout. Full27 max28.633s; actualroot17/116 <=28.630s,<=32628KiB. Independent saved0a89 extra3.5s Q/original both1181; only16fits55. Two Qfirstfits0.687/0.695s vsoriginal3.072s/miss are notp95. Separate90ab runnable16=55/28.628s only; no full27,1181or native outcome inferred. Root stays0a89/2026,E439/2029 retained.9completed/nonepending;accepted_at1793717686.5200238 sets floor15:54:46.520024Z plusserver. Nativequestion answered;Goal/P1 continue with full90ab/negative checks.", + "tags": [ + "writeback-result" + ], + "title": "Does the compact reverse-compiled robot program complete the native 50-case suite, and what baseline score does it achieve?" + } + ] +} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index 1ba9888b4c..5a48c74ce0 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -22,7 +22,8 @@ type Pack = ResearchProgressPack & {source_count: number; analysis_key: string; objective?: string; episodes?: TrajectoryEpisode[]; job?: Job | null; continuation_unavailable?: {base_pack_id: string}; continuation?: TrajectoryContinuation & {base_pack_id: string; base_title: string; base_available: boolean; base_episode_count: number}}; -type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]; unreadable_pack_ids?: string[]}; pack: Pack | null}; +type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]; unreadable_pack_ids?: string[]}; pack: Pack | null; + originalPack?: Pack | null; originalReadError?: string}; type Citation = AssessmentCitation & {evidence_kind?: "usage_counters"; artifact: string; text: string; applicability: string}; const errorText = (error: unknown) => error instanceof Error ? error.message : String(error); const locationFor = (goal: string) => { @@ -31,6 +32,44 @@ const locationFor = (goal: string) => { return {open: query.get("research_analysis") === goal, packId: owner === goal ? query.get("research_pack") : null}; }; +export async function readAnalysisSnapshot(goalId: string, packId: string | null, zh: boolean, + readSourceUrl?: string, signal?: AbortSignal, isCurrent: () => boolean = () => true): Promise { + const endpoint = "/api/chat/research-analysis", outputLanguage = zh ? "zh" : "en"; + const query = new URLSearchParams({goal_id: goalId, output_language: outputLanguage, ...(packId ? {pack_id: packId} : {})}); + const value = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}?${query}`), {signal}); + if (!value?.ok || !Array.isArray(value.catalog?.packs) || (value.pack && (value.pack.pack_id !== (packId ?? value.catalog.default_pack_id) + || value.pack.config.output_language !== outputLanguage || (value.pack.result && (value.pack.result.output_language !== outputLanguage + || value.pack.result.analysis_key !== value.pack.analysis_key || value.pack.result.method_version !== value.pack.method_version + || JSON.stringify(value.pack.result.scope) !== JSON.stringify(value.pack.scope)))))) { + throw new Error(zh ? "分析来源读回不一致。" : "Analysis source readback does not match."); + } + // Older registered packs have a successful, language-unspecified identity. + // Read that exact pack through the existing owner; never relabel its + // original output as a result generated in the interface language. The + // owning Goal can differ from the solver Goal in the pack's analysis scope. + let originalPack: Pack | null = null, originalReadError = ""; + if (value.pack && !value.pack.result && isCurrent() && !signal?.aborted) { + const originalQuery = new URLSearchParams({goal_id: goalId, pack_id: value.pack.pack_id}); + try { + const original = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}?${originalQuery}`), {signal}); + const p = original.pack; + if (!original.ok || !p || p.pack_id !== value.pack.pack_id + || p.method_version !== value.pack.method_version + || JSON.stringify(p.scope) !== JSON.stringify(value.pack.scope)) { + throw new Error(zh ? "原文分析来源读回不一致。" : "Original analysis source readback does not match."); + } + if (p.config.output_language === undefined && p.result) { + if (p.result.output_language !== undefined || p.result.analysis_key !== p.analysis_key + || p.result.method_version !== p.method_version || JSON.stringify(p.result.scope) !== JSON.stringify(p.scope)) { + throw new Error(zh ? "原文分析结果身份不一致。" : "Original analysis result identity does not match."); + } + originalPack = p; + } + } catch (e) {originalReadError = errorText(e);} + } + return {...value, originalPack, originalReadError}; +} + export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean, readSourceUrl?: string) { const endpoint = "/api/chat/research-analysis"; const outputLanguage = zh ? "zh" : "en"; @@ -46,7 +85,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean {scope: sourceScope, packId: location.packId, snapshot: null, error: "", loading: true}); // Scope changes hide old data synchronously, before effects start the next read. const snapshot = readState.scope !== sourceScope ? null : readState.packId === location.packId - ? readState.snapshot : readState.snapshot ? {...readState.snapshot, pack: null} : null; + ? readState.snapshot : readState.snapshot ? {...readState.snapshot, pack: null, originalPack: null, originalReadError: ""} : null; const readError = readState.scope === sourceScope && readState.packId === location.packId ? readState.error : ""; const loading = readState.scope !== sourceScope || readState.packId !== location.packId || readState.loading; const [error, setError] = useState(""); @@ -65,17 +104,19 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean const isCurrent = () => current === generation.current && sequence === readSequence.current && scope.current.sourceScope === sourceScope && scope.current.packId === packId && !signal?.aborted; setReadState(previous => ({scope: sourceScope, packId, snapshot: previous.scope === sourceScope - ? previous.packId === packId ? previous.snapshot : previous.snapshot ? {...previous.snapshot, pack: null} : null : null, + ? previous.packId === packId ? previous.snapshot : previous.snapshot ? {...previous.snapshot, pack: null, originalPack: null, originalReadError: ""} : null : null, error: previous.scope === sourceScope && previous.packId === packId ? previous.error : "", loading: true})); - const query = new URLSearchParams({goal_id: goalId, output_language: outputLanguage, ...(packId ? {pack_id: packId} : {})}); try { - const value = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}?${query}`), {signal}); - if (!value?.ok || !Array.isArray(value.catalog?.packs) || (value.pack && (value.pack.pack_id !== (packId ?? value.catalog.default_pack_id) - || value.pack.config.output_language !== outputLanguage || (value.pack.result && value.pack.result.output_language !== outputLanguage)))) { - throw new Error(zh ? "分析来源读回不一致。" : "Analysis source readback does not match."); - } + const value = await readAnalysisSnapshot(goalId, packId, zh, readSourceUrl, signal, isCurrent); if (isCurrent()) { - setReadState({scope: sourceScope, packId, snapshot: value, error: "", loading: false}); + setReadState(previous => { + const saved = previous.snapshot?.originalPack; + return {scope: sourceScope, packId, snapshot: {...value, + originalPack: value.originalReadError && previous.scope === sourceScope && previous.packId === packId + && saved && saved.pack_id === value.pack?.pack_id + ? {...saved, applicability: "historical", availability: "unavailable", reason_code: "source_unavailable"} + : value.originalPack}, error: "", loading: false}; + }); const observation = JSON.stringify([value.pack?.analysis_key, value.pack?.job?.attempt, value.pack?.job?.status]); if (observation !== lastJobObservation.current && (value.pack?.job?.status === "succeeded" || value.pack?.job?.status === "cancelled")) setError(""); lastJobObservation.current = observation; @@ -94,7 +135,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); setPending(false); setCancelPending(false); setError(""); setReadState(previous => ({scope: sourceScope, packId: next.packId, snapshot: previous.scope === sourceScope && previous.snapshot - ? previous.snapshot.pack?.pack_id === next.packId ? previous.snapshot : {...previous.snapshot, pack: null} : null, error: "", loading: true})); + ? previous.snapshot.pack?.pack_id === next.packId ? previous.snapshot : {...previous.snapshot, pack: null, originalPack: null, originalReadError: ""} : null, error: "", loading: true})); } scope.current = {goalId, packId: next.packId, outputLanguage, sourceScope}; setLocation(next); @@ -116,6 +157,8 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean }, [active, read, location.packId]); const currentSnapshot = snapshot; const pack = currentSnapshot?.pack ?? null; + const readingPack = currentSnapshot?.originalPack ?? pack; + const savedOriginal = Boolean(currentSnapshot?.originalPack); useEffect(() => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false);}, [goalId, readSourceUrl, snapshot?.pack?.pack_id, snapshot?.pack?.analysis_key]); useEffect(() => { if ((!citation && !citationBusy && !citationError) || !active) return; @@ -154,7 +197,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean requestAnimationFrame(() => {panel.current?.focus({preventScroll: true}); panel.current?.scrollIntoView({block: "start"});}); }; const execute = async (retry = false) => { - if (readSourceUrl && snapshot?.execution_enabled !== true) return; + if (snapshot?.execution_enabled === false || (readSourceUrl && snapshot?.execution_enabled !== true)) return; if (readError || !pack || pack.availability !== "available" || submitting.current) return; show(); submitting.current = true; setPending(true); setError(""); const current = generation.current; @@ -165,7 +208,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean finally {if (current === generation.current) {submitting.current = false; setPending(false);}} }; const cancel = async () => { - if (readSourceUrl && snapshot?.execution_enabled !== true) return; + if (snapshot?.execution_enabled === false || (readSourceUrl && snapshot?.execution_enabled !== true)) return; if (!pack || cancelPending) return; const current = generation.current; setCancelPending(true); @@ -176,11 +219,12 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean finally {if (current === generation.current) setCancelPending(false);} }; const readEvidence = async (selection: Record, route: string, origin?: HTMLButtonElement) => { - const p = pack; + const p = readingPack; if (!p) return; if (origin) sourceTrigger.current = origin; const current = ++citeGeneration.current; setCitation(null); setCitationBusy(true); setCitationError(""); - const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, output_language: outputLanguage, ...selection}); + const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, + ...(p.config.output_language ? {output_language: p.config.output_language} : {}), ...selection}); try {const value = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}/${route}?${query}`)); if (current === citeGeneration.current) setCitation(value);} catch (e) {if (current === citeGeneration.current) setCitationError(errorText(e));} finally {if (current === citeGeneration.current) setCitationBusy(false);} @@ -188,7 +232,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean const readCitation = (c: AssessmentCitation, origin?: HTMLButtonElement) => readEvidence({source_id: c.source_id, first_line: String(c.first_line), last_line: String(c.last_line)}, "citation", origin); const readUsageTurn = (turnId: string, origin: HTMLButtonElement) => readEvidence({turn_id: turnId}, "usage-turn", origin); const closeCitation = () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); requestAnimationFrame(() => sourceTrigger.current?.focus());}; - return {readOnly: Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true, citation, citationBusy, citationError, sourcePanel, readCitation, readUsageTurn, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel, + return {readOnly: currentSnapshot?.execution_enabled === false || (Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true), citation, citationBusy, citationError, sourcePanel, readCitation, readUsageTurn, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack: readingPack, requestPack: pack, savedOriginal, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel, close: () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); navigate(false); requestAnimationFrame(() => trigger.current?.focus());}, select: (id: string) => navigate(location.open, id, true)}; } type Controller = ReturnType; @@ -209,9 +253,9 @@ export function ResearchAnalysisButton({analysis: a}: {analysis: Controller}) { const sourceUnavailable = !a.loading && !a.readError && a.snapshot && (!a.pack || a.pack.availability !== "available"); // The action keeps its separate-usage consequence visible. Further method // explanation is optional; cached results and unavailable states are reads. - const readOnly = unavailable || a.readError || a.pack?.result || a.pending || a.recordedRunning; + const readOnly = unavailable || a.readError || a.snapshot?.originalReadError || a.pack?.result || a.pending || a.recordedRunning; return
{sourceUnavailable ?

{reasonCopy(a.pack?.reason_code ?? a.snapshot?.catalog.reason_code, a.zh)}

: null} {a.pack ?

{a.zh ? "分析请求设置（固定）：" : "Fixed analysis request: "}{a.pack.config.model} / {a.pack.config.effort} / {a.pack.config.service_tier}

: null} @@ -222,7 +266,7 @@ export function ResearchAnalysisButton({analysis: a}: {analysis: Controller}) {

{a.pack?.result ? (a.zh ? "所选范围已有分析，查看不调用 Agent。" : "An analysis exists for the selected scope. Viewing never invokes an Agent.") : a.pack?.continuation?.can_reuse ? (a.zh ? "本次复用已有判读，不调用模型；原用量保留在上一版。" : "This action reuses existing judgments without a model call; original usage stays with the previous version.") : (a.zh ? "点击深度分析使用本机 Codex 账户，单独产生分析用量；普通浏览不调用 Agent。" : "Deep analysis uses the local Codex account with separate analysis usage; ordinary browsing never invokes an Agent.")}

{a.zh ? "只分析已授权的固定轨迹，不改变研究历史。范围、方法与来源版本可在结果中查看。" : "Analyzes an authorized fixed trajectory without changing research history. Scope, method and source revisions remain available with the result."}

{a.pack?.continuation ?

{a.zh ? "接续保留上一版。新增、更正或删除可能影响旧判断，本版重新评估全部片段；仅证据内容与配置等价时复用已有判读。浏览与来源更新不启动分析。" : "Continuation preserves the previous version. Additions, corrections and removals can affect earlier judgments, so all episodes are reassessed. Existing judgments are reused only for equivalent evidence and configuration. Browsing and source updates never start analysis."}

: null} -
{a.pack && !a.pack.result && !unavailable ? {a.pack.continuation?.can_reuse ? (a.zh ? "无需新模型调用" : "No new model call") : <>{a.pack.config.model} · {a.zh ? "单独用量" : "Separate usage"}} : null}
; + {a.savedOriginal ? {a.zh ? "已保存原文" : "Saved original"} : a.pack && !a.pack.result && !unavailable ? {a.pack.continuation?.can_reuse ? (a.zh ? "无需新模型调用" : "No new model call") : <>{a.pack.config.model} · {a.zh ? "单独用量" : "Separate usage"}} : null}; } const analysisActionLabel = (p: Pack | null, zh: boolean) => p?.continuation?.can_reuse @@ -261,12 +305,14 @@ function AnalysisContinuation({pack: p, analysis: a}: {pack: Pack; analysis: Con } function AnalysisReadState({analysis: a}: {analysis: Controller}) { - if (a.readError) return
+ const error = a.readError || a.snapshot?.originalReadError; + if (error) return
+ {a.snapshot?.originalReadError && !a.readError ? {a.zh ? "已保存原文暂不可读，当前语言的分析状态仍保留。" : "The saved original could not be read. The current language’s analysis state is preserved."} : {a.pack ? (a.zh ? "刷新失败，保留上次结果。" : "Refresh failed. Showing the last result.") - : (a.zh ? "暂时无法读取分析。" : "Analysis could not be read.")} + : (a.zh ? "暂时无法读取分析。" : "Analysis could not be read.")}} -

{a.readError}

 +

{error}
; if (!a.pack && a.loading && (!a.snapshot || a.snapshot.catalog.packs.length > 0)) return
{Array.from({length: 6}, (_, i) => )}
; @@ -482,8 +528,9 @@ export function ResearchAnalysisPanel({analysis: a, onHistory}: {analysis: Contr if (!a.location.open) return null; const zh = a.zh, p = a.pack, result = p?.result; const resultJob = p?.result_job ?? p?.job; - const failed = p?.job?.status === "failed" || p?.job?.status === "cancelled"; - const sourceAvailable = p?.availability === "available"; + const currentJob = a.requestPack?.job; + const failed = currentJob?.status === "failed" || currentJob?.status === "cancelled"; + const sourceAvailable = a.requestPack?.availability === "available"; const available = sourceAvailable && !a.readOnly && !a.readError; const benefit = result?.metrics.find(m => m.metric === "benefit_test"); const headline = !result ? null : benefit?.no ? (zh ? "仍有研究片段尚未检验原目标收益" : "Some episodes have not tested original-objective utility") @@ -508,22 +555,24 @@ export function ResearchAnalysisPanel({analysis: a, onHistory}: {analysis: Contr {p ? <> {!result?.synthesis ?
{scope}{scopeHelp}
: null} + {a.savedOriginal ?

{zh ? "显示已保存的原文分析，未按当前界面语言生成。原判断、引用和用量保持不变。" : "Showing the saved original, not an analysis generated in the current interface language. Its judgments, citations and usage are unchanged."}

: null} {p.result && !sourceAvailable ?

{reasonCopy(p.reason_code, zh)}

: null} {p.result_attempt && p.result_attempt !== p.job?.attempt ?

{zh ? "当前重试尚未完成，显示上次已保存分析。" : "The current retry is incomplete; showing the previously saved analysis."}

 : null} - {result?.display?.coverage.missing && !a.pending && !a.recordedRunning ?
+ {!a.savedOriginal && result?.display?.coverage.missing && !a.pending && !a.recordedRunning ?
{zh ? "重新分析完整固定范围，产生新的分析用量；保留上一轮结果。" : "Reanalyzes the full fixed scope with new analysis usage; the previous result is preserved."}
: null} - {!result || a.pending || a.recordedRunning ?
- {a.pending || a.recordedRunning ? <>

{a.pending ? : null}{a.pending ? (zh ? "正在处理分析请求…" : "Processing analysis request…") : (zh ? "上次记录仍待分析结果；当前执行状态尚未核验。" : "The last receipt awaits a result; current execution is unverified.")}

- : result ? + {!result || a.savedOriginal || a.pending || a.recordedRunning ?
+ {a.pending || a.recordedRunning ? <>

{a.pending ? : null}{a.pending ? (zh ? "正在处理分析请求…" : "Processing analysis request…") : (zh ? "上次记录仍待分析结果；当前执行状态尚未核验。" : "The last receipt awaits a result; current execution is unverified.")}

+ : a.savedOriginal && !a.readOnly ? + : result ? : a.readOnly ? <>

{sourceAvailable ? (zh ? "此来源尚无已保存的分析。" : "This source has no saved analysis yet.") : reasonCopy(p.reason_code, zh)}

: } - {!result && !a.readOnly && !a.pending && !a.recordedRunning ? {p.continuation?.can_reuse ? (zh ? "复用已有判读，无需新模型调用。" : "Reuses existing judgments without a new model call.") : zh ? "使用本机 Codex 账户，单独产生分析用量；浏览不调用 Agent。" : "Uses the local Codex account with separate analysis usage. Browsing never invokes an Agent."} : null} + {(!result || a.savedOriginal) && !a.readOnly && !a.pending && !a.recordedRunning ? {a.requestPack?.continuation?.can_reuse ? (zh ? "复用已有判读，无需新模型调用。" : "Reuses existing judgments without a new model call.") : zh ? "使用本机 Codex 账户，单独产生分析用量；浏览不调用 Agent。" : "Uses the local Codex account with separate analysis usage. Browsing never invokes an Agent."} : null}
: null} - {failed && !a.pending ?

{p.job?.status === "cancelled" ? (zh ? "分析已取消，研究历史保持可读。" : "Analysis cancelled. Research history remains readable.") : a.readOnly ? (zh ? "上次分析未完成；失败记录已保留。" : "The last analysis did not complete; its failure record is preserved.") : (zh ? "上次分析未完成；可重试，原失败记录会保留。" : "The last analysis did not complete. Retry preserves its failure record.")}

: null} + {failed && !a.pending ?

{currentJob?.status === "cancelled" ? (zh ? "分析已取消，研究历史保持可读。" : "Analysis cancelled. Research history remains readable.") : a.readOnly ? (zh ? "上次分析未完成；失败记录已保留。" : "The last analysis did not complete; its failure record is preserved.") : (zh ? "上次分析未完成；可重试，原失败记录会保留。" : "The last analysis did not complete. Retry preserves its failure record.")}

: null} {failed && !a.pending ?

- {zh ? "本次尝试" : "This attempt"} · {p.job?.analysis_wall_seconds == null ? (zh ? "耗时未知" : "Time unknown") : `${p.job.analysis_wall_seconds.toFixed(1)}s`} · {zh ? "输入" : "input"} {p.job?.usage?.input_tokens?.toLocaleString() ?? "—"} / {zh ? "输出" : "output"} {p.job?.usage?.output_tokens?.toLocaleString() ?? "—"} tokens + {zh ? "本次尝试" : "This attempt"} · {currentJob?.analysis_wall_seconds == null ? (zh ? "耗时未知" : "Time unknown") : `${currentJob.analysis_wall_seconds.toFixed(1)}s`} · {zh ? "输入" : "input"} {currentJob?.usage?.input_tokens?.toLocaleString() ?? "—"} / {zh ? "输出" : "output"} {currentJob?.usage?.output_tokens?.toLocaleString() ?? "—"} tokens

: null} {result ? <> {result.synthesis ? diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx index 5d797a2d6b..a0ad487b59 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx @@ -607,8 +607,8 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re {recordsOpen ? : null} : null} ; diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-timeline.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-timeline.tsx index 39a3fd470b..96bac7196a 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-timeline.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-timeline.tsx @@ -201,11 +201,13 @@ export function GoalResearchTimeline({timeline, zh, selectedIds, selectedEventId {unknownTime ?

{zh ? "缺少记录时间的问题未放上时间轴" : "Questions without a record time are not placed on the axis"} · {unknownTime}

 {[...timeline.nodes.values()].filter(item => item.first === null).map(item => )}
: null} -
+ {timeline.decisions.length ?

{copy.trail}

- {!timeline.decisions.length ?

{timeline.structuredCount ? copy.none : copy.unstructured}

 - : } -
+ +
:
+ {zh ? "暂无结构化决策记录" : "No structured decision records"} +

{timeline.structuredCount ? copy.none : copy.unstructured}

+
}
; } diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 542d31ee87..8b6789b815 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -159,6 +159,13 @@ Requests without a language retain their original identity and stored artifacts; the frontend does not present an unspecified or other-language result as a new analysis in the selected language. Output-language metadata records the request, not independent verification of a model's translation or judgments. +When the selected language has no result, the reader also checks the same +registered pack's original, language-unspecified identity. A verified saved +original remains readable, with its original judgments, citations and usage, +and is labeled **Saved original**. It does not fill the selected language's +cache or substitute another language's artifact. New analysis, retry and cancel +still use the selected language's key and retain that attempt's failure/usage; +an explicit read-only response disables execution even on the local origin. ### Native results as explicit analysis context diff --git a/examples/personal-workspace-browser-smoke.mjs b/examples/personal-workspace-browser-smoke.mjs index 4978957dbb..20f21e90be 100644 --- a/examples/personal-workspace-browser-smoke.mjs +++ b/examples/personal-workspace-browser-smoke.mjs @@ -75,6 +75,7 @@ import { managedGoalResultsScenario } from "./personal-workspace-browser/managed import { researchResultsScenario } from "./personal-workspace-browser/research-results.mjs"; import { researchModeScenario } from "./personal-workspace-browser/research-mode.mjs"; import { researchHistoryScenario } from "./personal-workspace-browser/research-history.mjs"; +import { researchRealBenchmarkScenario } from "./personal-workspace-browser/research-real-benchmark.mjs"; import { researchFindingReaderScenario } from "./personal-workspace-browser/research-finding-reader.mjs"; import { researchRecordSearchScenario } from "./personal-workspace-browser/research-record-search.mjs"; import { researchRouteReadingScenario } from "./personal-workspace-browser/research-route-reading.mjs"; @@ -132,6 +133,7 @@ scenarioCatalog.push(configurationBackupScenario); scenarioCatalog.push(exploreCompositionOwnerScenario); scenarioCatalog.push(attentionUserActionScenario); scenarioCatalog.push(researchResultsScenario); +scenarioCatalog.push(researchRealBenchmarkScenario); scenarioCatalog.push(researchHistoryScenario); scenarioCatalog.push(researchMeasurementsScenario); scenarioCatalog.push(researchToolContextScenario, researchTurnCostsScenario); diff --git a/examples/personal-workspace-browser/research-history-backend.py b/examples/personal-workspace-browser/research-history-backend.py index 186c37c2d0..3225e8f358 100644 --- a/examples/personal-workspace-browser/research-history-backend.py +++ b/examples/personal-workspace-browser/research-history-backend.py @@ -1,6 +1,7 @@ """Disposable read-only HTTP owner for a captured Explore event log. -The default is three real EdgeBench research checkpoints. The history route +The default is the complete saved Explore log from a recent Grid benchmark run. +Older real checkpoints and synthetic boundary cases use explicit fixtures. The history route and pagination are production code; execution state is never restored and model dispatch is never called. Explicit fixture paths support regressions. """ @@ -53,9 +54,7 @@ def main(): help="Serve the production directory for scoped Goal navigation tests.") args = parser.parse_args() fixtures = Path(__file__).resolve().parents[2] / "apps/presentation/dashboard/src/data/fixtures" - sources = args.fixture or [fixtures / name for name in ( - "edgebench-research-history.json", "edgebench-graph-checkpoint-history.json", - "edgebench-ad-checkpoint-history.json")] + sources = args.fixture or [fixtures / "edgebench-grid-context-on-20261103-history.json"] captures = [] original = {} for source in sources: @@ -92,7 +91,7 @@ def main(): registry = root / "registry.json" registry.write_text(json.dumps({"schema_version": 1, "common_runtime_root": str(runtime), "goals": goals})) original[registry] = registry.read_bytes() - # Optional synthetic analysis uses the real pack verifier, process/job + # Optional boundary-test analysis uses the real pack verifier, process/job # owner and HTTP reader. It never enables model dispatch from the page. fixture_config = {**DEFAULT_CONFIG, "research_synthesis": False} fixture_config.pop("research_synthesis_revision", None) @@ -109,6 +108,9 @@ def main(): include_tool_context=analysis.get("include_tool_context", False), native_outcomes_json=analysis.get("native_outcomes_json")) for language, output in analysis["outputs"].items(): + # Exercise saved artifacts from before language-scoped + # generation without rewriting their registered identity. + language = None if language == "original" else language provider = write_python_host_fixture(root / f"analysis-provider-{index}-{language}", ("#!python\n" if sys.platform == "win32" else f"#!{sys.executable}\n") + "import sys\nfrom pathlib import Path\n" @@ -184,7 +186,7 @@ def terminate(*_): print(json.dumps({"port": server.server_port, "goal_id": goal_id, "goal_ids": [item["goal_id"] for item in captures], "event_count": sum(len(item["events"]) for item in captures), - "url": f"http://127.0.0.1:{server.server_port}/chat/?goalId={goal_id}&view=overview"}), flush=True) + "url": f"http://127.0.0.1:{server.server_port}/chat/?goalId={goal_id}&view=research"}), flush=True) try: server.serve_forever() finally: diff --git a/examples/personal-workspace-browser/research-history.md b/examples/personal-workspace-browser/research-history.md index b6101c9eee..9c2cf52246 100644 --- a/examples/personal-workspace-browser/research-history.md +++ b/examples/personal-workspace-browser/research-history.md @@ -1,6 +1,54 @@ -# Three real research checkpoints - -The default research-history preview uses three actual EdgeBench checkpoints +# Real benchmark reference fixtures + +The default read-only preview uses the complete saved Explore log from the +3 November Grid Turing Robot context-on run. The `research-real-benchmark` +browser scenario reads both context arms independently through the real HTTP +history owner. It verifies complete original text and append order, download +readback, source preservation, no writes, and desktop/390px reading. These are +the core populated-page references; synthetic fixtures cover boundaries such +as races, artificial paging pressure and missing fields. + +| Source | Records | Node / finding records | Recorded interval (UTC) | Original JSONL SHA-256 | +| --- | ---: | ---: | --- | --- | +| [Grid context on](../../apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-on-20261103-history.json) | 33 | 8 / 25 | 2026-11-03 06:02:35–15:06:44 | `292f493ee5d88e0f99546776ee2a46958e19517ceae192c10288276f270f404d` | +| [Grid context off](../../apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-off-20261103-history.json) | 11 | 2 / 9 | 2026-11-03 06:04:18–14:55:52 | `d91047d06aa7ef43249bc79e70d8ba53d37a88240c124dcd17a75034ea524375` | + +All saved Explore events are included, without sampling, rewriting or redaction. +No credentials or private absolute paths were found in the selected display +state. Native authority/credential stores, session transcripts, evaluator +internals and task workspaces are excluded. Each fixture carries its original +run identity and limitations. Both runs retain `runner_invalid`, diagnostic +scope and `score_countable=false`; displaying their observations does not +qualify a causal comparison. Recording intervals are not active solver time. + +The records expose real reading pressure: long repeated questions, multiple +measurement scopes, null execution observations, rejected local variants and an +unresolved timeout. All findings happen to carry the source's `confirmed` +label; one reports **no measured gain** and another preserves an unknown native +outcome. The frontend must preserve that distinction rather than infer benefit +from a status label. Neither log records explicit relationship edges or parents. +Missing structured decisions now take an on-demand disclosure rather than an +empty first-screen section. Opening a record reuses the existing observation, +interpretation and measurement reader, with complete original wording on demand; +the same layout applies without a generated display analysis. Exported records +remain original data. + +Run `uv run --extra test python examples/personal-workspace-browser/research-history-backend.py` +after building the Chat bundle to open the default source. Pass `--fixture` with +the context-off file to inspect it separately. Both sources genuinely use +`benchmark-goal` in separate runtimes: do not rename or merge them into invented +Goals. This reader materializes display events in disposable scaffolding; it +does not restore execution state or observe an installed/live worker. + +Run `LOOPX_PERSONAL_WORKSPACE_PACKAGED=1 LOOPX_PERSONAL_WORKSPACE_SCENARIO=research-real-benchmark node examples/personal-workspace-browser-smoke.mjs` +for the core reading journey. The Python history suite independently freezes the +original JSONL digests and sizes. The two source byte totals are 80,992 and +20,575; their exact original serialization can be reconstructed as described +below. No model or solver is called by this scenario. + +## Earlier research checkpoints + +The broader `research-history` regression explicitly loads three actual EdgeBench checkpoints sealed on 15 October 2026 and **restored by subsequent experiments**. It replaces the older Squirrel example. The Explore records retain their original text, timestamps, IDs, references, append order and scoped findings. @@ -88,7 +136,7 @@ can only display relationships actually supplied by the native source. ## Ordinary reading journey -Open the preview to read the complete Portfolio recording interval. Switch +Open the older three-fixture scenario to read the complete Portfolio recording interval. Switch between the three Goals in the existing sidebar. One date or point selection opens all corresponding original records, versions, scoped negative evidence and references. Refresh and browser Back/Forward preserve context. Reading diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index 9e0c47fea1..0e393d7c3f 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -745,7 +745,9 @@ export const researchHistoryScenario = { await synthesisOwner.close(); } const capture = JSON.parse(await readFile(resolve(repoRoot, "apps/presentation/dashboard/src/data/fixtures/edgebench-research-history.json"))); - let owner = await startHistoryOwner(); + let owner = await startHistoryOwner([ + "edgebench-research-history.json", "edgebench-graph-checkpoint-history.json", "edgebench-ad-checkpoint-history.json", + ].map(name => resolve(repoRoot, "apps/presentation/dashboard/src/data/fixtures", name))); let collisionOwner, emptyOwner, temporary; let context; const pages = []; @@ -766,7 +768,12 @@ export const researchHistoryScenario = { if (!analysisFixture) return route.fulfill({json: {ok: true, catalog: {availability: "unavailable", reason_code: "no_available_pack", default_pack_id: null, packs: []}, pack: null}}); const query = new URL(route.request().url()).searchParams; const language = query.get('output_language'); - assert(['zh', 'en'].includes(language), 'Every analysis and original-citation read binds the interface language'); + if (language === null && new URL(route.request().url()).pathname === '/api/chat/research-analysis') { + const original = analysisReadback(["b", "c", "d"].find(id => query.get("pack_id") === id.repeat(64)) ?? "a"); + original.pack.result = null; original.pack.job = null; + return route.fulfill({json: original}); + } + assert(['zh', 'en'].includes(language), 'Language-scoped analysis and citation reads retain their requested language'); if (new URL(route.request().url()).pathname.endsWith('/citation')) { if (holdCitation) {citationStarted(); await new Promise(resolve => {releaseCitation = resolve;});} if (failedCitation) return route.fulfill({status: 503, json: {error: 'Synthetic citation transport unavailable'}}); diff --git a/examples/personal-workspace-browser/research-real-benchmark.mjs b/examples/personal-workspace-browser/research-real-benchmark.mjs new file mode 100644 index 0000000000..df3a5e3826 --- /dev/null +++ b/examples/personal-workspace-browser/research-real-benchmark.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import {readFile} from "node:fs/promises"; +import {resolve} from "node:path"; +import {outputDir, repoRoot} from "./fixture.mjs"; +import {startHistoryOwner} from "./research-history.mjs"; + +// Core whole-page reference: unmodified display state from two real terminal +// runs. Synthetic scenarios remain separate boundary/race/overflow tests. +export const researchRealBenchmarkScenario = { + id: "research-real-benchmark", + async run({browser, collectCoverage}) { + const coverageEntries = []; + const observations = []; + for (const arm of ["on", "off"]) { + const path = resolve(repoRoot, `apps/presentation/dashboard/src/data/fixtures/edgebench-grid-context-${arm}-20261103-history.json`); + const capture = JSON.parse(await readFile(path, "utf8")); + const owner = await startHistoryOwner(path); + let page; + try { + page = await browser.newPage({viewport: {width: 1440, height: 900}}); + if (collectCoverage) await page.coverage.startJSCoverage({resetOnNavigation: false}); + await page.addInitScript(() => localStorage.setItem("loopx-pw-locale", "en")); + await page.goto(`${owner.base}/chat/?goalId=benchmark-goal&view=research`); + const history = page.locator(".research-history"); + await page.waitForFunction(() => document.querySelector(".research-history")?.getAttribute("aria-busy") === "false"); + assert.match(await history.locator(".research-history-coverage").innerText(), new RegExp(`${capture.events.length} / ${capture.events.length}`)); + assert.equal(await history.locator(".research-history-edges > g > path").count(), 0, "No relation is inferred from matching questions or adjacent times"); + const absentDecisions = history.locator(".research-trail-empty"); + assert.equal(await absentDecisions.evaluate(element => element.open), false, "The real source's missing structured decisions use an on-demand disclosure"); + await absentDecisions.locator(":scope > summary").click(); + assert.match(await absentDecisions.innerText(), /No explicit decision recorded/); + await absentDecisions.locator(":scope > summary").click(); + await page.screenshot({path: resolve(outputDir, `research-real-grid-${arm}-desktop.png`), animations: "disabled"}); + + await history.getByRole("button", {name: "Search research records", exact: true}).click(); + const records = history.locator(".research-record-search"); + const search = records.getByRole("searchbox", {name: "Search research records"}); + assert.equal(await records.locator(".research-record-entry").count(), capture.events.length); + const received = page.waitForEvent("download"); + await records.getByRole("button", {name: "Download matches (JSON)", exact: true}).click(); + const download = await received; + const exported = JSON.parse(await readFile(await download.path(), "utf8")); + assert.equal(exported.goal_id, "benchmark-goal"); + assert.deepEqual(exported.coverage, {loaded_records: capture.events.length, exported_records: capture.events.length, history_complete: true, refresh_failed: false}); + assert.deepEqual(exported.records.map(event => event.event_id), capture.events.map(event => event.event_id), "The full append order survives export"); + for (const [index, original] of capture.events.entries()) { + const actual = exported.records[index]; + for (const field of ["title", "summary", "status", "recorded_at", "result_id", "goal_id", "run_id"]) { + assert.equal(actual[field], original[field] ?? "", `${arm}/${original.event_id}/${field} keeps the original content`); + } + assert.deepEqual(actual.measurements ?? [], original.measurements ?? []); + } + + // A source-authored 'confirmed' record can report no gain. Never infer + // positive benefit from that label or rewrite the original conclusion. + const chosen = arm === "on" + ? capture.events.find(event => event.summary.includes("no measured gain from either tested family")) + : capture.events.at(-1); + assert(chosen); + await search.fill(chosen.event_id); + const entry = records.locator(".research-record-entry"); + assert.equal(await entry.count(), 1); + await entry.locator(":scope > summary").click(); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 900}); + await entry.scrollIntoViewIfNeeded(); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); + assert.equal(await entry.locator(":scope > summary > strong").innerText(), chosen.title); + await page.screenshot({path: resolve(outputDir, `research-real-grid-${arm}-record-${width}.png`), animations: "disabled"}); + } + if (arm === "on") assert.match(await entry.locator(".research-finding-sections").first().innerText(), /no measured gain/); + const original = entry.locator(":scope > details").filter({has: page.locator("summary", {hasText: "Full original text"})}); + await original.locator(":scope > summary").click(); + assert.equal(await original.locator(":scope > p").first().innerText(), chosen.summary, "Long negative/unknown evidence is readable without abbreviation"); + await original.locator(":scope > summary").click(); + await search.fill(""); + await history.getByRole("button", {name: "Refresh history", exact: true}).click(); + await page.waitForFunction(() => document.querySelector(".research-history")?.getAttribute("aria-busy") === "false"); + const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json(); + assert.equal(readback.source_unchanged, true); + assert.equal(readback.mutation_attempts, 0, "Reading a real benchmark capture never executes a solver/model or writes source state"); + observations.push({arm, source_run_id: capture.provenance.source_run_id, records: capture.events.length, source_sha256: capture.source_sha256}); + if (collectCoverage) coverageEntries.push(...await page.coverage.stopJSCoverage()); + } finally { + if (page) await page.close(); + await owner.close(); + } + } + assert.notEqual(observations[0].source_run_id, observations[1].source_run_id); + return {coverageEntries, observations}; + }, +}; diff --git a/tests/presentation/test_explore_history.py b/tests/presentation/test_explore_history.py index 1f0fd4ef1f..23a36fccce 100644 --- a/tests/presentation/test_explore_history.py +++ b/tests/presentation/test_explore_history.py @@ -20,6 +20,8 @@ ("edgebench-research-history.json", "b876dd7660e0aab19db717f60e86865494a74b8fa27b42a5a67ee295ef524397", 104922), ("edgebench-graph-checkpoint-history.json", "b105de360ed7d015ae5d7bf96b545879e12507b1fc97eaeb5fb6fa2c0bbc00ab", 200333), ("edgebench-ad-checkpoint-history.json", "de04b679e9f382b650e891e5c7dd7146bc690500435c511f107c9d24da4f4efa", 145296), + ("edgebench-grid-context-on-20261103-history.json", "292f493ee5d88e0f99546776ee2a46958e19517ceae192c10288276f270f404d", 80992), + ("edgebench-grid-context-off-20261103-history.json", "d91047d06aa7ef43249bc79e70d8ba53d37a88240c124dcd17a75034ea524375", 20575), ]) def test_checkpoint_fixture_reconstructs_original_captured_log_bytes(name, digest, size): fixture = Path(__file__).resolve().parents[2] / "apps/presentation/dashboard/src/data/fixtures" / name From 1353d017ee5d589ba0c2555f03962b6453cc644a Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:45:18 +0800 Subject: [PATCH 2/2] fix(workspace): verify saved analysis against real Graph state Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../edgebench-graph-saved-analysis.json | 3043 +++++++++++++++++ .../goal-research-analysis.tsx | 2 + .../research-history-backend.py | 23 +- .../research-history.md | 39 + .../research-real-benchmark.mjs | 70 + .../test_research_history_fixture.py | 63 +- 6 files changed, 3224 insertions(+), 16 deletions(-) create mode 100644 apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json new file mode 100644 index 0000000000..0212c2acae --- /dev/null +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json @@ -0,0 +1,3043 @@ +{ + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "display_name": "Graph · saved native-feedback analysis", + "events": [], + "provenance": { + "kind": "captured_read_only_analysis_api", + "captured_at": "2026-11-03T18:24:19.614475+00:00", + "source_run_id": "graph-official-native3600-integration-20261011-04", + "limitations": [ + "Historical 11–12 October run; this supplements the recent Grid history fixtures, not a newer whole-history case.", + "Six purposively selected native-feedback episodes; population and causal effect remain unknown.", + "Complete saved analysis responses, including original judgments, costs and language-specific cache misses, are unchanged.", + "No Explore events were recorded in this selected analysis Goal. Empty history is not a reconstructed solver history.", + "Only the included native-report-4 citation range is portable; uncaptured citations are explicitly unavailable.", + "No authority store, execution state, raw session transcript, model process or solver is restored by this display fixture." + ] + }, + "analysis_readbacks": { + "original": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "response_sha256": "0e4bb4b931784abe1faf8cba49922aeb36f6c96f9e1730fdc77d82e7ebd3da2f", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "packs": [ + { + "pack_id": "c570ee0653992fd555a2457371a17a57c02935f783d26619b0dbd530ba97730b", + "title": "Graph Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/graph/native-outcomes.json", + "artifact_sha256": "58530963ff2624eef198dcf2d90799a8d690dbdfdb1e5f66ebff0a1172350ee7", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": null, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "ae3ed1f8886e215bbb014761b79271bfb17ead9f4720c1078b18c1c24a0a2aff", + "title": "Graph Official · 末段最终保留 · 参照 agent-16", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-final-selection-native.json", + "artifact_sha256": "65dce13953ee3ca618e251516f66738f974bff385ed38c1b48383c29e2e9e6cb", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0, + "final_candidate_delta": -0.0010154616249691273 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-122", + "original-message-161", + "original-message-199", + "original-message-216" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-1743", + "original-message-1787", + "original-message-1833", + "original-message-1882" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3831", + "original-message-3856", + "original-message-3895", + "original-message-3947" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-11", + "question": "原生提交 agent-11 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-11", + "original-feedback-11", + "original-message-6278", + "original-message-6297", + "original-message-6306", + "original-message-6317" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-15", + "question": "原生提交 agent-15 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-15", + "original-feedback-15", + "original-message-9103", + "original-message-9139", + "original-message-9183", + "original-message-9199" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-18", + "question": "原生提交 agent-18 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-18", + "original-feedback-18", + "original-message-11709", + "original-message-11724", + "original-message-11743", + "original-message-11752" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": { + "schema_version": "trajectory_assessment_job_v1", + "ok": true, + "status": "succeeded", + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "method_version": "research-episodes-v2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "attempt": 1, + "started_at_unix": 1792345600.145857, + "analysis_wall_seconds": 447.04057866701623, + "usage": { + "input_tokens": 53768, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 19278, + "reasoning_output_tokens": 13463 + }, + "served_service_tier": null, + "reused": false + }, + "result": { + "schema_version": "trajectory_assessment_v1", + "method_version": "research-episodes-v2", + "analysis_key": "34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "evidence_manifest": [ + { + "id": "original-objective", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/agent_prompt.md", + "artifact_sha256": "e10af906e94a1284a8dbdec3ccabaf13b0c5e6628fc680de903d5f9a54112b12", + "first_line": 1 + }, + { + "id": "original-runtime", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/runtime-receipt.json", + "artifact_sha256": "8ea6e5c6e87c75a41164857e6c74214ae85ea9961ecb0f320f40e7d33ffb61e2", + "first_line": 1 + }, + { + "id": "released-native-inventory", + "artifact": "product-goal-sources/official/graph/native-observations.jsonl", + "artifact_sha256": "c2c371ecbe723f7bfa21ddc65961aac972530f50b6af746aee52dbba9e7b77ef", + "first_line": 1 + }, + { + "id": "original-terminal-result", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/final_result.json", + "artifact_sha256": "6bebe82f59ab01d0b0b963f895cd4c613d6a7f86d4f812d513552bef444cfcae", + "first_line": 1 + }, + { + "id": "native-report-1", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-1/report.json", + "artifact_sha256": "d1e6ec64ba0dd6ea1d150e2aa203178ac672c3715328c5e544f420eb9c74d811", + "first_line": 1 + }, + { + "id": "original-message-122", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 122 + }, + { + "id": "original-message-161", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 161 + }, + { + "id": "original-message-199", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 199 + }, + { + "id": "original-message-216", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 216 + }, + { + "id": "original-feedback-1", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 195 + }, + { + "id": "native-report-4", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-4/report.json", + "artifact_sha256": "a41532dacf9d8b9930e7654e52ee13870338125a774434e65e753f3fe3a0c3be", + "first_line": 1 + }, + { + "id": "original-message-1743", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 1743 + }, + { + "id": "original-message-1787", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 1787 + }, + { + "id": "original-message-1833", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 1833 + }, + { + "id": "original-message-1882", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 1882 + }, + { + "id": "original-feedback-4", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 1801 + }, + { + "id": "native-report-8", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-8/report.json", + "artifact_sha256": "d4d6883e27a92f029281a89fcf26ed3f88e0813a8ae4c4aac99a29f3d805873c", + "first_line": 1 + }, + { + "id": "original-message-3831", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 3831 + }, + { + "id": "original-message-3856", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 3856 + }, + { + "id": "original-message-3895", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 3895 + }, + { + "id": "original-message-3947", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 3947 + }, + { + "id": "original-feedback-8", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 3887 + }, + { + "id": "native-report-11", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-11/report.json", + "artifact_sha256": "404c317f17ef05562daaf29ae7a7656f3056303b8e91fca7a0b2af48ae22afd2", + "first_line": 1 + }, + { + "id": "original-message-6278", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 6278 + }, + { + "id": "original-message-6297", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 6297 + }, + { + "id": "original-message-6306", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 6306 + }, + { + "id": "original-message-6317", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 6317 + }, + { + "id": "original-feedback-11", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 6300 + }, + { + "id": "native-report-15", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-15/report.json", + "artifact_sha256": "3296c877e8bd7424680af385719143682cc014c339f3f8b6da7b81175c824b91", + "first_line": 1 + }, + { + "id": "original-message-9103", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 9103 + }, + { + "id": "original-message-9139", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 9139 + }, + { + "id": "original-message-9183", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 9183 + }, + { + "id": "original-message-9199", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 9199 + }, + { + "id": "original-feedback-15", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 9176 + }, + { + "id": "native-report-18", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-18/report.json", + "artifact_sha256": "3de7073f0959ad2ce0e089782cdb23760fad92db6ab6af9024363b352e8c629c", + "first_line": 1 + }, + { + "id": "original-message-11709", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 11709 + }, + { + "id": "original-message-11724", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 11724 + }, + { + "id": "original-message-11743", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 11743 + }, + { + "id": "original-message-11752", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 11752 + }, + { + "id": "original-feedback-18", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl", + "artifact_sha256": "0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "first_line": 11739 + } + ], + "calibration": "uncalibrated_model", + "assessments": [ + { + "episode_id": "native-feedback-1", + "metric": "goal_alignment", + "verdict": "yes", + "reason": "The reported intervention reduced unnecessary prediction work and reused kernel slices. Its full-size comparison preserved outputs while reducing measured runtime, connecting the work to execution cost on the classification path. It does not establish the cause of the native worker failure.", + "citations": [ + { + "source_id": "original-message-161", + "first_line": 161, + "last_line": 161 + }, + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + } + ] + }, + { + "episode_id": "native-feedback-1", + "metric": "benefit_test", + "verdict": "unknown", + "reason": "The native attempt explicitly produced no valid result payload. Successful local evidence covered interface behavior, equivalence and synthetic timing, but its representativeness for scored native utility is not established.", + "citations": [ + { + "source_id": "native-report-1", + "first_line": 14, + "last_line": 15 + }, + { + "source_id": "original-message-122", + "first_line": 122, + "last_line": 122 + }, + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + }, + { + "source_id": "original-message-216", + "first_line": 216, + "last_line": 216 + } + ] + }, + { + "episode_id": "native-feedback-1", + "metric": "discrimination", + "verdict": "no", + "reason": "The native experiment was explicitly invalid. Its worker-termination report cannot distinguish candidate resource problems from other causes, and the local equivalence comparison does not diagnose that failure.", + "citations": [ + { + "source_id": "native-report-1", + "first_line": 14, + "last_line": 20 + }, + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + }, + { + "source_id": "original-message-216", + "first_line": 216, + "last_line": 216 + } + ] + }, + { + "episode_id": "native-feedback-1", + "metric": "information_gain", + "verdict": "yes", + "reason": "The reported completed comparison newly answered a local correctness question: the optimization preserved tested predictions and confidence values. That informed keeping the faster implementation. Native classification quality and the worker-failure cause remained unresolved.", + "citations": [ + { + "source_id": "original-message-161", + "first_line": 161, + "last_line": 161 + }, + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + }, + { + "source_id": "original-message-216", + "first_line": 216, + "last_line": 216 + } + ] + }, + { + "episode_id": "native-feedback-1", + "metric": "evidence_adoption", + "verdict": "unknown", + "reason": "The optimization report still described native feedback as pending. After acknowledging failure, the assistant supplied a future memory/runtime and repeated-call plan, without a completed feedback-linked follow-up.", + "citations": [ + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + }, + { + "source_id": "original-message-216", + "first_line": 216, + "last_line": 216 + } + ] + }, + { + "episode_id": "native-feedback-1", + "metric": "redundancy", + "verdict": "no", + "reason": "This was the first native submission. The subsequent full-size comparison tested an altered implementation for output preservation and runtime, providing a justified equivalence check.", + "citations": [ + { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + }, + { + "source_id": "original-message-161", + "first_line": 161, + "last_line": 161 + }, + { + "source_id": "original-message-199", + "first_line": 199, + "last_line": 199 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "goal_alignment", + "verdict": "yes", + "reason": "The candidate targeted solve cost, matching behavior and confidence calibration, while excluding an ensemble that reportedly regressed locally. Native evaluation measured the corresponding classification, calibration and runtime quantities, establishing a concrete objective connection.", + "citations": [ + { + "source_id": "original-message-1743", + "first_line": 1743, + "last_line": 1743 + }, + { + "source_id": "native-report-4", + "first_line": 46, + "last_line": 60 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "benefit_test", + "verdict": "yes", + "reason": "A valid full native evaluation covered all ten splits and measured final score, classification quality, calibration and runtime. The bundle's slightly lower score still constitutes a completed utility test.", + "citations": [ + { + "source_id": "native-report-4", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "native-report-4", + "first_line": 24, + "last_line": 35 + }, + { + "source_id": "released-native-inventory", + "first_line": 3, + "last_line": 4 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "discrimination", + "verdict": "no", + "reason": "The native experiment bundled solver, matching, confidence and threading changes. It therefore cannot separate the speed/calibration effects from the new matching rules, the distinction explicitly proposed afterward. The reported local comparisons do not provide that native ablation.", + "citations": [ + { + "source_id": "original-message-1743", + "first_line": 1743, + "last_line": 1743 + }, + { + "source_id": "original-message-1833", + "first_line": 1833, + "last_line": 1833 + }, + { + "source_id": "original-message-1882", + "first_line": 1882, + "last_line": 1882 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "information_gain", + "verdict": "yes", + "reason": "The new valid score, 0.835929, was below round 3's 0.836494. This resolved the observed-score comparison relevant to adopting the bundle, while leaving individual contributions unresolved.", + "citations": [ + { + "source_id": "released-native-inventory", + "first_line": 3, + "last_line": 4 + }, + { + "source_id": "native-report-4", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "original-message-1833", + "first_line": 1833, + "last_line": 1833 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "evidence_adoption", + "verdict": "yes", + "reason": "After proposing separation of speed/calibration from matching changes, the follow-up reports nine completed comparisons of a conservative candidate, preserving incumbent predictions and calibration gains. This is reported experimental follow-through; raw local results are not supplied.", + "citations": [ + { + "source_id": "original-message-1833", + "first_line": 1833, + "last_line": 1833 + }, + { + "source_id": "original-message-1882", + "first_line": 1882, + "last_line": 1882 + } + ] + }, + { + "episode_id": "native-feedback-4", + "metric": "redundancy", + "verdict": "no", + "reason": "The changed bundle received native utility testing, followed by a conservative variant addressing a separate equivalence and recovery question. The repeated local comparisons had an explicit validation purpose.", + "citations": [ + { + "source_id": "original-message-1743", + "first_line": 1743, + "last_line": 1743 + }, + { + "source_id": "original-message-1833", + "first_line": 1833, + "last_line": 1833 + }, + { + "source_id": "original-message-1882", + "first_line": 1882, + "last_line": 1882 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "goal_alignment", + "verdict": "yes", + "reason": "The candidate combined arithmetic/CSR work with calibration and a guarded majority extension supported by reported local cases. Native testing directly checked whether these execution and prediction changes improved the scored classification objective.", + "citations": [ + { + "source_id": "original-message-3831", + "first_line": 3831, + "last_line": 3831 + }, + { + "source_id": "original-message-3856", + "first_line": 3856, + "last_line": 3856 + }, + { + "source_id": "native-report-8", + "first_line": 46, + "last_line": 60 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "benefit_test", + "verdict": "yes", + "reason": "The candidate completed a valid ten-split native evaluation measuring quality, calibration, runtime penalty and final score. Its disappointing net score is a valid negative utility result.", + "citations": [ + { + "source_id": "native-report-8", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "native-report-8", + "first_line": 24, + "last_line": 35 + }, + { + "source_id": "native-report-8", + "first_line": 46, + "last_line": 60 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "discrimination", + "verdict": "unknown", + "reason": "The histogram refactor reportedly had inconsistent timing, while the native candidate bundled several other changes. No explicit controlled contrast separating competing explanations for the native result is supplied.", + "citations": [ + { + "source_id": "original-message-3831", + "first_line": 3831, + "last_line": 3831 + }, + { + "source_id": "original-message-3856", + "first_line": 3856, + "last_line": 3856 + }, + { + "source_id": "original-message-3947", + "first_line": 3947, + "last_line": 3947 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "information_gain", + "verdict": "yes", + "reason": "Native evaluation newly measured the bundle at 0.828945 with 53.985-second average runtime, below the recorded 0.836494 incumbent score. This supplied decision-relevant negative evidence beyond the favorable synthetic checks, without isolating component effects.", + "citations": [ + { + "source_id": "original-message-3856", + "first_line": 3856, + "last_line": 3856 + }, + { + "source_id": "native-report-8", + "first_line": 46, + "last_line": 60 + }, + { + "source_id": "released-native-inventory", + "first_line": 3, + "last_line": 3 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "evidence_adoption", + "verdict": "unknown", + "reason": "The response announces restoration and a model-skipping experiment. Subsequent inventory entries establish further submissions but do not identify restored code or a completed feedback-linked experiment.", + "citations": [ + { + "source_id": "original-message-3947", + "first_line": 3947, + "last_line": 3947 + }, + { + "source_id": "released-native-inventory", + "first_line": 9, + "last_line": 10 + } + ] + }, + { + "episode_id": "native-feedback-8", + "metric": "redundancy", + "verdict": "no", + "reason": "The native trial tested the scored utility of the documented revision beyond arithmetic equivalence and five synthetic cases. Those earlier checks had different coverage, giving the native evaluation a justified purpose.", + "citations": [ + { + "source_id": "original-message-3831", + "first_line": 3831, + "last_line": 3831 + }, + { + "source_id": "original-message-3856", + "first_line": 3856, + "last_line": 3856 + }, + { + "source_id": "native-report-8", + "first_line": 24, + "last_line": 35 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "goal_alignment", + "verdict": "unknown", + "reason": "The excerpt establishes a frozen candidate and an incumbent-score comparison, but does not identify round 11's intervention or the bottleneck it addressed. Submission and scoring alone do not establish that connection.", + "citations": [ + { + "source_id": "original-message-6278", + "first_line": 6278, + "last_line": 6278 + }, + { + "source_id": "original-message-6297", + "first_line": 6297, + "last_line": 6297 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "benefit_test", + "verdict": "yes", + "reason": "The candidate completed valid native evaluation across ten fully covered splits, including classification metrics, runtime and final score. Missing implementation detail does not negate this direct utility test.", + "citations": [ + { + "source_id": "native-report-11", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "native-report-11", + "first_line": 24, + "last_line": 35 + }, + { + "source_id": "native-report-11", + "first_line": 46, + "last_line": 60 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "discrimination", + "verdict": "unknown", + "reason": "Native validity is documented, but the intervention, competing explanations and separating experimental contrast are not supplied. The score comparison alone cannot establish explanatory discrimination.", + "citations": [ + { + "source_id": "native-report-11", + "first_line": 14, + "last_line": 15 + }, + { + "source_id": "original-message-6278", + "first_line": 6278, + "last_line": 6278 + }, + { + "source_id": "original-message-6306", + "first_line": 6306, + "last_line": 6306 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "information_gain", + "verdict": "yes", + "reason": "A retention threshold of 0.84213 was stated before feedback. The valid 0.841600 result resolved that trial's observed-score decision against retention. It does not establish a repeatable runtime regression.", + "citations": [ + { + "source_id": "original-message-6297", + "first_line": 6297, + "last_line": 6297 + }, + { + "source_id": "native-report-11", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "original-message-6306", + "first_line": 6306, + "last_line": 6306 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "evidence_adoption", + "verdict": "yes", + "reason": "The preset retention rule was followed by a rollback statement and a later report that solution.py retains round 10 at 0.84213. This supports a reported interim retention outcome; file contents and terminal retention remain unverified.", + "citations": [ + { + "source_id": "original-message-6297", + "first_line": 6297, + "last_line": 6297 + }, + { + "source_id": "original-message-6306", + "first_line": 6306, + "last_line": 6306 + }, + { + "source_id": "original-message-6317", + "first_line": 6317, + "last_line": 6317 + } + ] + }, + { + "episode_id": "native-feedback-11", + "metric": "redundancy", + "verdict": "unknown", + "reason": "Distinct submission identities and a frozen-source statement do not reveal what changed from round 10. Without implementation differences or an explicit replication purpose, repetition cannot be determined.", + "citations": [ + { + "source_id": "released-native-inventory", + "first_line": 10, + "last_line": 11 + }, + { + "source_id": "original-message-6278", + "first_line": 6278, + "last_line": 6278 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "goal_alignment", + "verdict": "unknown", + "reason": "Root-key reuse, combined gradient/Hessian accumulation and feature-row reuse are described as upcoming experiments. The supplied excerpt does not establish that these were executed interventions in round 15 or document that candidate's bottleneck connection.", + "citations": [ + { + "source_id": "original-message-9103", + "first_line": 9103, + "last_line": 9103 + }, + { + "source_id": "original-message-9139", + "first_line": 9139, + "last_line": 9139 + }, + { + "source_id": "original-message-9183", + "first_line": 9183, + "last_line": 9183 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "benefit_test", + "verdict": "yes", + "reason": "Round 15 completed valid native evaluation on all ten splits and measured final score, quality and execution penalties. This directly tested native utility despite the incomplete intervention description.", + "citations": [ + { + "source_id": "native-report-15", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "native-report-15", + "first_line": 24, + "last_line": 35 + }, + { + "source_id": "native-report-15", + "first_line": 46, + "last_line": 60 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "discrimination", + "verdict": "unknown", + "reason": "The focused equivalence and timing experiments are explicitly prepared or future work. Their completed results are absent, and the native submission's implementation contrast is unspecified. The reported slowdown does not establish its cause.", + "citations": [ + { + "source_id": "original-message-9103", + "first_line": 9103, + "last_line": 9103 + }, + { + "source_id": "original-message-9183", + "first_line": 9183, + "last_line": 9183 + }, + { + "source_id": "original-message-9199", + "first_line": 9199, + "last_line": 9199 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "information_gain", + "verdict": "yes", + "reason": "The valid 0.841197 result supplied a new negative observed-score comparison against round 14's 0.845760. That informed the stated retention decision, while the explanation for longer runtime remained unresolved.", + "citations": [ + { + "source_id": "released-native-inventory", + "first_line": 14, + "last_line": 15 + }, + { + "source_id": "native-report-15", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "original-message-9199", + "first_line": 9199, + "last_line": 9199 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "evidence_adoption", + "verdict": "unknown", + "reason": "Rollback and local profiling are announced, but no restored-source outcome or completed profiling result follows in the supplied excerpt. Round 16's later score does not identify which feedback-linked action occurred.", + "citations": [ + { + "source_id": "original-message-9199", + "first_line": 9199, + "last_line": 9199 + }, + { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + ] + }, + { + "episode_id": "native-feedback-15", + "metric": "redundancy", + "verdict": "unknown", + "reason": "The proposed next experiments have distinct stated purposes, but they cannot be mapped to the already submitted round-15 source. The inventory alone cannot establish whether that submission repeated an earlier intervention or tested a new one.", + "citations": [ + { + "source_id": "original-message-9103", + "first_line": 9103, + "last_line": 9103 + }, + { + "source_id": "original-message-9139", + "first_line": 9139, + "last_line": 9139 + }, + { + "source_id": "released-native-inventory", + "first_line": 14, + "last_line": 15 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "goal_alignment", + "verdict": "yes", + "reason": "The frozen sorted-CSR/cache revision was tested on the native path. The reported fixed-quality comparison linked its execution time to a score penalty, giving the intervention a concrete connection to the execution-cost bottleneck.", + "citations": [ + { + "source_id": "original-message-11724", + "first_line": 11724, + "last_line": 11724 + }, + { + "source_id": "native-report-18", + "first_line": 46, + "last_line": 60 + }, + { + "source_id": "original-message-11752", + "first_line": 11752, + "last_line": 11752 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "benefit_test", + "verdict": "yes", + "reason": "The sorted-CSR/cache candidate completed valid native evaluation over ten fully covered splits. Its measured score, quality and runtime constitute a direct utility test, including the negative comparison with the incumbent.", + "citations": [ + { + "source_id": "native-report-18", + "first_line": 8, + "last_line": 15 + }, + { + "source_id": "native-report-18", + "first_line": 24, + "last_line": 35 + }, + { + "source_id": "original-message-11752", + "first_line": 11752, + "last_line": 11752 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "discrimination", + "verdict": "unknown", + "reason": "The passive-wait timings came from separate local runs, and the Torch-first/OpenMP explanation had only prepared checks. The native outcome for sorted CSR/cache does not supply a controlled contrast resolving those explanations or timing noise.", + "citations": [ + { + "source_id": "original-message-11709", + "first_line": 11709, + "last_line": 11709 + }, + { + "source_id": "original-message-11743", + "first_line": 11743, + "last_line": 11743 + }, + { + "source_id": "native-report-18", + "first_line": 14, + "last_line": 15 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "information_gain", + "verdict": "yes", + "reason": "The completed trial measured native score 0.848529 and average runtime 27.348 seconds, below the released incumbent score of 0.849544. This informed the announced rejection of this trial; it does not establish a reproducible cache-induced slowdown.", + "citations": [ + { + "source_id": "native-report-18", + "first_line": 46, + "last_line": 60 + }, + { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + }, + { + "source_id": "original-message-11752", + "first_line": 11752, + "last_line": 11752 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "evidence_adoption", + "verdict": "unknown", + "reason": "After acknowledged feedback, the excerpt contains an announced restoration and a future passive-wait test. Neither the restored source nor a completed post-feedback experiment is supplied, so realized adoption cannot be established.", + "citations": [ + { + "source_id": "original-message-11752", + "first_line": 11752, + "last_line": 11752 + } + ] + }, + { + "episode_id": "native-feedback-18", + "metric": "redundancy", + "verdict": "unknown", + "reason": "The sorted-CSR/cache source is named and the passive-wait variant is kept separate, but implementation descriptions for rounds 16–17 are absent. The evidence cannot establish whether round 18 repeats an already tested revision or serves a justified recovery or replication purpose.", + "citations": [ + { + "source_id": "original-message-11709", + "first_line": 11709, + "last_line": 11709 + }, + { + "source_id": "original-message-11724", + "first_line": 11724, + "last_line": 11724 + }, + { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 18 + } + ] + } + ], + "metrics": [ + { + "metric": "goal_alignment", + "yes": 4, + "no": 0, + "unknown": 2, + "eligible": 6, + "assessed_fraction": 0.6666666666666666, + "missing_data_bounds": [ + 0.6666666666666666, + 1 + ] + }, + { + "metric": "benefit_test", + "yes": 5, + "no": 0, + "unknown": 1, + "eligible": 6, + "assessed_fraction": 0.8333333333333334, + "missing_data_bounds": [ + 0.8333333333333334, + 1 + ] + }, + { + "metric": "discrimination", + "yes": 0, + "no": 2, + "unknown": 4, + "eligible": 6, + "assessed_fraction": 0.3333333333333333, + "missing_data_bounds": [ + 0, + 0.6666666666666666 + ] + }, + { + "metric": "information_gain", + "yes": 6, + "no": 0, + "unknown": 0, + "eligible": 6, + "assessed_fraction": 1, + "missing_data_bounds": [ + 1, + 1 + ] + }, + { + "metric": "evidence_adoption", + "yes": 2, + "no": 0, + "unknown": 4, + "eligible": 6, + "assessed_fraction": 0.3333333333333333, + "missing_data_bounds": [ + 0.3333333333333333, + 1 + ] + }, + { + "metric": "redundancy", + "yes": 0, + "no": 3, + "unknown": 3, + "eligible": 6, + "assessed_fraction": 0.5, + "missing_data_bounds": [ + 0, + 0.5 + ] + } + ], + "limitations": [ + "This is an uncalibrated assessment of six purposively sampled episodes, not causal attribution or a population estimate.", + "Implementation snapshots, the underlying task specification and most raw local-test results are absent. Narrative reports support only the reported experiments and interim outcomes.", + "Information-gain findings from native scores concern the observed trial and retention decision. Single-run timing differences do not establish repeatable performance effects or component contributions.", + "Bounded follow-up excerpts limit adoption assessment. Announced rollbacks, temporal adjacency and later submission scores do not establish execution.", + "The first native failure is not a zero-quality measurement. Its synthetic timing evidence lacks supplied validation as a representative native-utility proxy.", + "The historical run was score_countable=false and timed out. The recorded best score does not verify the terminal retained candidate, contribution attribution or monetary cost." + ] + }, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "zh": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&output_language=zh", + "response_sha256": "cb8fdf1a61915f06168816c82367e9d037bdf49424028cd2793dfe61364def64", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "packs": [ + { + "pack_id": "c570ee0653992fd555a2457371a17a57c02935f783d26619b0dbd530ba97730b", + "title": "Graph Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "85c45438fee3e78c445a08b1ea15a892601b0968b274eccd1943a8e63163835d", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/graph/native-outcomes.json", + "artifact_sha256": "58530963ff2624eef198dcf2d90799a8d690dbdfdb1e5f66ebff0a1172350ee7", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": null, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "ae3ed1f8886e215bbb014761b79271bfb17ead9f4720c1078b18c1c24a0a2aff", + "title": "Graph Official · 末段最终保留 · 参照 agent-16", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "85c45438fee3e78c445a08b1ea15a892601b0968b274eccd1943a8e63163835d", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-final-selection-native.json", + "artifact_sha256": "65dce13953ee3ca618e251516f66738f974bff385ed38c1b48383c29e2e9e6cb", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0, + "final_candidate_delta": -0.0010154616249691273 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "85c45438fee3e78c445a08b1ea15a892601b0968b274eccd1943a8e63163835d", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "85c45438fee3e78c445a08b1ea15a892601b0968b274eccd1943a8e63163835d", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-122", + "original-message-161", + "original-message-199", + "original-message-216" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-1743", + "original-message-1787", + "original-message-1833", + "original-message-1882" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3831", + "original-message-3856", + "original-message-3895", + "original-message-3947" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-11", + "question": "原生提交 agent-11 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-11", + "original-feedback-11", + "original-message-6278", + "original-message-6297", + "original-message-6306", + "original-message-6317" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-15", + "question": "原生提交 agent-15 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-15", + "original-feedback-15", + "original-message-9103", + "original-message-9139", + "original-message-9183", + "original-message-9199" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-18", + "question": "原生提交 agent-18 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-18", + "original-feedback-18", + "original-message-11709", + "original-message-11724", + "original-message-11743", + "original-message-11752" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "en": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&output_language=en", + "response_sha256": "d76b579672d87f8b071d871ecfd5797b986f295b8dc93bc967471c0e1aab0b2b", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "packs": [ + { + "pack_id": "c570ee0653992fd555a2457371a17a57c02935f783d26619b0dbd530ba97730b", + "title": "Graph Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "3ad8fd4a77e33cddeb6968988d20759a671f834da533a8d05097ab9b1586ce17", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/graph/native-outcomes.json", + "artifact_sha256": "58530963ff2624eef198dcf2d90799a8d690dbdfdb1e5f66ebff0a1172350ee7", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": null, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "ae3ed1f8886e215bbb014761b79271bfb17ead9f4720c1078b18c1c24a0a2aff", + "title": "Graph Official · 末段最终保留 · 参照 agent-16", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "3ad8fd4a77e33cddeb6968988d20759a671f834da533a8d05097ab9b1586ce17", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-final-selection-native.json", + "artifact_sha256": "65dce13953ee3ca618e251516f66738f974bff385ed38c1b48383c29e2e9e6cb", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0, + "final_candidate_delta": -0.0010154616249691273 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + }, + { + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "3ad8fd4a77e33cddeb6968988d20759a671f834da533a8d05097ab9b1586ce17", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1", + "title": "Graph Official · 最终交付增量 · 参照首个有效 agent-2", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "3ad8fd4a77e33cddeb6968988d20759a671f834da533a8d05097ab9b1586ce17", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "heartbeat-094410/official-graph-first-valid-native.json", + "artifact_sha256": "d2edb04a3d5d6c062702262f3bc395f9562ccb995c8f410b4b76a1307f7dd42b", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-graph-analysis-20261011-04", + "case_id": "graph_node_classification", + "run_id": "graph-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1", + "window": "Original captured native session 2026-10-11T12:28:51.627Z to 2026-10-12T06:28:34.282Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:51.627Z", + "end": "2026-10-12T06:28:34.282Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 18, + "valid_count": 17, + "initial": null, + "best_valid": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "retained": { + "submission_index": 16, + "result_id": "6e610c29e0ae", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T03:59:48.720Z", + "observed_at": "2026-10-12T04:01:22.153Z", + "value": 0.849544264387785, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "final_valid": { + "submission_index": 18, + "result_id": "ba86adc7653e", + "candidate_ref": "agent-18", + "submitted_at": "2026-10-12T06:00:45.288Z", + "observed_at": "2026-10-12T06:02:40.679Z", + "value": 0.8485288027628158, + "citation": { + "source_id": "released-native-inventory", + "first_line": 18, + "last_line": 18 + } + }, + "retained_improvement": { + "baseline": { + "submission_index": 2, + "result_id": "e00dc3959c91", + "candidate_ref": "agent-2", + "submitted_at": "2026-10-11T13:37:19.792Z", + "observed_at": "2026-10-11T13:39:12.474Z", + "value": 0.8172464473565724, + "citation": { + "source_id": "released-native-inventory", + "first_line": 2, + "last_line": 2 + } + }, + "retention_citation": { + "source_id": "original-terminal-result", + "first_line": 7, + "last_line": 9 + }, + "value": 0.032297817031212595, + "final_candidate_delta": 0.03128235540624347 + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-122", + "original-message-161", + "original-message-199", + "original-message-216" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-1743", + "original-message-1787", + "original-message-1833", + "original-message-1882" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3831", + "original-message-3856", + "original-message-3895", + "original-message-3947" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-11", + "question": "原生提交 agent-11 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-11", + "original-feedback-11", + "original-message-6278", + "original-message-6297", + "original-message-6306", + "original-message-6317" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-15", + "question": "原生提交 agent-15 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-15", + "original-feedback-15", + "original-message-9103", + "original-message-9139", + "original-message-9183", + "original-message-9199" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-18", + "question": "原生提交 agent-18 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-18", + "original-feedback-18", + "original-message-11709", + "original-message-11724", + "original-message-11743", + "original-message-11752" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + } + }, + "analysis_citations": [ + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&analysis_key=34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2&source_id=native-report-4&first_line=8&last_line=15", + "response_sha256": "cde1487cf4528b25b737f7b50c253f992b73f5e7dc3a8cc3d7abd440f0040b62", + "response": { + "source_id": "native-report-4", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-4/report.json", + "first_line": 8, + "last_line": 15, + "text": " \"pass_rate\": 1.0,\n \"score\": 0.8359291793199752,\n \"score_0_100\": 83.59291793199752,\n \"timed_out\": false,\n \"runtime_seconds\": 63.152724742889404,\n \"test_details\": [],\n \"valid\": true,\n \"summary\": \"valid=true final_score=0.835929 overall_score=0.865158 macro_f1=0.850742 micro_f1=0.851681 balanced_accuracy=0.852224 one_minus_ece=0.989116 avg_time=18.698s checks=4/4\",", + "applicability": "current" + } + } + ] +} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index 5a48c74ce0..f016a45158 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -534,6 +534,8 @@ export function ResearchAnalysisPanel({analysis: a, onHistory}: {analysis: Contr const available = sourceAvailable && !a.readOnly && !a.readError; const benefit = result?.metrics.find(m => m.metric === "benefit_test"); const headline = !result ? null : benefit?.no ? (zh ? "仍有研究片段尚未检验原目标收益" : "Some episodes have not tested original-objective utility") + : benefit?.yes && benefit.unknown ? (zh ? `${benefit.yes} / ${benefit.eligible} 个片段记录了原目标收益检验；${benefit.unknown} 个证据不足` + : `${benefit.yes} / ${benefit.eligible} episodes recorded original-objective utility tests; ${benefit.unknown} ${benefit.unknown === 1 ? "remains" : "remain"} unknown`) : benefit?.unknown ? (zh ? "原目标收益的检验情况仍未知" : "Original-objective utility testing remains unknown") : benefit?.yes ? (zh ? "模型记录了收益检验；这不等于收益为正" : "The model identified a utility test; this does not imply positive utility") : (zh ? "此方法未覆盖原目标收益检验" : "This method does not cover original-objective utility testing"); diff --git a/examples/personal-workspace-browser/research-history-backend.py b/examples/personal-workspace-browser/research-history-backend.py index 3225e8f358..e1bcee84cf 100644 --- a/examples/personal-workspace-browser/research-history-backend.py +++ b/examples/personal-workspace-browser/research-history-backend.py @@ -65,6 +65,11 @@ def main(): assert capture["goal_id"] not in {item["goal_id"] for item in captures}, "Duplicate captured Goal" captures.append(capture) original[source] = source_bytes + # Frozen API responses retain real analysis identity and usage without + # registering execution state or launching a synthetic analysis job. + for record in [*capture.get("analysis_readbacks", {}).values(), *capture.get("analysis_citations", [])]: + encoded = json.dumps(record["response"], ensure_ascii=False, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == record["response_sha256"], "Changed analysis readback" goal_id = captures[0]["goal_id"] source_digests = {capture["goal_id"]: hashlib.sha256(original[source]).hexdigest() for source, capture in zip(sources, captures)} @@ -118,7 +123,7 @@ def main(): "print('{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}')\n") selected = inspect_pack(**scope, pack_id=registered["pack_id"], output_language=language) execute_pack(**scope, pack_id=registered["pack_id"], analysis_key=selected["analysis_key"], output_language=language, codex=str(provider)) - metrics = {"history_reads": 0, "mutation_attempts": 0} + metrics = {"history_reads": 0, "analysis_reads": 0, "citation_reads": 0, "mutation_attempts": 0} class Handler(ChatRequestHandler): def _send_json(self, payload, *, status=200): @@ -146,6 +151,21 @@ def do_GET(self): "fixture_sha256": source_digests[goal_id], "fixtures": source_digests}) if path == "/api/chat/explore-results": metrics["history_reads"] += 1 + query = parse_qs(parsed.query) + capture = next((item for item in captures if query.get("goal_id") == [item["goal_id"]]), None) + if capture and capture.get("analysis_readbacks") and path.startswith("/api/chat/research-analysis"): + if path == "/api/chat/research-analysis": + metrics["analysis_reads"] += 1 + language = query.get("output_language", ["original"])[0] + record = capture["analysis_readbacks"].get(language) + if record and query.get("pack_id", [record["response"]["pack"]["pack_id"]]) == [record["response"]["pack"]["pack_id"]]: + return self._send_json(record["response"]) + elif path == "/api/chat/research-analysis/citation": + metrics["citation_reads"] += 1 + for record in capture.get("analysis_citations", []): + if query == parse_qs(urlparse(record["request_path"]).query): + return self._send_json(record["response"]) + return self._send_error("This readback was not captured; the fixture cannot recover the original source.", status=404) if path in {"/healthz", "/status.json", "/api/chat/explore-results", "/api/chat/capabilities", "/api/chat/endpoints", "/api/chat/sessions", "/api/chat/goal-configuration", "/api/chat/research-analysis", "/api/chat/research-analysis/citation", "/api/chat/research-analysis/usage-turn", @@ -163,6 +183,7 @@ def do_POST(self): do_PUT = do_POST do_DELETE = do_POST + do_PATCH = do_POST server = ChatHTTPServer(("127.0.0.1", args.port), Handler) server.registry_path, server.runtime_root = registry, runtime diff --git a/examples/personal-workspace-browser/research-history.md b/examples/personal-workspace-browser/research-history.md index 9c2cf52246..407682ca31 100644 --- a/examples/personal-workspace-browser/research-history.md +++ b/examples/personal-workspace-browser/research-history.md @@ -48,6 +48,45 @@ below. No model or solver is called by this scenario. ## Earlier research checkpoints +### Saved analysis reference + +The core scenario also opens the [actual Graph saved analysis](../../apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json) +from the 11–12 October native run in Chinese and English. Its six purposively +selected episodes preserve all 36 model judgments, complete explanations, +negative/unknown verdicts and original pack/run/revision identities. This older +case supplements the recent Grid logs; it is not another recent complete history. +The analysis headline preserves the mixed result: five recorded utility tests +and one unknown, without turning a test into a positive benefit. + +| Judgment | Yes / no / unknown | +| --- | --- | +| Goal alignment | 4 / 0 / 2 | +| Utility tested | 5 / 0 / 1 | +| Discrimination | 0 / 2 / 4 | +| Information gain | 6 / 0 / 0 | +| Evidence adoption | 2 / 0 / 4 | +| Avoidable repetition | 0 / 3 / 3 | + +The recorded analysis used **447.0 seconds, 53,768 input and 19,278 output +tokens**. These are analysis costs; solver token usage and money remain unknown. +The invalid first submission remains missing rather than zero, and the native +best/retained result remains distinct from the final submitted candidate. + +This fixture freezes the real HTTP readbacks, including the Chinese/English +cache misses and the saved unspecified-language original. The packaged frontend +must recover that original without inventing a translation, changing its usage +or invoking a model. The preview replays captured GET responses; it does **not** +import the original analysis store or qualify the live execution backend. The +existing synthetic process fixture separately protects that owner boundary. +Tests pin the original HTTP byte digests and read all 36 explanations in both +interfaces at desktop/390px widths. + +Only one public-safe native-result citation range is bundled. Other cited ranges +return an explicit unavailable response; no transcript or failure stack is +substituted. The selected Goal has no Explore log: its empty history is preserved. +Pass `--fixture apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json` +to the same read-only preview to inspect this case. All mutation verbs are denied. + The broader `research-history` regression explicitly loads three actual EdgeBench checkpoints sealed on 15 October 2026 and **restored by subsequent experiments**. It replaces the older Squirrel example. The Explore records retain their original text, diff --git a/examples/personal-workspace-browser/research-real-benchmark.mjs b/examples/personal-workspace-browser/research-real-benchmark.mjs index df3a5e3826..6d085a0560 100644 --- a/examples/personal-workspace-browser/research-real-benchmark.mjs +++ b/examples/personal-workspace-browser/research-real-benchmark.mjs @@ -4,6 +4,75 @@ import {resolve} from "node:path"; import {outputDir, repoRoot} from "./fixture.mjs"; import {startHistoryOwner} from "./research-history.mjs"; +async function readSavedGraphAnalysis(browser, collectCoverage) { + const path = resolve(repoRoot, "apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json"); + const capture = JSON.parse(await readFile(path, "utf8")); + const saved = capture.analysis_readbacks.original.response.pack; + const owner = await startHistoryOwner(path); + let page; + const coverageEntries = []; + try { + page = await browser.newPage({viewport: {width: 1440, height: 900}}); + if (collectCoverage) await page.coverage.startJSCoverage({resetOnNavigation: false}); + for (const language of ["zh", "en"]) { + await page.addInitScript(value => localStorage.setItem("loopx-pw-locale", value), language); + await page.goto(`${owner.base}/chat/?goalId=${capture.goal_id}&view=research`); + const history = page.locator(".research-history"); + await history.getByRole("button", {name: language === "zh" ? "查看分析" : "View analysis", exact: true}).click(); + const analysis = history.locator(".research-analysis[role=region]"); + await analysis.locator(".analysis-episode").first().waitFor(); + assert.match(await analysis.locator(".analysis-warning").first().innerText(), language === "zh" ? /已保存的原文/ : /saved original/); + assert.match(await analysis.locator(".analysis-finding h4").innerText(), language === "zh" + ? /5 \/ 6 个片段.*1 个证据不足/ : /5 \/ 6 episodes.*1 remains unknown/, + "One unknown episode must not hide five recorded utility tests; testing still does not establish positive benefit"); + assert.equal(await analysis.locator(".analysis-episode").count(), 6); + assert.equal(await analysis.locator("[data-analysis-judgment]").count(), 36, "All real judgments retain negative and unknown verdicts"); + for (const episode of saved.episodes) { + const item = analysis.locator(`[data-analysis-episode="${episode.id}"]`); + await item.locator(":scope > summary").click(); + for (const summary of await item.locator(".analysis-judgments > details > summary").all()) await summary.click(); + const text = await item.innerText(); + for (const assessment of saved.result.assessments.filter(row => row.episode_id === episode.id)) { + assert(text.includes(assessment.reason), `${episode.id}/${assessment.metric}: complete original reasoning remains readable`); + } + } + const citation = capture.analysis_citations[0].response; + const cited = saved.result.assessments.find(row => row.citations.some(c => c.source_id === citation.source_id + && c.first_line === citation.first_line && c.last_line === citation.last_line)); + assert(cited, "The portable excerpt is an actual judgment citation"); + const episode = analysis.locator(`[data-analysis-episode="${cited.episode_id}"]`); + await episode.locator(":scope > summary").click(); + for (const summary of await episode.locator(".analysis-judgments > details:not([open]) > summary").all()) await summary.click(); + await episode.getByRole("button", {name: `${citation.source_id} · ${language === "zh" ? "行" : "lines"} ${citation.first_line}–${citation.last_line}`, exact: true}).first().click(); + const source = history.locator(".analysis-source"); + await source.locator("pre").waitFor(); + assert.equal(await source.locator("pre").innerText(), citation.text); + await source.getByRole("button", {name: language === "zh" ? "关闭原文" : "Close source", exact: true}).click(); + const method = analysis.locator(".analysis-method"); + await method.locator(":scope > summary").click(); + assert.match(await method.innerText(), /447\.0s.*53,768.*19,278/); + assert.match(await method.innerText(), /graph-official-native3600-integration-20261011-04/); + assert.equal(await analysis.getByRole("button", {name: /用中文分析|Analyze selected scope/}).count(), 0, "A frozen readback never offers execution"); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 900}); + await analysis.scrollIntoViewIfNeeded(); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); + await page.screenshot({path: resolve(outputDir, `research-real-graph-analysis-${language}-${width}.png`), animations: "disabled"}); + } + await page.setViewportSize({width: 1440, height: 900}); + } + const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json(); + assert.equal(readback.source_unchanged, true); + assert.equal(readback.mutation_attempts, 0); + assert(readback.analysis_reads >= 4, "Both interfaces read their cache miss and the same saved original"); + if (collectCoverage) coverageEntries.push(...await page.coverage.stopJSCoverage()); + return coverageEntries; + } finally { + if (page) await page.close(); + await owner.close(); + } +} + // Core whole-page reference: unmodified display state from two real terminal // runs. Synthetic scenarios remain separate boundary/race/overflow tests. export const researchRealBenchmarkScenario = { @@ -87,6 +156,7 @@ export const researchRealBenchmarkScenario = { } } assert.notEqual(observations[0].source_run_id, observations[1].source_run_id); + coverageEntries.push(...await readSavedGraphAnalysis(browser, collectCoverage)); return {coverageEntries, observations}; }, }; diff --git a/tests/presentation/test_research_history_fixture.py b/tests/presentation/test_research_history_fixture.py index bf2d6e2ef7..b7fc566274 100644 --- a/tests/presentation/test_research_history_fixture.py +++ b/tests/presentation/test_research_history_fixture.py @@ -72,8 +72,9 @@ def test_synthetic_provider_uses_selected_python_with_real_arguments_pipes_and_e assert json.loads(result.stdout) == [arguments, "Synthetic input", str(tmp_path), sys.executable] -def test_captured_analysis_fixture_serves_real_saved_results_without_source_mutation(tmp_path, monkeypatch): - fixture = ROOT / "apps/presentation/dashboard/src/data/fixtures/research-synthesis-readback.json" +@pytest.mark.parametrize("fixture_name", ["research-synthesis-readback.json", "edgebench-graph-saved-analysis.json"]) +def test_captured_analysis_fixture_serves_real_saved_results_without_source_mutation(tmp_path, monkeypatch, fixture_name): + fixture = ROOT / "apps/presentation/dashboard/src/data/fixtures" / fixture_name capture = json.loads(fixture.read_bytes()) class InspectingServer(backend.ChatHTTPServer): @@ -94,7 +95,38 @@ def read(path): "goal_id": capture["goal_id"], "view": "history", })) assert {event["title"] for event in history["items"]} == {event["title"] for event in capture["events"]} - for language in capture["analyses"][0]["outputs"]: + if "analysis_readbacks" in capture: + # Pinned original HTTP bytes, not a self-updating digest. + expected = { + "original": "0e4bb4b931784abe1faf8cba49922aeb36f6c96f9e1730fdc77d82e7ebd3da2f", + "zh": "cb8fdf1a61915f06168816c82367e9d037bdf49424028cd2793dfe61364def64", + "en": "d76b579672d87f8b071d871ecfd5797b986f295b8dc93bc967471c0e1aab0b2b", + } + for language, record in capture["analysis_readbacks"].items(): + with urlopen(base + record["request_path"], timeout=10) as response: + raw = response.read() + assert hashlib.sha256(raw).hexdigest() == expected[language] == record["response_sha256"] + view = capture["analysis_readbacks"]["original"]["response"]["pack"] + assert view["episode_count"] == 6 and len(view["result"]["assessments"]) == 36 + assert {row["verdict"] for row in view["result"]["assessments"]} == {"yes", "no", "unknown"} + assert view["job"]["usage"]["input_tokens"] == 53768 + assert view["job"]["usage"]["output_tokens"] == 19278 + assert view["native_outcomes"]["projection"]["initial"] is None + assert "solver_usage" not in view and not events + citation = capture["analysis_citations"][0] + assert read(citation["request_path"]) == citation["response"] + for query in [ + {"output_language": "fr"}, {"pack_id": "wrong-pack"}, + ]: + with pytest.raises(HTTPError) as rejected: + read("/api/chat/research-analysis?" + urlencode({"goal_id": capture["goal_id"], **query})) + assert rejected.value.code == 404 + with pytest.raises(HTTPError) as rejected: + read(citation["request_path"].replace("native-report-4", "uncaptured-source")) + assert rejected.value.code == 404 + # No analysis artifacts, provider output or job state are synthesized. + assert not list(self.runtime_root.parent.glob("analysis-*")) + for language in capture.get("analyses", [{}])[0].get("outputs", {}): view = read("/api/chat/research-analysis?" + urlencode({ "goal_id": capture["goal_id"], "output_language": language, }))["pack"] @@ -106,21 +138,22 @@ def read(path): assert readback["source_unchanged"] is True assert readback["mutation_attempts"] == 0 assert readback["fixture_sha256"] == hashlib.sha256(fixture.read_bytes()).hexdigest() - evidence = self.runtime_root.parent / f"analysis-{capture['goal_id']}-0" / "facts.txt" - original = evidence.read_bytes() - evidence.write_bytes(b"X" + original[1:]) - try: - with pytest.raises(PackUnavailable, match="source_changed"): - backend.execute_pack(runtime_root=self.runtime_root, owner_goal_id=capture["goal_id"], - pack_id=view["pack_id"], analysis_key=view["analysis_key"], output_language=language, - codex="must-not-launch-for-changed-source") - finally: - evidence.write_bytes(original) - for method in ("POST", "PUT", "DELETE"): + if "analyses" in capture: + evidence = self.runtime_root.parent / f"analysis-{capture['goal_id']}-0" / "facts.txt" + original = evidence.read_bytes() + evidence.write_bytes(b"X" + original[1:]) + try: + with pytest.raises(PackUnavailable, match="source_changed"): + backend.execute_pack(runtime_root=self.runtime_root, owner_goal_id=capture["goal_id"], + pack_id=view["pack_id"], analysis_key=view["analysis_key"], output_language=language, + codex="must-not-launch-for-changed-source") + finally: + evidence.write_bytes(original) + for method in ("POST", "PUT", "DELETE", "PATCH"): with pytest.raises(HTTPError) as rejected: urlopen(Request(base + "/api/chat/turns", data=b"{}", method=method), timeout=10) assert rejected.value.code == 405 - assert read("/fixture/readback")["mutation_attempts"] == 3 + assert read("/fixture/readback")["mutation_attempts"] == 4 assert read("/fixture/readback")["source_unchanged"] is True finally: self.shutdown()