{ "metadata": { "name": "Harness RSI Evidence Dataset", "version": "0.1.0", "annotations_sha256": "056dadb12881581310e4a01c500e2c4c5f7c8980ecd0e4650bfa114eded2dfc8", "released_on": "2026-10-18", "unit": "category-source record; not an individual intervention or independent experiment", "frozen_evidence_sha256": "b0d5a4ae1ccae267f147af5480f8461d9be7092f25f966e8934614f9fcb9edb2", "annotation_basis": "AI-assisted manual screening of the existing curated evidence text. Structured signals link to exact excerpts of that text and inherit its primary-source locators. This release is not a new independent primary-source audit or experimental reproduction.", "license_status": "No dataset-specific reuse license has been granted. Public access is not a blanket license; source materials retain their own rights.", "limitations": [ "Outcome and retention screening is partial and not a representative sample. Unscreened records are not negative-free records.", "One record can contain several differently scoped signals. Signals are not independent trials and must not be used to estimate an intervention success rate.", "Unreported failures cannot be recovered from the source literature; this dataset does not eliminate publication or survivorship bias.", "A null field means not assessed in this annotation pass, not absent or not reported in the original paper.", "A frozen task set alone does not isolate component-level causality; models, budgets, selection access and combined changes also matter." ], "evidence_cutoff": "2026-10-01", "record_count": 95, "distinct_source_count": 63, "screened_record_count": 4, "unreviewed_record_count": 91, "signal_count": 6 }, "records": [ { "id": "program-generation-and-editing--sica2025", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "sica2025", "title": "SICA: local editing and symbol navigation", "sections": [ { "label": "Setting and seed", "text": "LiveCodeBench participates in one mixed-task evolution run. The seed already has subagents and file-context management but modifies files only through whole-file overwrites." }, { "label": "Reported evolution", "text": "Whole-file overwrites are extended with diff- and range-based editing, followed by a Python AST symbol locator in iteration 9. Appendix D's update sequence also reports a SmartEditor InputNormalizer (update 6), reduced diff-context output (8 and 13), MainAgent detection and routing of editing tasks (10), and AST parsing restricted to relevant code regions instead of entire files (14). These change the reusable editing interface and the amount of code processed or returned." }, { "label": "Evidence and limits", "text": "These are reported implemented updates; retention in the final utility-winning agent remains unconfirmed. The normalizer's exact transformations were not independently checked in code. The 15-round run combines 50 SWE tasks, 50 LiveCodeBench tasks and synthetic editing/navigation tasks; synthetic navigation mainly motivates the AST tool. No individual ablation establishes the effect of these updates, and their appearance in two benchmark records is not independent replication. The seed's XML tool-call format, subagents and context management are inherited capabilities." } ], "sources": [ { "name": "SICA (v2)", "url": "https://arxiv.org/html/2504.15228v2", "locator": "§3, §3.1; §4 Figure 3/Table 1; Appendix A–C; Appendix D, updates 6, 8–10, 13–14" } ], "benchmarks": [ { "id": "livecodebench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--tthe2026", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "tthe2026", "title": "TTHE: verification only after exposed failures", "sections": [ { "label": "Setting and seed", "text": "Test-time adaptation uses a recent hard subset of 60 LiveCodeBench problems, starting from a ReAct scaffold without the reported conditional verification-and-repair route; the release window is unspecified." }, { "label": "Reported evolution", "text": "The selected scaffold retains a low-cost solving path and adds verification and repair when public tests fail. Additional execution is thus conditional on an observed failure, rather than required for every problem." }, { "label": "Evidence and limits", "text": "The route is reported as retained. Adaptation and scoring use the same batch, making this a transductive result rather than a sealed generalization test. There is no individual-change ablation. The SWE empty-patch retry and DS-1000 tasks in the shared configuration do not establish additional LiveCodeBench mechanisms." } ], "sources": [ { "name": "TTHE (v1)", "url": "https://arxiv.org/html/2607.08124v1", "locator": "§4.4–4.5; §5.1,5.6; Appendix Figure5" } ], "benchmarks": [ { "id": "livecodebench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--dgm2025", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "dgm2025", "title": "Darwin Gödel Machine: recoverable edits and candidate delivery", "sections": [ { "label": "Setting and seed", "text": "Aider Polyglot evolution starts from bash, a whole-file editor and one agent attempt. Optimization uses 60 problems; 165 further problems support generalization evaluation." }, { "label": "Reported evolution", "text": "The retained implementation adds replacement, insertion, range editing and undo, plus up to three attempts with workspace resets. Patch/test summaries inform later attempts; execution success and test counts guide selection, after which the selected patch is replayed. Local revision becomes recoverable, while candidate exploration is separated from final delivery." }, { "label": "Evidence and limits", "text": "The full-benchmark score increases from 14.2% to 30.7% (Section 4.4). This includes the optimization subset and measures the complete lineage, not either mechanism in isolation. Appendix F.2 gives the retained code, but its diff has not been linked to a repository commit. Internal tests do not access private ground truth. The coarse seed interface is a documented limitation, not an isolated failure-trace experiment." } ], "sources": [ { "name": "DGM (v3)", "url": "https://arxiv.org/html/2505.22954v3", "locator": "§3–4.4; Appendix C.1, E.2–E.5, F.1–F.2" }, { "name": "DGM (v3)", "url": "https://arxiv.org/html/2505.22954v3#A6.SS2", "locator": "Appendix F.2: SolutionAttempt/run_tests/forward; tools/edit.py; HTML L3622–4395" } ], "benchmarks": [ { "id": "aider-polyglot", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--evolvenet2026", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "evolvenet2026", "title": "EvolveNet: separating library-specific and shared behavior", "sections": [ { "label": "Setting and seed", "text": "DS-1000 evolution starts from a 17-line shared seed program, before client-specific improvements are merged." }, { "label": "Reported evolution", "text": "The round-2 example keeps schema preanalysis inside the Pandas branch while moving broadly applicable execution retries into shared cross-library logic. The merge preserves the intended scope of each client specialization." }, { "label": "Evidence and limits", "text": "This is a reported merge discovery, not independently inspected final source; final retention remains unconfirmed. The protocol uses local feedback, server validation and held-out tests, but supplies no causal ablation of this merge. The Pandas-specific example does not establish a concrete discovery on SWE or LiveCodeBench, nor does a short seed alone identify a particular tool architecture." } ], "sources": [ { "name": "EvolveNet (v1)", "url": "https://arxiv.org/html/2608.04968v1", "locator": "§6.1–6.9; Appendix D" } ], "benchmarks": [ { "id": "ds-1000", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--aflow2024", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "aflow2024", "title": "AFlow: composing generation, selection and conditional repair", "sections": [ { "label": "Setting and seed", "text": "MBPP search begins with an empty workflow template and manually predefined operators. The LLM revises reusable code and prompts using parent workflows, modification history and execution feedback." }, { "label": "Reported evolution", "text": "The selected round-14 program generates three candidates, selects with ScEnsemble and invokes Test. Failed tests route their output to FIX_CODE_PROMPT for repair. The evolved object is this executable composition and its conditional branch, rather than a newly invented operator or a single task answer." }, { "label": "Evidence and limits", "text": "Each candidate receives five validation runs, and the cited listing is the retained best MBPP program. The failure branch passes the problem, failed solution and returned test error to the repair prompt; the listing returns the repaired candidate without a further test call. A separate test-generation example is not this final chain. Component-level causal gains are not established, and the workflow must not be relabeled as HumanEval's selected implementation." } ], "sources": [ { "name": "AFlow (v4)", "url": "https://arxiv.org/pdf/2410.10762v4", "locator": "§4–5; Appendix A.3–A.6; Appendix B.1, optimal MBPP workflow (PDF p.22)" } ], "benchmarks": [ { "id": "mbpp", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--harnessbank2026", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "harnessbank2026", "title": "HarnessBank: recovery after runaway reasoning", "sections": [ { "label": "Setting and seed", "text": "The EvoAgentBench LiveCode setting starts from a vanilla harness that can return an empty turn after reasoning exceeds its budget. The precise initial implementation is unspecified." }, { "label": "Reported evolution", "text": "A retained recovery branch disables thinking only after runaway reasoning prevents output from resuming. Ordinary execution keeps its original reasoning mode, so the change is event-triggered recovery rather than globally disabling thinking or uniformly expanding the budget." }, { "label": "Evidence and limits", "text": "Training and test data are separate, with one sealed test after training-based selection. The relationship between EvoAgentBench LiveCode and official LiveCodeBench, including release alignment, remains unverified. The recovery example cannot be assigned to official LiveCodeBench or Terminal-Bench, and it does not isolate the causal contribution of the branch from the complete harness." } ], "sources": [ { "name": "HarnessBank (v2)", "url": "https://arxiv.org/html/2607.13683v2", "locator": "§4.1–4.2,4.5–4.6; Table1" } ], "benchmarks": [ { "id": "evoagentbench-livecode", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--mermaidflow2025", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "mermaidflow2025", "title": "MermaidFlow: inheriting testing and candidate diversity", "sections": [ { "label": "Setting and seed", "text": "HumanEval search represents workflows as editable Mermaid graphs over supplied operators. The illustrated crossover starts from two existing parent workflows, not a newly identified universal seed." }, { "label": "Reported evolution", "text": "Workflow_8 combines the test node introduced by Workflow_4 with Workflow_5's more diverse ensemble. Verification and candidate diversity can therefore be inherited from different parents through executable graph composition." }, { "label": "Evidence and limits", "text": "Section 5.4 and Figure 4 document this generated child. Appendix B.3 separately provides a selected HumanEval implementation; the crossover child is not automatically that final winner. The reported score does not isolate the testing node's effect, and the operators themselves are supplied rather than evolved by this example." } ], "sources": [ { "name": "MermaidFlow", "url": "https://arxiv.org/abs/2505.22967v1", "locator": "Section 5.4, Figure 4; Appendix B.3" } ], "benchmarks": [ { "id": "humaneval", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "program-generation-and-editing--flowevo2026", "domain": "coding", "category": "Program Generation and Editing", "category_id": "program-generation-and-editing", "key": "flowevo2026", "title": "FlowEvo: reusing skills directly or as construction context", "sections": [ { "label": "Setting and seed", "text": "HumanEval and MBPP use generated workflows augmented by a persistent skill bank. Successful executions can supply procedures for subsequent tasks; Appendix A separates bank growth from frozen-bank MBPP evaluation." }, { "label": "Reported evolution", "text": "Acquired skills can execute directly when compatible, or provide structured context for constructing a new workflow. This changes what is retained between problems and how the next solver uses it." }, { "label": "Evidence and limits", "text": "On the 71 exact-transfer cases, direct reuse reaches 85.9% versus 81.7% for dynamic generation with lower token use; text memory reaches 87.3% at higher cost. No router pair separates significantly on the full evaluation split. These comparisons support conditional reuse, not universal replay superiority or a final coding-specific skill lineage. The detailed ALFWorld listing is not a coding artifact." } ], "sources": [ { "name": "FlowEvo", "url": "https://openreview.net/forum?id=hU2N7IIkcE", "locator": "Sections 3–4; Appendix A" } ], "benchmarks": [ { "id": "humaneval", "role": "study" }, { "id": "mbpp", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--dgm2025", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "dgm2025", "title": "Darwin Gödel Machine: editing, patch selection and context recovery", "sections": [ { "label": "Setting and seed", "text": "SWE-bench evolution begins with a fixed model, bash and whole-file editing. Test execution and test summarization are already seed capabilities." }, { "label": "Reported evolution", "text": "The best lineage adds line-range viewing and unique string replacement. Appendix F.1's added is_patch_valid rejects empty or whitespace-only patches, patches with no parsed modified files, and changes classified as test-only by a filename heuristic. The execution loop resets the workspace and supplies the failure reason before another attempt, with up to three attempts in total. It also compares candidates with internal tests and summarizes older messages after context overflow before resuming. Testing itself is inherited; the evolved changes govern its use and the recovery paths." }, { "label": "Evidence and limits", "text": "The changes are reported as retained. Screening on 10+50 tasks precedes evaluation on 200 tasks that include those screening tasks; this is not an independent 500-task test. No individual causal ablations apportion the overall gain among the changes. Context summarization does not guarantee preservation of every earlier tool result." } ], "sources": [ { "name": "DGM (v3)", "url": "https://arxiv.org/html/2505.22954v3", "locator": "§3–4.4; Appendix C.1, E.2–E.5, F.1–F.2" }, { "name": "DGM (v3)", "url": "https://arxiv.org/html/2505.22954v3#A6.SS1", "locator": "Appendix F.1: coding_agent.py is_patch_valid/forward; tools/edit.py; llm_withtools.py summarize_messages; HTML L1747–3620" } ], "benchmarks": [ { "id": "swe-bench-dgm-200-task-setting", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--sica2025", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "sica2025", "title": "SICA: extending the inherited editing interface", "sections": [ { "label": "Setting and seed", "text": "SWE-bench Verified shares one mixed-task run with LiveCodeBench and synthetic tasks. The seed already contains subagents, an asynchronous overseer and file-context management, but editing is limited to overwriting files." }, { "label": "Reported evolution", "text": "Evolution adds diff- and range-based editing, followed by an AST symbol locator in iteration 9. Appendix D further records editor-input normalization, task-based edit routing, smaller diff-context returns and parsing only relevant AST regions; the program-generation entry gives the update numbers. The existing subagents, overseer and XML tool-call parser are inherited architecture." }, { "label": "Evidence and limits", "text": "The fixed 50-task SWE subset participates in a 15-iteration mixed-task run, not an independent repository-only search. These are reported implemented discoveries whose retention in the final utility winner has not been verified from its diff. Overall SWE improvements cannot be assigned to individual changes, and the AST discovery is primarily associated with synthetic navigation tasks." } ], "sources": [ { "name": "SICA (v2)", "url": "https://arxiv.org/html/2504.15228v2", "locator": "§3, §3.1; §4 Figure 3/Table 1; Appendix A–C; Appendix D, updates 6, 8–10, 13–14" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--multitask_selfharness2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "multitask_selfharness2026", "title": "Multi-task Self-Harness: one lineage across three repository settings", "sections": [ { "label": "Setting and seed", "text": "The seed is a 49-line prompt-and-bash loop that appends raw outputs without timeouts, compression or recovery. SWE-Bench Pro contributes 60 evolution tasks and 100 held-out tasks from a pool of 731. Verified (100 tasks) and DeepSWE (53 tasks) are frozen out-of-distribution evaluations, excluded from evolution." }, { "label": "Reported evolution", "text": "Outputs previously appended verbatim are bounded: above 48,000 characters, the harness preserves the beginning and end and inserts a notice suggesting a narrower command or saving output to a file. Outputs older than the latest six are shortened to at most 6,000 characters. These rules appear in iteration 1 and remain in the selected iteration 5. Separate history compaction preserves opening and recent messages, and a fresh-context completion review checks the task against the filesystem." }, { "label": "Evidence and limits", "text": "Section 5.2 reports that the seed aborts on 12 SWE-Bench Pro tasks because test or build output exceeds the input limit; iteration 5 solves seven of them. This is a complete-version comparison, not an ablation of truncation. The same lineage serves all three settings, so they are not independent discoveries. History compaction rarely triggers in coding; it is distinct from per-output trimming. The iteration-6 budget cap is excluded. DeepSWE currently has no second extracted study supplying a concrete change for comparison." } ], "sources": [ { "name": "Multi-Task Self-Harness (v1)", "url": "https://arxiv.org/html/2609.38372v1", "locator": "§3.2–3.4; §4.1 Table 3; §5.1–5.7 Table 4; Appendix C" } ], "benchmarks": [ { "id": "swe-bench-pro", "role": "study" }, { "id": "swe-bench-verified", "role": "transfer" }, { "id": "deepswe", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--selfharness2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "selfharness2026", "title": "Self-Harness: model-specific patch verification", "sections": [ { "label": "Setting and seed", "text": "Three independent model-specific searches on a 100-task SWE-bench Verified subset start from a minimal DeepAgent harness. Seed MiniMax and Qwen traces make changes but stop without completing executable verification." }, { "label": "Reported evolution", "text": "MiniMax separates empty-diff inspection from targeted testing through two subagents. Qwen combines a patch verifier with pre-submission diff/test instructions. In GLM, build_verification_instruction previously requested test execution and debugging on failure. The evolved instructions distinguish missing imports or dependencies, ask the agent to install the missing package and rerun the exact test, and resume source debugging only if it still fails. This is a retained instruction change, not a programmatic gate enforcing that sequence." }, { "label": "Evidence and limits", "text": "The modifications are retained in separate promoted harnesses. Appendix B.2 reports successful verification in the paired MiniMax and Qwen traces. Its GLM Astropy case (Figure 16) moves from syntax-only checking to dependency repair and executable verification: all four previously failing FAIL-TO-PASS tests then pass. These are whole-harness case comparisons, not ablations of particular instructions or subagents. Table 1 separately gives MiniMax overall Pass of 46.0% versus 52.5%. Both 67 held-in and 33 held-out tasks may affect acceptance, so the latter are not an untouched test set. These are separate searches within one paper, not frozen transfers from Terminal-Bench." } ], "sources": [ { "name": "Self-Harness (v3)", "url": "https://arxiv.org/html/2606.09498v3", "locator": "§3.3,4.1; Figure3; Appendix A.2" }, { "name": "Self-Harness (v3)", "url": "https://arxiv.org/html/2606.09498v3", "locator": "§4.3 Figures5–8; Appendix B.1–B.2" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--tthe2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "tthe2026", "title": "TTHE: treating an empty patch as a rerun condition", "sections": [ { "label": "Setting and seed", "text": "Test-time adaptation on a 40-task hard SWE-bench Verified slice begins with stock mini-swe-agent, whose first attempt can terminate with an empty patch." }, { "label": "Reported evolution", "text": "The adapted scaffold reruns an empty-patch attempt with explicit reproduce–localize–repair–retest instructions and increases the step budget during multiround search. The change converts an otherwise terminal outcome into a conditional recovery path." }, { "label": "Evidence and limits", "text": "The selected mechanism is reported as retained. Selection without gold labels and subsequent scoring use the same batch, so this is transductive adaptation rather than unseen-task generalization. The budget and instructions change together. LiveCodeBench's public-test mechanism is not assigned to SWE, and the example's small-patch constraint is not general repository-repair advice." } ], "sources": [ { "name": "TTHE (v1)", "url": "https://arxiv.org/html/2607.08124v1", "locator": "§4.4–4.5; §5.1,5.6; Appendix Figure5" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--harnesscompass2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "harnesscompass2026", "title": "HarnessCompass: accumulating conditional repair guidance", "sections": [ { "label": "Setting and seed", "text": "Search on SWE-bench Verified starts from a short prompt and shell tool, without memory, middleware, skills or subagents. The split contains 50 search and 450 evaluation tasks, with additional cross-model evaluation." }, { "label": "Reported evolution", "text": "Retained memory adds conditional guidance for identifying responsibility along execution paths and covering affected callers when changing shared interfaces. An ungated control instead accumulates more task-specific repair records." }, { "label": "Evidence and limits", "text": "The examples are reusable guidance, not enforced tool behavior. They support a distinction between conditional repair knowledge and memorized fixes, but neither rule has an independent ablation. Gains from the complete structure and guidance tracks cannot be attributed to memory alone. Evaluation on unseen tasks does not make the two rules independent discoveries." } ], "sources": [ { "name": "HarnessCompass (v1)", "url": "https://arxiv.org/html/2608.01918v1", "locator": "Experiments Setup; Appendix C Two kinds of accumulated memory" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--ahe2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "ahe2026", "title": "AHE: transferring execution and delivery controls", "sections": [ { "label": "Setting and seed", "text": "All 500 SWE-bench Verified tasks evaluate frozen transfer from Terminal-Bench 2 evolution. The source seed, NexAU_0, exposes only bash and has no middleware, skills or long-term memory." }, { "label": "Reported evolution", "text": "The transferred harness exposes per-call timeouts, protects accepted artifacts from destructive actions, forwards cross-step risk reminders to the next model turn, and checks acceptance criteria before completion. These occupy distinct tool, interception, context and completion modules; reminders are not equivalent to execution-level blocking." }, { "label": "Evidence and limits", "text": "The changes are reported as retained, but every concrete edit was discovered in the terminal source domain. They are not SWE discoveries. One lineage introduces interacting components, and no isolated SWE contribution is established for any single mechanism. Frozen transfer therefore supports reuse of the complete harness, not independent evolution on another benchmark." } ], "sources": [ { "name": "AHE (v4)", "url": "https://arxiv.org/html/2604.25850v4", "locator": "§4.1–4.4; Appendix A Table 4; B.1" }, { "name": "AHE (v4)", "url": "https://arxiv.org/html/2604.25850v4", "locator": "Appendix C.1–C.2; Figures 8–11; commits 169c34c,4e0aab9,ff0cf3d,9651986,ca35f53,a4a4a29" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--darwinx2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "darwinx2026", "title": "DarwinX: transferring artifact and acceptance guidance", "sections": [ { "label": "Setting and seed", "text": "SWE-bench Verified is a 500-task frozen transfer target after skill evolution on Terminal-Bench 2.1. The source seed is base Monet without the seven later verification/artifact skills." }, { "label": "Reported evolution", "text": "The retained bundle extracts acceptance contracts, grounds deliverables in real tool results, checks and repairs artifacts, repeats checks after modifications, and repairs safety/interface-contract violations before completion." }, { "label": "Evidence and limits", "text": "The seven skills are jointly selected execution guidance, not separately ablated mechanisms or an unavoidable submission blocker. The bundle's SWE evaluation does not show where the skills were discovered. Terminal-Bench, TerminalWorld and different model configurations have distinct protocols; evidence from those configurations must not be merged into a single repository evolution run." } ], "sources": [ { "name": "DarwinX (v1)", "url": "https://arxiv.org/html/2608.07545v1", "locator": "§3,§8.1 Table6; Appendix B Table8" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--mixture_branches2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "mixture_branches2026", "title": "Mixture of Self-Improving Branches: retrieval before the first action", "sections": [ { "label": "Setting and seed", "text": "SWE-bench Lite uses mini-swe-agent as its original baseline. The immediate parent of the illustrated modification already has structured editing; the 300 tasks are split into 50 development and 250 test tasks." }, { "label": "Reported evolution", "text": "The candidate extracts up to eight issue identifiers, ranks files by matches, reads two source files and one test file, and injects bounded content into the initial message. This changes initialization through targeted retrieval, rather than adding the parent's existing unique-match replacement tool." }, { "label": "Evidence and limits", "text": "Appendix E.4 supplies a discovery example; retention in the final deployed head is unverified. Development-selected heads and test-best candidates are reported separately and must not be conflated. The example does not establish an individual causal gain or a universal best retrieval budget." } ], "sources": [ { "name": "Mixture of Self-Improving Branches (v1)", "url": "https://arxiv.org/html/2609.37834v1", "locator": "§4.1; Tables1–4" }, { "name": "Mixture of Self-Improving Branches (v1)", "url": "https://arxiv.org/html/2609.37834v1", "locator": "Appendix E.3–E.4" } ], "benchmarks": [ { "id": "swe-bench-lite", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--rho2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "rho2026", "title": "RHO: connecting requirements to executable verification", "sections": [ { "label": "Setting and seed", "text": "SWE-Bench Pro uses a domain-specific optimization from vanilla Codex, before its repair-verify helper and requirements ledger. It is not a frozen transfer of the terminal configuration." }, { "label": "Reported evolution", "text": "A retained callable helper finds toolchains outside PATH, runs lightweight language checks, reports generated artifacts and accepts custom verification commands. A separate requirements ledger is revisited before submission." }, { "label": "Evidence and limits", "text": "The setup draws 100 training and 100 held-out tasks from the ScaleAI test set using evaluator revision 0c64e26f00b9. Held-out trajectories show helper calls, distinguishing this configuration from the unexecuted terminal scripts. No single-change ablation isolates the helper, and terminal playbook findings cannot be substituted for these repository observations." } ], "sources": [ { "name": "RHO (v3)", "url": "https://arxiv.org/html/2606.05922v3", "locator": "§5 setup; §6.1; Appendix E.1–E.2" }, { "name": "RHO (v3)", "url": "https://arxiv.org/html/2606.05922v3", "locator": "Appendix H.1–H.2; §6.1" } ], "benchmarks": [ { "id": "swe-bench-pro", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--one_recipe2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "one_recipe2026", "title": "OneRecipe: timed reminders for language-specific verification", "sections": [ { "label": "Setting and seed", "text": "Multi-SWE-Bench evolution runs by language and model from a minimal mini-SWE plugin: a short prompt, bash, and empty hooks and memory. Full mini-SWE is a separate baseline; evaluation also considers cross-language transfer." }, { "label": "Reported evolution", "text": "The retained C++/Haiku example uses before_query to detect recent testing or submission cues and inject compilation/testing reminders. Prompts and memory constrain patch scope, discourage test-file edits and request targeted verification." }, { "label": "Evidence and limits", "text": "The procedure uses three rounds, 80 steps and USD 1.5 per rollout. The hook nudges rather than blocks submission. make/ctest depend on the language ecosystem, and the test-file restriction reflects the hidden gold-test-patch convention. Aggregate transfer results do not isolate the contribution of either guidance content or trigger timing." } ], "sources": [ { "name": "OneRecipe (v1)", "url": "https://arxiv.org/html/2608.10178v1", "locator": "§3.1; §4.3–4.6; Appendices A.6–A.8, B.1–B.3, G, H" } ], "benchmarks": [ { "id": "multi-swe-bench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--codeskill2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "codeskill2026", "title": "CODESKILL: task procedures and event-triggered diagnostics", "sections": [ { "label": "Setting and seed", "text": "A separately trained manager maintains natural-language skills around frozen coding policies. EnvBench uses a ReAct-style agent and SWE-bench Verified uses mini-SWE-agent; both expose bash-based actions." }, { "label": "Reported evolution", "text": "The published merged Maven task-level skill distinguishes environment, configuration and dependency failures before code edits. Its merged event-level counterpart diagnoses a missing test-jar, checks its producing module, arranges generation/installation, and verifies the artifact before retrying the build. Persistent updates change skill instructions, not the supplied tools." }, { "label": "Evidence and limits", "text": "These are bank examples rather than individually scored skill ablations. Project build-file edits are downstream actions, not evolved harness code. The test tasks are held out from manager-policy training, but Section 4.1 constructs their skill bank in two preparatory passes: extraction from no-skill runs, then evolution from skill-conditioned runs. Final evaluation filters out skills whose recorded provenance includes the current instance. This is test-set bank preparation, not evaluation of an unchanged training-only bank; benchmark gains do not isolate either Maven example." } ], "sources": [ { "name": "CODESKILL", "url": "https://arxiv.org/abs/2605.25430v2", "locator": "Sections 3–4; Appendix D, Figures 4–5" } ], "benchmarks": [ { "id": "swe-bench-verified", "role": "study" }, { "id": "envbench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--gse2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "gse2026", "title": "GSE: generalizing diagnostic skills across repositories", "sections": [ { "label": "Setting and seed", "text": "OpenHands and mini-SWE-agent are evaluated on bug-revealing test generation for 108 Java bugs derived from Multi-SWE-Bench, and false-positive filtering for 500 IndustrialBugs reports. These settings evaluate defect analysis, not patch resolution." }, { "label": "Reported evolution", "text": "Figure 3 consolidates narrow collection-size and permission rules into a skill distinguishing value-triggered from state-triggered defects, linked to existing test-construction skills. The filtering case replaces conflicting project-specific nullability rules with reasoning about nullability guarantees and propagates that abstraction to related skills." }, { "label": "Evidence and limits", "text": "One-project-held-out evaluation tests reuse across repositories. The examples illustrate learned abstraction and compatibility between skills. Aggregate results and architecture-level ablations do not isolate either example's causal effect, and the simplified Figure 3 illustration should not be treated as a full final-bank artifact." } ], "sources": [ { "name": "GSE", "url": "https://arxiv.org/abs/2608.06153v1", "locator": "Sections 4–5; Figure 3" } ], "benchmarks": [ { "id": "multi-swe-bench-java-test-generation", "role": "study" }, { "id": "industrialbugs", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "repository-level-software-engineering--skillforge2026", "domain": "coding", "category": "Repository-Level Software Engineering", "category_id": "repository-level-software-engineering", "key": "skillforge2026", "title": "SkillForge: acquiring repository-specific repair knowledge", "sections": [ { "label": "Setting and seed", "text": "Before real issues are presented, synthetic issue solving acquires project-specific skills associated with repository entities. The downstream repair agent retrieves this knowledge rather than rewriting its controller during the illustrated repair." }, { "label": "Reported evolution", "text": "In Django-11206, the acquired guidance steers the agent away from an exponent heuristic toward the existing Decimal pipeline, precision-dependent numerical equivalence and boundary-case checks." }, { "label": "Evidence and limits", "text": "The paired traces in Section IV-E pass 0/2 and 2/2 fail-to-pass tests without and with skills. This demonstrates an application of acquired knowledge, not a separate causal estimate for each instruction. The persistent object is entity-grounded guidance; the particular code patch is a task output. Knowledge acquisition and downstream repair therefore remain distinct stages." } ], "sources": [ { "name": "SkillForge", "url": "https://arxiv.org/abs/2608.18933v1", "locator": "Section IV-E; Figure 6" } ], "benchmarks": [ { "id": "swe-bench-skillforge-django-case", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--metaharness2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "metaharness2026", "title": "Meta-Harness: discovering the environment before reasoning", "sections": [ { "label": "Setting and seed", "text": "Terminal-Bench 2.0 search and evaluation share 89 tasks. Search considers Terminus 2 and Terminus-KIRA; the illustrated discovery builds on KIRA, inheriting native tool calling, a 30KB output cap and a completion checklist." }, { "label": "Reported evolution", "text": "The retained bootstrap gathers directory contents, language/package-manager versions and memory before the first model call, then injects a snapshot into the initial context. A 15-second timeout and silent failure keep the startup probe from blocking execution." }, { "label": "Evidence and limits", "text": "Appendix B.3 reports gains on seven tasks relative to KIRA, especially where specialized tools are needed, and identifies two to four initial environment-discovery turns as the bottleneck. This within-benchmark comparison supplies no independent coding holdout and is not an isolated causal ablation. The inherited checklist and output cap are not new discoveries; the public author artifact identifies version 2.0." } ], "sources": [ { "name": "Meta-Harness (v1)", "url": "https://arxiv.org/html/2603.28052v1", "locator": "§4.3; Appendix B.3 / Figure 9" } ], "benchmarks": [ { "id": "terminal-bench-2-0", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--ahe2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "ahe2026", "title": "AHE: preserving validated delivery state", "sections": [ { "label": "Setting and seed", "text": "Terminal-Bench 2.0 evolution uses 89 tasks for ten iterations. NexAU_0 begins with bash, a default five-minute shell timeout, and no middleware, skills or long-term memory; SWE is reserved for frozen transfer." }, { "label": "Reported evolution", "text": "Retained edits expose command-specific timeouts, protect validated artifacts, detect cross-step risks, inject next-turn reminders and check completion. In configure-git-webserver, an intermediate overrideable guard still permits cleanup to erase a working deployment. Paired iteration-8 updates harden the guard and make warnings salient before the next action." }, { "label": "Evidence and limits", "text": "That task changes from 0/2 to 2/2 successful rollouts, with destructive cleanup blocked and the live state submitted. This supports the reported mechanism without separating the paired edits. Manifests and commits identify changes, but the historical run has not been reproduced. Components interact non-additively, and Section 4.4.1 reports costs from redundant checks on long-horizon tasks. The current official configuration corroborates version 2.0." } ], "sources": [ { "name": "AHE (v4)", "url": "https://arxiv.org/html/2604.25850v4", "locator": "§4.1–4.4; Appendix A Table 4; B.1" }, { "name": "AHE (v4)", "url": "https://arxiv.org/html/2604.25850v4", "locator": "Appendix C.1–C.2; Figures 8–11; commits 169c34c,4e0aab9,ff0cf3d,9651986,ca35f53,a4a4a29" } ], "benchmarks": [ { "id": "terminal-bench-2-0", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--modularrsi2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "modularrsi2026", "title": "ModularRSI: correcting tool dispatch and completion state", "sections": [ { "label": "Setting and seed", "text": "An initial modular harness evolves on related tasks outside Terminal-Bench, then undergoes frozen evaluation on version 2.0. The separate Section 6.4 method comparison specifies Terminus-2 and does not establish that seed for every main experiment." }, { "label": "Reported evolution", "text": "In Appendix D.4's Go repair case, the initial dispatcher rejects multiline write_file input and sends it to the shell, producing a command-not-found error. The reported revision identifies write_file or edit_block by command name and sends the remaining payload to the editing helper. The paper reports promotion in generation 9 and unchanged multiline handling in generation 12. A separate loop update re-presents pending acceptance conditions when the agent signals completion without commands." }, { "label": "Evidence and limits", "text": "On the same Go task, three runs per version all succeed; their interaction counts (called episodes in the source) change from [41,24,43] to [26,18,31], reducing the mean from 36 to 25. These are external evolution-task observations, not Terminal-Bench test cases or an isolated dispatcher ablation: parser recovery and other changes occur concurrently. A source check of the released merged module finds the editing-command branch nested under a cat-only condition, making that branch unreachable for the editing names in this snapshot. This is not the historical generation 9/12 artifact; those source snapshots and original rollouts remain unchecked. The current code therefore does not independently confirm the paper-reported fix or invalidate its historical observations." } ], "sources": [ { "name": "ModularRSI (v1)", "url": "https://arxiv.org/html/2609.14857v1", "locator": "§4,5.3,6.1,6.4" }, { "name": "ModularRSI (v1)", "url": "https://arxiv.org/html/2609.14857v1", "locator": "Appendix D.2–D.4; Tables 10, Listings 7–9" } ], "benchmarks": [ { "id": "terminal-bench-2-0", "role": "transfer" }, { "id": "go-repair-modularrsi-case", "role": "case" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--selfharness2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "selfharness2026", "title": "Self-Harness: model-specific artifact recovery", "sections": [ { "label": "Setting and seed", "text": "Separate model searches use 64 filtered Terminal-Bench 2.0 tasks. The minimal DeepAgent seed provides read, write, edit and shell tools plus the default task prompt, without the later runtime limits." }, { "label": "Reported evolution", "text": "MiniMax's final harness encourages early creation of required artifacts and redirects stalled work through a total tool-message cap. Qwen instead adds dependency/artifact checks, retry discipline, loop recovery and middleware triggered by tool errors." }, { "label": "Evidence and limits", "text": "These are retained changes in different promoted harnesses, not one shared patch. Held-out trajectories are withheld from diagnosis, but held-out scores participate in the promotion regression gate, so evaluation is not sealed. This entry records only the extracted MiniMax and Qwen terminal changes; GLM's repository dependency-recovery mechanism must not be transferred to this setting." } ], "sources": [ { "name": "Self-Harness (v3)", "url": "https://arxiv.org/html/2606.09498v3", "locator": "§3.3,4.1; Figure3; Appendix A.2" }, { "name": "Self-Harness (v3)", "url": "https://arxiv.org/html/2606.09498v3", "locator": "§4.3 Figures5–8; Appendix B.1–B.2" } ], "benchmarks": [ { "id": "terminal-bench-2-0", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--mixture_branches2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "mixture_branches2026", "title": "Mixture of Self-Improving Branches: checking completion once", "sections": [ { "label": "Setting and seed", "text": "Terminal-Bench 2.0 uses 88 tasks split into 30 development and 58 test tasks. The illustrated candidate's immediate parent is verification-enhanced Terminus-KIRA with an existing tool-execution loop." }, { "label": "Reported evolution", "text": "The first completion signal triggers a verification prompt and sets a one-time flag. Subsequent tool use leaves the flag intact, so a later completion exits without reopening the same verification request." }, { "label": "Evidence and limits", "text": "Appendix E.3 documents a discovered candidate, not a confirmed final head deployed by the router. Its retained state flag bounds repeated checking, but final retention and an isolated effect are unverified. Test-best candidates in the discovery pool must remain distinct from development-selected results. The parent's verification capabilities are inherited rather than new mechanisms." } ], "sources": [ { "name": "Mixture of Self-Improving Branches (v1)", "url": "https://arxiv.org/html/2609.37834v1", "locator": "§4.1; Tables1–4" }, { "name": "Mixture of Self-Improving Branches (v1)", "url": "https://arxiv.org/html/2609.37834v1", "locator": "Appendix E.3–E.4" } ], "benchmarks": [ { "id": "terminal-bench-2-0", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--multitask_selfharness2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "multitask_selfharness2026", "title": "Multi-task Self-Harness: bounded observations and fresh-context review", "sections": [ { "label": "Setting and seed", "text": "Mixed-task evolution includes 60 Terminal-Bench 2.1 tasks, with 29 held out. The seed is a 49-line prompt-and-bash Python loop that accumulates outputs verbatim without compaction or recovery." }, { "label": "Reported evolution", "text": "Iteration 5 retains the beginning and end of long outputs, further shortens old observations, and trims overlong history to its opening and recent messages. Instructions allow recovery of omitted information from files. The first stop decision opens a fresh-context review of the original task and current filesystem." }, { "label": "Evidence and limits", "text": "These changes are reported as retained and reused across tasks, not rediscovered independently for each benchmark. History trimming rarely triggers in coding, and candidates are not screened by validation score. The 150-second/eight-call cap added at iteration 6 is excluded from this iteration-5 account; no individual-change ablation is established." } ], "sources": [ { "name": "Multi-Task Self-Harness (v1)", "url": "https://arxiv.org/html/2609.38372v1", "locator": "§3.2–3.4; §4.1 Table 3; §5.1–5.7 Table 4; Appendix C" } ], "benchmarks": [ { "id": "terminal-bench-2-1", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--rrsi2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "rrsi2026", "title": "RRSI: bounding audits and waiting behavior", "sections": [ { "label": "Setting and seed", "text": "Evolution uses all 89 Terminal-Bench 2.1 tasks, beginning with Terminus-2 before the reported bounded audit and polling guidance. This record concerns candidate R0-A only." }, { "label": "Reported evolution", "text": "R0-A adds a bounded completion audit and nonblocking polling guidance for long-running work. Both are jointly accepted at that iteration, modifying how the agent checks completion and waits for results." }, { "label": "Evidence and limits", "text": "Acceptance at R0 is documented, but survival in the final harness is unknown. The polling mechanism is guidance, not evidence of a scheduler rewrite. Rejected candidates are not included among the evolved features. SWE does not participate in selection, and this within-benchmark acceptance does not establish held-out generalization or separate effects for the two instructions." } ], "sources": [ { "name": "RRSI (v2)", "url": "https://arxiv.org/html/2609.24972v2", "locator": "§4.1,4.3; Appendix C; Appendix D.1 Table 5" }, { "name": "RRSI (v2)", "url": "https://arxiv.org/html/2609.24972v2", "locator": "Appendix E Table 6 (Coding R0-A,R0-B,R8-B)" } ], "benchmarks": [ { "id": "terminal-bench-2-1", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--darwinx2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "darwinx2026", "title": "DarwinX: making delivery requirements explicit in skills", "sections": [ { "label": "Setting and seed", "text": "Evolution and evaluation use the same 89 Terminal-Bench 2.1 tasks. Base Monet starts without the seven later verification/artifact-contract skills." }, { "label": "Reported evolution", "text": "The retained bundle extracts acceptance conditions, checks artifact paths, formats and values, repairs failures and rechecks after changes. It also requires real tool evidence and addresses safety/interface constraints before completion." }, { "label": "Evidence and limits", "text": "The authors describe seven jointly selected skills, not per-skill causal ablations. Their content guides execution rather than demonstrating unavoidable submission gates. TerminalWorld uses a separate train/test protocol, and SWE is a transfer target; neither supplies additional evolution evidence for this terminal entry. Whole-bundle results therefore cannot be assigned independently to each instruction." } ], "sources": [ { "name": "DarwinX (v1)", "url": "https://arxiv.org/html/2608.07545v1", "locator": "§3,§8.1 Table6; Appendix B Table8" } ], "benchmarks": [ { "id": "terminal-bench-2-1", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--rho2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "rho2026", "title": "RHO: organizing terminal work through a requirements playbook", "sections": [ { "label": "Setting and seed", "text": "A terminal-specific optimization uses 30 training and 59 held-out tasks, starting from vanilla Codex with GPT-5.5 high in a configurable workspace. The paper names Terminal-Bench 2 and pins commit 53ff2b87d621 without an established minor-version mapping." }, { "label": "Reported evolution", "text": "A retained playbook records task requirements, organizes host-to-container commands, checks final artifacts at their container paths and revisits the specification before completion. Scripts are generated alongside it." }, { "label": "Evidence and limits", "text": "Behavioral analysis explicitly reports that terminal trajectories do not run those scripts; the operative observed change is guidance. SWE-Bench Pro's executed repair-verify helper must not be assigned here. Cross-benchmark reuse and an individual causal effect are unproven. This setting remains version 2.x, minor version unspecified, rather than being silently assigned to 2.0 or 2.1." } ], "sources": [ { "name": "RHO (v3)", "url": "https://arxiv.org/html/2606.05922v3", "locator": "§5 setup; §6.1; Appendix E.1–E.2" }, { "name": "RHO (v3)", "url": "https://arxiv.org/html/2606.05922v3", "locator": "Appendix H.1–H.2; §6.1" } ], "benchmarks": [ { "id": "terminal-bench-2-x-minor-unspecified", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "terminal-based-engineering-tasks--codeskill2026", "domain": "coding", "category": "Terminal-Based Engineering Tasks", "category_id": "terminal-based-engineering-tasks", "key": "codeskill2026", "title": "CODESKILL: transferring a skill-management policy", "sections": [ { "label": "Setting and seed", "text": "Terminal-Bench 2 is held out from skill-manager policy training and evaluated through mini-SWE-agent with frozen downstream model weights. The paper does not establish a minor version for this record." }, { "label": "Reported evolution", "text": "The learned manager extracts task-level and event-driven instructions and maintains the bank through additions, revisions, merges and removals. Section 4.1 specifies two preparatory passes over the test set before final evaluation: initial extraction, followed by skill-conditioned evolution. During final evaluation, skills whose tracked provenance includes the current task are excluded. Transfer therefore concerns the manager with test-set bank preparation; it is neither a training-only frozen bank nor a demonstrated streaming update after each final-evaluation query." }, { "label": "Evidence and limits", "text": "This supports terminal evaluation coverage and an update mechanism. The published Maven examples demonstrate skill granularity but are not tied to particular terminal tasks or a terminal-specific final harness. Accordingly, no terminal-specific content or individual mechanism gain is inferred from them; policy training, bank evolution and task execution remain separate processes." } ], "sources": [ { "name": "CODESKILL", "url": "https://arxiv.org/abs/2605.25430v2", "locator": "Sections 4.1, 4.3; Appendix C–D" } ], "benchmarks": [ { "id": "terminal-bench-2-x-minor-unspecified", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "machine-learning-engineering--aide2_2026", "domain": "coding", "category": "Machine Learning Engineering", "category_id": "machine-learning-engineering", "key": "aide2_2026", "title": "AIDE²: search diversity and bounded experiment history", "sections": [ { "label": "Setting and seed", "text": "MLE-bench Lite22 is a held-out transfer evaluation of AIDE85, evolved on external tasks. Its AIDE0 ancestor greedily selects the best candidate and accumulates full candidate histories during drafting and improvement." }, { "label": "Reported evolution", "text": "The retained runtime searches multiple strategy arms with periodic cross-arm forks. It summarizes initial and recent solutions, adds selected error signatures when bugs recur, catches nearly empty code and prompts generalization to unseen test data. Search can change strategy while retaining a strong candidate." }, { "label": "Evidence and limits", "text": "Mean private percentile rises from 0.678 ± 0.006 to 0.722 ± 0.011; errors are one standard error. End-of-run median prompt-size reduction reaches 7×. These are whole-runtime transfer results, not module ablations, and AIDE47 scores higher on this benchmark (0.730 ± 0.005). Lite22 never enters evolutionary selection. The ineffective ranking penalty, rejected ensembles and scoring-script repair are not retained task-runtime mechanisms in this account." } ], "sources": [ { "name": "AIDE² (v1)", "url": "https://arxiv.org/html/2609.26457v1", "locator": "§2.1–2.2; §3.2–3.6; Appendix A/Table 2; Appendix D–E" } ], "benchmarks": [ { "id": "mle-bench-lite22", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "record_screened", "selection_protocol": null, "controlled_variables": null, "signals": [ { "id": "whole-runtime-transfer", "outcome": "improvement", "retention": "retained", "attribution": "full_system_comparison", "components": [ "context", "control" ], "benchmark_ids": [ "mle-bench-lite22" ], "scope": "AIDE85 versus AIDE0 on MLE-bench Lite22; not a claim that AIDE85 beats every intermediate checkpoint.", "summary": "The complete retained runtime improves over the ancestor; individual edits are not causally isolated, and AIDE47 scores higher on this benchmark.", "section": "Evidence and limits", "evidence_quote": "These are whole-runtime transfer results, not module ablations, and AIDE47 scores higher on this benchmark (0.730 ± 0.005).", "source_index": 0, "primary_source": { "name": "AIDE² (v1)", "url": "https://arxiv.org/html/2609.26457v1", "locator": "§2.1–2.2; §3.2–3.6; Appendix A/Table 2; Appendix D–E" } } ] }, { "id": "machine-learning-engineering--sia2026", "domain": "coding", "category": "Machine Learning Engineering", "category_id": "machine-learning-engineering", "key": "sia2026", "title": "SIA: turning denoising experiments into structured feedback", "sections": [ { "label": "Setting and seed", "text": "The MAGIC denoising hyperparameter task starts from a Meta-Agent-generated system prompt, single-tool dispatch loop and output parser around gpt-oss-120b. It is task-specific harness iteration, not MLE-bench evaluation." }, { "label": "Reported evolution", "text": "With model weights fixed, the Feedback-Agent adds a batched configuration driver and a parser exposing parameter–score pairs. These reusable tools organize experiments and their feedback; particular neighbor counts, diffusion steps, bandwidths and preprocessing choices are task-level candidates rather than harness changes." }, { "label": "Evidence and limits", "text": "Section 7.2 reports the discoveries but provides no final-version diff or individual-tool ablation, leaving final retention unconfirmed. Table 2 specifies no separate train/test split for this task. Later GRPO updates and postprocessing belong to the weight-update phase and are excluded from this harness-only record." } ], "sources": [ { "name": "SIA (v2)", "url": "https://arxiv.org/html/2605.27276v2", "locator": "§3.2–3.3; §5.3; §6.1 Table 2; §6.2–6.3.3; §7.2" } ], "benchmarks": [ { "id": "magic-denoising", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "machine-learning-engineering--aibuildai2_2026", "domain": "coding", "category": "Machine Learning Engineering", "category_id": "machine-learning-engineering", "key": "aibuildai2_2026", "title": "AIBuildAI-2: carrying experiment takeaways into future runs", "sections": [ { "label": "Setting and seed", "text": "MLE-bench and live competitions use a hierarchical ML agent initialized with a human-authored topical index and roughly 1,000 knowledge documents in about 30 categories." }, { "label": "Reported evolution", "text": "After a task, an L2 builder turns the working directory into a categorized takeaway document. An L1 builder revises the category's instruction and its pointers to L2 documents; the top-level category index remains fixed. A second stream incorporates external material, and future executions retrieve relevant guidance selectively." }, { "label": "Evidence and limits", "text": "The paper specifies this persistent-update procedure but does not expose a complete before/after post-run document with an isolated evaluation. System-level results do not establish the effect of a particular evolved instruction. The trained models or ensembles produced during competitions are task outputs, distinct from changes to the agent's reusable knowledge." } ], "sources": [ { "name": "AIBuildAI-2", "url": "https://arxiv.org/abs/2605.27873v1", "locator": "Figure 1; Methods: Knowledge builder agents and Knowledge system evolution" } ], "benchmarks": [ { "id": "mle-bench", "role": "study" }, { "id": "live-ml-competitions", "role": "setting" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "machine-learning-engineering--disco2026", "domain": "coding", "category": "Machine Learning Engineering", "category_id": "machine-learning-engineering", "key": "disco2026", "title": "DisCo: refining task-oriented research guidance", "sections": [ { "label": "Setting and seed", "text": "MLE-bench uses a fixed Codex runtime and repository-grounded guidance constructed for each competition. The initial plan scopes needed capabilities and collects relevant sources while excluding the original competition webpage and competition-specific content." }, { "label": "Reported evolution", "text": "Diagnostic trials produce logs and results that drive repeated revisions of modeling, preprocessing, training and checking guidance. Final skills are descriptive and exclude runnable training/inference scripts; the resulting graph is frozen for downstream execution." }, { "label": "Evidence and limits", "text": "Appendix A.2.1 separates construction/exploration from the matched running budget. Construction already encounters the task, so this is not unseen-task transfer. The retained object is operational knowledge around a fixed controller, not an evolved Codex runtime or a particular trained model. Reported results do not isolate the contribution of an individual ML rule." } ], "sources": [ { "name": "DisCo", "url": "https://arxiv.org/abs/2609.02749v1", "locator": "Appendix A.2.1" } ], "benchmarks": [ { "id": "mle-bench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "arithmetic-word-problems--adas2024", "domain": "reasoning", "category": "Arithmetic Word Problems", "category_id": "arithmetic-word-problems", "key": "adas2024", "title": "ADAS: checking an intermediate representation", "sections": [ { "label": "Setting and seed", "text": "Search on MGSM starts from an archive containing supplied agent designs, rather than a single named seed." }, { "label": "Discovered behavior", "text": "The example named Verified Multimodal Agent generates a representation of the problem, passes it to a second module for feedback and revision, and conditions the final reasoning module on that revised representation. The listing passes textual fields; the name alone does not establish an image-processing pipeline." }, { "label": "Outcome and scope", "text": "This is a documented discovered agent, not the paper's highest-scoring named agent. Table 2 separately evaluates the top three MGSM agents on GSM8K and GSM-Hard; Appendix A adds SVAMP and ASDiv. Those transfer scores cannot be assigned to Code 6's mechanism." } ], "sources": [ { "name": "ADAS", "url": "https://proceedings.iclr.cc/paper_files/paper/2025/file/36b7acf6f6010652b3f2a433774a66fe-Paper-Conference.pdf", "locator": "Sections 3, 4.2–4.3; Appendix G, Code 6" } ], "benchmarks": [ { "id": "mgsm", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "arithmetic-word-problems--aflow2024", "domain": "reasoning", "category": "Arithmetic Word Problems", "category_id": "arithmetic-word-problems", "key": "aflow2024", "title": "AFlow: selecting and checking candidate answers", "sections": [ { "label": "Setting and seed", "text": "GSM8K search starts from an empty workflow template with supplied generation, selection and programming operators; their later call order is not part of the seed." }, { "label": "Discovered behavior", "text": "The selected program generates five solutions, selects with ScEnsemble, and invokes Programmer for a computational review, falling back to the selected solution if no output is produced." }, { "label": "Outcome and scope", "text": "Figure 6 traces accepted additions and prompt refinements. The operators are supplied; their composition is searched. The reported workflow is retained, but its aggregate score does not isolate each addition." } ], "sources": [ { "name": "AFlow", "url": "https://proceedings.iclr.cc/paper_files/paper/2025/file/5492ecbce4439401798dcd2c90be94cd-Paper-Conference.pdf", "locator": "Section 4; Figures 5–6" } ], "benchmarks": [ { "id": "gsm8k", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "arithmetic-word-problems--mermaidflow2025", "domain": "reasoning", "category": "Arithmetic Word Problems", "category_id": "arithmetic-word-problems", "key": "mermaidflow2025", "title": "MermaidFlow: computational candidates and final formatting", "sections": [ { "label": "Setting and artifact", "text": "The published GSM8K workflow combines one textual solution with five Programmer outputs produced under different analysis instructions, selects with ScEnsemble, then invokes Programmer to refine and format the selected answer." }, { "label": "Comparison and limits", "text": "Relative to AFlow's post-selection computational review, this artifact places computational generation before selection as well. The listing supports this composition, not a claim that all calls execute in parallel or that each added call improves accuracy independently." } ], "sources": [ { "name": "MermaidFlow", "url": "https://arxiv.org/abs/2505.22967v1", "locator": "Appendix B.1" } ], "benchmarks": [ { "id": "gsm8k", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "arithmetic-word-problems--flowevo2026", "domain": "reasoning", "category": "Arithmetic Word Problems", "category_id": "arithmetic-word-problems", "key": "flowevo2026", "title": "FlowEvo: carrying useful procedures across problems", "sections": [ { "label": "Settings and update", "text": "GSM8K and MATH-500 evaluate a workflow generator augmented with a persistent skill bank. Successful procedures are compiled and can be retrieved as context for later workflow generation." }, { "label": "Evidence and limits", "text": "The paper identifies skill-conditioned generation as the relevant reuse mode for code and math. It does not provide the same detailed mathematical skill lineage as its ALFWorld example. We therefore record a supported accumulation mechanism and mathematical evaluation, rather than inventing a particular evolved mathematical tactic or attributing ALFWorld's replay gains to mathematics." } ], "sources": [ { "name": "FlowEvo", "url": "https://openreview.net/forum?id=hU2N7IIkcE", "locator": "Sections 3–4; Appendix D.3" } ], "benchmarks": [ { "id": "gsm8k", "role": "study" }, { "id": "math-500", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--aflow2024", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "aflow2024", "title": "AFlow: combining programmatic and textual solutions", "sections": [ { "label": "Documented artifact", "text": "From the same template family, a separate MATH workflow executes a program, rewrites its output as a solution, generates three textual alternatives, and selects among the four with ScEnsemble. This is the published selected workflow for the level-5 setting, not a frozen transfer of its GSM8K procedure." } ], "sources": [ { "name": "AFlow", "url": "https://proceedings.iclr.cc/paper_files/paper/2025/file/5492ecbce4439401798dcd2c90be94cd-Paper-Conference.pdf", "locator": "Appendix B.1, MATH workflow, PDF p. 21" } ], "benchmarks": [ { "id": "math-level-5", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--metaharness2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "metaharness2026", "title": "Meta-Harness: subject-dependent retrieval", "sections": [ { "label": "Setting and seed", "text": "Search uses 250 OlympiadBench / hard Omni-MATH problems and zero-shot, few-shot, and ACE initializations." }, { "label": "Discovered behavior", "text": "The selected BM25 harness routes by mathematical subject. Combinatorics deduplicates and reranks examples; geometry combines a hard reference with lexical neighbors; number theory favors explicit techniques; the default route adjusts example count. Two search lineages contribute to the merged policy." }, { "label": "Outcome and scope", "text": "A frozen harness is evaluated on 200 unseen IMO-AnswerBench, IMO-ProofBench, and ArXivMath problems: average accuracy is 38.8%, versus 34.1% without retrieval. Evaluation includes the search backbone and four unseen models. These are whole-policy results, not independent ablations of routing rules or formally checked proofs." } ], "sources": [ { "name": "Meta-Harness", "url": "https://openreview.net/forum?id=tmbOUyFx3R", "locator": "Section 4.2; Appendix B.2; Appendix C.3" } ], "benchmarks": [ { "id": "olympiadbench", "role": "study" }, { "id": "omni-math", "role": "study" }, { "id": "imo-answerbench", "role": "transfer" }, { "id": "imo-proofbench", "role": "transfer" }, { "id": "arxivmath", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--coharness2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "coharness2026", "title": "Co-Harness: repairing execution before adding diversity", "sections": [ { "label": "Setting and seed", "text": "The reported AIME24 case starts from a failing code-interpreter configuration." }, { "label": "Discovered behavior", "text": "Changes repair launch settings and process cleanup, batch inference, restore thinking on follow-up turns, and combine trajectories from two seeds by majority vote. Later prompt changes regress and are rolled back." }, { "label": "Outcome and scope", "text": "The final ensemble is reported within the time budget. We do not reproduce the claimed speedup: the stated ratio disagrees with the accompanying times. Appendix H also describes 90 questions despite the AIME24 label. These ambiguities limit quantitative interpretation, while the sequence of runtime edits is explicit. The separate AIME24/25 and HMMT25 co-evolution results also include model training." } ], "sources": [ { "name": "Co-Harness", "url": "https://openreview.net/forum?id=qDC8LmkpSR", "locator": "Section 4.3; Appendix H" } ], "benchmarks": [ { "id": "aime24-co-harness-reported-setting", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--harnessbank2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "harnessbank2026", "title": "HarnessBank: matching recovery to the backbone", "sections": [ { "label": "Setting and seed", "text": "EvoAgentBench's Omni-MATH setting uses a study-specific vanilla harness and disjoint training/test tasks." }, { "label": "Discovered behavior", "text": "For the Qwen backbone, recovery disables thinking after a runaway rather than on every call. The evolved stack transfers to another Qwen model, but does not help Gemini's different failure pattern; its matched intervention increases reasoning instead." }, { "label": "Outcome and scope", "text": "Table 1 reports test success rising from 54.3% to 66.0% for the selected Qwen harness. Cross-model results support a failure-dependent intervention, not a universal setting. The full stack's gain is not attributed to the recovery switch alone." } ], "sources": [ { "name": "HarnessBank", "url": "https://arxiv.org/abs/2607.13683v2", "locator": "Sections 4.1–4.2, 4.6; Tables 1–2" } ], "benchmarks": [ { "id": "evoagentbench-omni-math", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--mermaidflow2025", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "mermaidflow2025", "title": "MermaidFlow: diversified solvers followed by refinement", "sections": [ { "label": "Setting and artifact", "text": "The published MATH workflow obtains five Custom responses under distinct solver/outline instructions and one Programmer result, selects with ScEnsemble, and sends the selection to a refinement call. Its reusable object is the operator graph and associated prompts." }, { "label": "Comparison and limits", "text": "Like AFlow, it combines textual and programmatic candidates, but the supplied composition and final refinement differ. The shared use of AFlow-derived operators makes this a related design lineage, not a fully independent invention of those operators." } ], "sources": [ { "name": "MermaidFlow", "url": "https://arxiv.org/abs/2505.22967v1", "locator": "Appendix B.2" } ], "benchmarks": [ { "id": "math", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--mixture_branches2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "mixture_branches2026", "title": "Mixture of Self-Improving Branches: constructive checks and disagreement resolution", "sections": [ { "label": "Setting and search", "text": "Olympiad-level math uses Meta-Harness-based branch search with changing development subsets and proposal guidance. The reported Math–Gemini branches differentiate concrete answer checking and proof-obligation routing from solution construction and agreement challenges." }, { "label": "Concrete candidate", "text": "Appendix E.1's gap_directed_construction retrieves worked examples using a plan, rehearses a reference to produce a calibration note, and separates constructing an attainable answer from deriving necessary conditions. Agreement triggers a challenge; disagreement triggers a concrete check before adjudication." }, { "label": "Selection limit", "text": "Appendix E describes retrospectively test-best candidates from a development-ranked pool. Its 58% candidate score is not the score of the development-selected deployed head. The mechanism is documented, but those selection protocols are not interchangeable." } ], "sources": [ { "name": "Mixture of Self-Improving Branches", "url": "https://arxiv.org/abs/2609.37834v1", "locator": "Section 4.3, Figure 2; Appendix E.1–E.2" } ], "benchmarks": [ { "id": "olympiad-math-mixture-study", "role": "setting" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--wda2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "wda2026", "title": "WDA: distributing refinement across selected stages", "sections": [ { "label": "Setting and seed", "text": "LiveBench Math extends this category to challenging informal mathematical reasoning, rather than a named olympiad. Search starts from a minimal solver prompt and can revise instructions or split them across stages." }, { "label": "Selected structure", "text": "The published Qwen search selects two stages, [P11, P12], while the GPT search selects three, [P2, P3, P10], with their ancestries reported separately. These are fixed-per-task pipelines, not a runtime policy choosing depth for each question." }, { "label": "Outcome and limits", "text": "The reported selected test scores are 68.85% and 63.93%, respectively. They support selected pipeline structures, not a universal benefit from longer refinement or an isolated effect of any one stage." } ], "sources": [ { "name": "WDA", "url": "https://arxiv.org/abs/2609.32458v1", "locator": "Section 5; Appendices A.3 and A.13" } ], "benchmarks": [ { "id": "livebench-math", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "competition-and-advanced-mathematics--whale2026", "domain": "reasoning", "category": "Competition and Advanced Mathematics", "category_id": "competition-and-advanced-mathematics", "key": "whale2026", "title": "WHALE: making tool use effective through joint updates", "sections": [ { "label": "Setting and seed", "text": "A study-specific tool-integrated math harness and Qwen3.5-2B are optimized using DAPO-Math-17K subsets and tested on AIME 2024/2025." }, { "label": "Observed contrast", "text": "The Appendix D.2 base trace never uses the interpreter and exhausts its answer budget, whereas the final model–harness pair solves the same counting problem. Harness-only search has little effect from the initial pair; alternating weight updates and harness search improves the reported results." }, { "label": "Attribution limit", "text": "This is evidence about a co-evolved pair and its behavior, not a final harness-only diff proving which change caused tool use. It complements fixed-backbone harness studies but cannot be pooled with them as an independent pure-harness gain." } ], "sources": [ { "name": "WHALE", "url": "https://arxiv.org/abs/2609.00196v1", "locator": "Sections 4–5; Appendices B and D.2" } ], "benchmarks": [ { "id": "dapo-math-17k", "role": "study" }, { "id": "aime-2024", "role": "evaluation" }, { "id": "aime-2025", "role": "evaluation" } ], "evidence_status": "Joint model–harness evidence", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "formal-theorem-proving--lean_coevolution2026", "domain": "reasoning", "category": "Formal Theorem Proving", "category_id": "formal-theorem-proving", "key": "lean_coevolution2026", "title": "Self-modifying Lean agents: bounded repair and checked names", "sections": [ { "label": "Setting and seed", "text": "A single-attempt/check seed evolves on warm-up lemmas, miniF2F-valid and PutnamBench tasks." }, { "label": "Discovered behavior", "text": "The accepted lineage adds bounded compiler-guided repair, batched lemma-name checks, and verified signatures. On unknown-name or type errors, the workflow can exclude a bad name, return a checked alternative and retry. The c141 branch plans and proves auxiliary lemmas before attempting assembly, but mismatched lemma contexts, assembly failures and longer-prompt risks prevent this richer structure from becoming the strongest route. The winning c144 remains mainly a whole-proof repair workflow; its proof-context labels do not establish a fully verified dependency graph." }, { "label": "Outcome and scope", "text": "On 244 held-out miniF2F-test problems, c144 reaches 45.1%, versus 12.7% for the seed. This comparison includes curriculum coevolution and multiple edits; it does not isolate the lemma tools or demonstrate unrestricted proof-graph improvement." } ], "sources": [ { "name": "Self-modifying Lean agents", "url": "https://arxiv.org/abs/2607.17352v1", "locator": "Sections 3.1, 4.3–4.5; Appendices A, C–D" } ], "benchmarks": [ { "id": "minif2f-valid", "role": "study" }, { "id": "putnambench", "role": "study" }, { "id": "minif2f-test", "role": "evaluation" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "formal-theorem-proving--rsiexam_lean2026", "domain": "reasoning", "category": "Formal Theorem Proving", "category_id": "formal-theorem-proving", "key": "rsiexam_lean2026", "title": "RSI-Exam: proof extraction and use of limited checks", "sections": [ { "label": "Setting and seed", "text": "The fixed-model seed proposes a proof, checks it, and repairs once; each theorem permits three model calls and four Lean checks." }, { "label": "Observed revisions", "text": "GPT-5.6-sol and GLM 5.3 optimizer runs fix extraction that truncated nested proofs; other revisions add compiler repairs or alter diagnostics. These are optimizer identities, not the pinned task model. In the GLM run, visible accuracy rises from 8/54 to 13/54 after extraction repair and to 20/54 after a combined diagnostic/prompt change; later variants regress." }, { "label": "Scope", "text": "These are version-level observations from one official resource, not independent papers. Visible checkpoint scores are not hidden-test gains. Final retention of each component is not established by the summary alone." } ], "sources": [ { "name": "RSI-Exam", "url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html", "locator": "Lean task contract and public rollouts; accessed 2026-10-05" } ], "benchmarks": [ { "id": "rsi-exam-lean", "role": "study" } ], "evidence_status": "Benchmark-hosted observation", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "formal-theorem-proving--proverinterface2026", "domain": "reasoning", "category": "Formal Theorem Proving", "category_id": "formal-theorem-proving", "key": "proverinterface2026", "title": "Evolved prover interfaces: from compilation to interactive control", "sections": [ { "label": "Seed and retained changes", "text": "Evolution starts from a Rocq MCP server exposing whole-file compilation. Retained additions include a persistent session with step/rollback/state interaction, a try tool that tests up to eight candidates and commits the first success, and error hints. A heavily used lemma-search proposal yields no accuracy gain and is rejected; exemplar retrieval is also rejected. File-manipulation and project-verification tools added for Phase 2 are supplied setup changes, not empirically selected mutations." }, { "label": "Outcome and limits", "text": "Held-out miniF2F-Rocq evaluation compares the resulting interface across four models. Its Lean port lowers cost/time per solve on a PutnamBench subset, but solves fewer problems than lean-lsp-mcp. Human setup and review remain; this is bounded interface evolution, not autonomous improvement of every system component." } ], "sources": [ { "name": "Evolved prover interfaces", "url": "https://arxiv.org/abs/2609.39544v3", "locator": "Sections 2–5; Tables 1 and 8" } ], "benchmarks": [ { "id": "minif2f-rocq", "role": "evaluation" }, { "id": "putnambench-lean-interface-port", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "record_screened", "selection_protocol": null, "controlled_variables": null, "signals": [ { "id": "rejected-lemma-search", "outcome": "no_gain", "retention": "rejected", "attribution": "reported_proposal_test", "components": [ "tools" ], "benchmark_ids": [], "scope": "Lemma-search proposal in interface evolution; no benchmark-specific effect size assigned.", "summary": "Frequent use does not establish benefit: the lemma-search proposal is rejected without an accuracy gain.", "section": "Seed and retained changes", "evidence_quote": "A heavily used lemma-search proposal yields no accuracy gain and is rejected; exemplar retrieval is also rejected.", "source_index": 0, "primary_source": { "name": "Evolved prover interfaces", "url": "https://arxiv.org/abs/2609.39544v3", "locator": "Sections 2–5; Tables 1 and 8" } }, { "id": "lean-port-tradeoff", "outcome": "mixed", "retention": "not_assessed", "attribution": "full_system_comparison", "components": [ "tools", "control" ], "benchmark_ids": [ "putnambench-lean-interface-port" ], "scope": "Lean port on the reported PutnamBench subset, compared with lean-lsp-mcp.", "summary": "Lower cost/time per solve coexists with fewer solved problems; this is not uniformly better performance.", "section": "Outcome and limits", "evidence_quote": "Its Lean port lowers cost/time per solve on a PutnamBench subset, but solves fewer problems than lean-lsp-mcp.", "source_index": 0, "primary_source": { "name": "Evolved prover interfaces", "url": "https://arxiv.org/abs/2609.39544v3", "locator": "Sections 2–5; Tables 1 and 8" } } ] }, { "id": "formal-theorem-proving--proofevolve2026", "domain": "reasoning", "category": "Formal Theorem Proving", "category_id": "formal-theorem-proving", "key": "proofevolve2026", "title": "ProofEvolve: verified resources that persist across problems", "sections": [ { "label": "Seed and change", "text": "The persistent schema library grows through extraction of kernel-checked subproofs; typed recombination exposes remaining premises as subgoals. This component-level accumulation is distinct from evolving a single target proof." }, { "label": "Reuse evidence", "text": "A controlled compositional study compares growing and reset libraries. A separate LeanWorkbook experiment retrieves self-generated verified proofs from disjoint source theorems, improving target success over no retrieval; random retrieval provides little benefit. That experiment supplies context for one whole-proof attempt, rather than testing typed DAG recombination. Its gain must not be attributed to every component of the full prover." } ], "sources": [ { "name": "ProofEvolve", "url": "https://arxiv.org/abs/2608.26334v1", "locator": "Sections 3 and 5.6" } ], "benchmarks": [ { "id": "leanworkbook", "role": "study" }, { "id": "compositional-proof-study", "role": "setting" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "legal-charge-classification--metaharness2026", "domain": "reasoning", "category": "Legal Charge Classification", "category_id": "legal-charge-classification", "key": "metaharness2026", "title": "Meta-Harness", "sections": [ { "label": "Setting and seed", "text": "LawBench charge prediction. Zero-shot, few-shot, ACE and MCE initializations." }, { "label": "Reported evolution", "text": "The published classification frontier includes a two-call draft verifier: after an initial prediction from five similar examples, it retrieves five same-label confirmers and five different-label challengers, then asks whether to revise the draft. With fewer than five stored examples it falls back to a single few-shot call. The selected label-primed program instead combines one query-relevant example per label with nearby contrasting pairs in one prompt. These are alternative frontier programs, not successive stages of one final classifier." }, { "label": "Evidence and limits", "text": "These are shared classification programs evaluated across datasets, not a separately discovered legal decision procedure. Search-set selection precedes held-out evaluation." } ], "sources": [ { "name": "Meta-Harness", "url": "https://openreview.net/forum?id=tmbOUyFx3R", "locator": "Section 4.1; Appendix B.1" } ], "benchmarks": [ { "id": "lawbench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "legal-charge-classification--mce2026", "domain": "reasoning", "category": "Legal Charge Classification", "category_id": "legal-charge-classification", "key": "mce2026", "title": "MCE", "sections": [ { "label": "Setting and seed", "text": "LawBench. A fixed classifier receives a task prompt and the output of a query-to-context function; Algorithm 1 initializes learned context as empty. The initial pattern-extraction skill belongs to the context-engineering agent, not the classifier's execution harness." }, { "label": "Reported evolution", "text": "The learned target-side context directs full-case reading, extraction of discrete acts, act-to-charge matching and validation. Its retrieval program always supplies format rules and the decomposition framework, adding warnings when structural markers suggest multiple acts. The separate skill that teaches the context engineer how to construct this artifact is an optimizer-side change." }, { "label": "Evidence and limits", "text": "The task-agent evidence is the reusable context and query-dependent retrieval program in Appendix D.4.3. The optimizer skill in D.4.2 is a distinct object. Neither individual guidance lines nor retrieval triggers are independently ablated; the LawBench configuration differs from other studies." } ], "sources": [ { "name": "MCE", "url": "https://openreview.net/forum?id=P1jHroBS5E", "locator": "Sections 3.1–3.4 and 4.1; Appendix D.4.2–D.4.3" } ], "benchmarks": [ { "id": "lawbench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "legal-charge-classification--taskcoevolve2026", "domain": "reasoning", "category": "Legal Charge Classification", "category_id": "legal-charge-classification", "key": "taskcoevolve2026", "title": "Task-CoEvolve", "sections": [ { "label": "Setting and seed", "text": "LawBench, within the three-dataset classification search. Zero-shot and all-example few-shot harnesses in the Meta-Harness implementation." }, { "label": "Reported evolution", "text": "The selected 20-percent-budget harness fuses word-bigram and character n-gram TF-IDF rankings, then downweights labels already represented while filling a bounded context." }, { "label": "Evidence and limits", "text": "One shared classifier serves Law, S2D and USPTO. The proposed benefit of word phrases for legal cases is an explanatory hypothesis, not a separately ablated legal mechanism." } ], "sources": [ { "name": "Task-CoEvolve", "url": "https://arxiv.org/abs/2608.20169v2", "locator": "Experimental setup; Appendix D.1" } ], "benchmarks": [ { "id": "lawbench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "legal-charge-classification--drevo2026", "domain": "reasoning", "category": "Legal Charge Classification", "category_id": "legal-charge-classification", "key": "drevo2026", "title": "DREvo", "sections": [ { "label": "Setting and seed", "text": "Law. No-memory and all-example few-shot configurations around GPT-OSS-120B." }, { "label": "Reported evolution", "text": "The proposer recalibrates historical evidence against current component state and converts it into targeted edit intent; the study evaluates the selected evolved classifier on held-out legal examples." }, { "label": "Evidence and limits", "text": "The reviewed primary text does not expose a legal-specific final program diff. We count the reported evolution experiment, not a new legal reasoning rule or the optimizer design as a learned task mechanism." } ], "sources": [ { "name": "DREvo", "url": "https://arxiv.org/abs/2607.26722v2", "locator": "Experimental Setups; Overall Performance; Robustness Analysis" } ], "benchmarks": [ { "id": "lawbench", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "symptom-to-disease-classification--metaharness2026", "domain": "reasoning", "category": "Symptom-to-Disease Classification", "category_id": "symptom-to-disease-classification", "key": "metaharness2026", "title": "Meta-Harness", "sections": [ { "label": "Setting and seed", "text": "Symptom2Disease (22 labels). The same classification seed family, evaluated on symptom descriptions." }, { "label": "Reported evolution", "text": "Label-conditioned retrieval and query-local contrasting examples provide reusable alternatives to placing the complete accumulated history in context." }, { "label": "Evidence and limits", "text": "The same frontier programs described above are reused across classification datasets; this is not a second independent discovery. Held-out classification accuracy does not establish clinical validity." } ], "sources": [ { "name": "Meta-Harness", "url": "https://openreview.net/forum?id=tmbOUyFx3R", "locator": "Section 4.1; Appendix B.1" } ], "benchmarks": [ { "id": "symptom2disease", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "symptom-to-disease-classification--mce2026", "domain": "reasoning", "category": "Symptom-to-Disease Classification", "category_id": "symptom-to-disease-classification", "key": "mce2026", "title": "MCE", "sections": [ { "label": "Setting and seed", "text": "Symptom2Disease. The same fixed-model query-to-context interface starts without learned context in Algorithm 1. The initial symptom-profile skill guides the context engineer; it is not itself the target classifier's prompt or tool." }, { "label": "Reported evolution", "text": "The evolved target-side context contains distinctions between commonly confused labels. Its query-to-context program detects symptom patterns and returns relevant discriminator sections, falling back to the full guide if no condition matches. Separately, the optimizer's context-construction skill shifts from generalized profiles toward conservative evidence-backed additions and stopping criteria; those skill instructions are not executed by the target classifier." }, { "label": "Evidence and limits", "text": "Appendix D.3.3 exposes reusable classifier context and retrieval examples, but does not isolate their individual effects. The evidence and stopping thresholds in D.3.2 belong to the context optimizer's skill. They are neither target-runtime guards nor general diagnostic guidance." } ], "sources": [ { "name": "MCE", "url": "https://openreview.net/forum?id=P1jHroBS5E", "locator": "Sections 3.1–3.4 and 4.1; Appendix D.3.2–D.3.3" } ], "benchmarks": [ { "id": "symptom2disease", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "symptom-to-disease-classification--taskcoevolve2026", "domain": "reasoning", "category": "Symptom-to-Disease Classification", "category_id": "symptom-to-disease-classification", "key": "taskcoevolve2026", "title": "Task-CoEvolve", "sections": [ { "label": "Setting and seed", "text": "Symptom2Disease. Zero-shot and all-example few-shot seeds." }, { "label": "Reported evolution", "text": "Its selected shared classifier combines word and character retrieval and penalizes repeated-label coverage in the prompt." }, { "label": "Evidence and limits", "text": "S2D has a separate held-out test set, but the artifact is shared with the legal and reaction-classification settings. The tokenizer explanation is not a clinical causal result." } ], "sources": [ { "name": "Task-CoEvolve", "url": "https://arxiv.org/abs/2608.20169v2", "locator": "Experimental setup; Appendix D.1" } ], "benchmarks": [ { "id": "symptom2disease", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "symptom-to-disease-classification--drevo2026", "domain": "reasoning", "category": "Symptom-to-Disease Classification", "category_id": "symptom-to-disease-classification", "key": "drevo2026", "title": "DREvo", "sections": [ { "label": "Setting and seed", "text": "Symptom2Disease. No-memory and all-example few-shot configurations around GPT-OSS-120B." }, { "label": "Reported evolution", "text": "Validation-guided evolution with recalibrated historical evidence produces a selected classifier evaluated on a separate S2D test split." }, { "label": "Evidence and limits", "text": "The experiment establishes task coverage; the reviewed text does not specify the final S2D retrieval implementation. Component-drift robustness is evidence about the optimizer, not a retained classifier feature." } ], "sources": [ { "name": "DREvo", "url": "https://arxiv.org/abs/2607.26722v2", "locator": "Experimental Setups; Overall Performance; Robustness Analysis" } ], "benchmarks": [ { "id": "symptom2disease", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "symptom-to-disease-classification--hase2026", "domain": "reasoning", "category": "Symptom-to-Disease Classification", "category_id": "symptom-to-disease-classification", "key": "hase2026", "title": "HASE", "sections": [ { "label": "Setting and seed", "text": "Symptom2Disease. A recency-based few-shot retrieval tool and initially empty memory." }, { "label": "Reported evolution", "text": "Reported tool revisions add symptom-overlap or stronger lexical retrieval; across phases, tools and memory incorporate symptom extraction and label-grouped diagnostic indicators." }, { "label": "Evidence and limits", "text": "Training-time repair examples permit a second attempt, whereas phase-boundary review and final evaluation freeze committed files. Joint weight updates preclude interpreting the full gain as harness-only improvement." } ], "sources": [ { "name": "HASE", "url": "https://arxiv.org/abs/2607.03935v1", "locator": "Section 4.2.1; Appendices A.1, A.4 and E" } ], "benchmarks": [ { "id": "symptom2disease", "role": "study" } ], "evidence_status": "Joint model–harness evidence", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "qa-over-supplied-records--evolvemem2026", "domain": "reasoning", "category": "QA over Supplied Records", "category_id": "qa-over-supplied-records", "key": "evolvemem2026", "title": "EvolveMem", "sections": [ { "label": "Setting and seed", "text": "LoCoMo; MemBench. A BM25-only configuration around a fixed answering model, with semantic/structured views, name-stripped alternatives and query decomposition disabled." }, { "label": "Reported evolution", "text": "Failure-log diagnosis activates name-stripped alternative retrieval and multi-hop query decomposition, then introduces a verification pass for low-confidence answers. The first two are described as dormant configuration capabilities rather than newly authored tools. Guarded updates can revert regressions; the evolved object is the retrieval/answer configuration." }, { "label": "Evidence and limits", "text": "Section 4.5 removes components and reruns evolution, providing procedure-level ablation evidence rather than a fixed-checkpoint edit test. Per-question failure logs provide supervision, so this is not label-free adaptation. Table 4 and Appendix C.1 present different round-specific configurations whose alignment is unverified here; we do not merge them into one quantitative trajectory." } ], "sources": [ { "name": "EvolveMem", "url": "https://arxiv.org/abs/2605.13941v1", "locator": "Sections 4.1, 4.3 and 4.5; Appendix C.1" } ], "benchmarks": [ { "id": "locomo", "role": "study" }, { "id": "membench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "qa-over-supplied-records--memcodex2026", "domain": "reasoning", "category": "QA over Supplied Records", "category_id": "qa-over-supplied-records", "key": "memcodex2026", "title": "MemCodex", "sections": [ { "label": "Setting and seed", "text": "LoCoMo; LongMemEval; MemoryAgentBench; PersonaMind. Initial hierarchical memory architecture g0 with executable layer methods." }, { "label": "Reported evolution", "text": "Evolution rewrites construction, indexing, retrieval and routing programs within the supplied hierarchy. Its initial architecture already supports threshold-based adaptive reading; the evolved artifact changes those methods rather than introducing adaptive depth from scratch." }, { "label": "Evidence and limits", "text": "Initial hierarchy design is not itself discovered. The study separately analyzes question-level and record-disjoint splits, acceptance gates and cross-benchmark transfer; no universal optimal depth follows." } ], "sources": [ { "name": "MemCodex", "url": "https://arxiv.org/abs/2609.39765v1", "locator": "Sections 2, 4.2–4.3; Figure 5" } ], "benchmarks": [ { "id": "locomo", "role": "study" }, { "id": "longmemeval", "role": "study" }, { "id": "memoryagentbench", "role": "study" }, { "id": "personamind", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "qa-over-supplied-records--harnessevolve2026", "domain": "reasoning", "category": "QA over Supplied Records", "category_id": "qa-over-supplied-records", "key": "harnessevolve2026", "title": "HarnessEvolve", "sections": [ { "label": "Setting and seed", "text": "OfficeQA. OpenClaw with initial skills, AGENTS.md, SOUL.md and tools." }, { "label": "Reported evolution", "text": "Figure 3(a) expands an existing financial-metrics skill: fix the statistic and operands before computing, distinguish changes in shares from growth rates, specify the denominator, convert log returns when the requested quantity requires it, and check constants and revised values against the corpus. These are reusable instructions, not enforced executable checks. The old skill already requests corpus-grounded currency conversion; that inherited instruction is not a new discovery. Selected skills are subsequently tested in other execution frameworks." }, { "label": "Evidence and limits", "text": "Appendix A, Figure 3(a) supplies the inspected before/after artifact; the individual rules are not separately ablated. Framework transfer reuses learned skills rather than adding independent discoveries, and the illustration alone does not establish each line's presence in the best final checkpoint." } ], "sources": [ { "name": "HarnessEvolve", "url": "https://arxiv.org/abs/2609.00829v1", "locator": "Section 4.1; Tables 2–3; Appendix Figure 3(a)" } ], "benchmarks": [ { "id": "officeqa", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "qa-over-supplied-records--hcl2026", "domain": "reasoning", "category": "QA over Supplied Records", "category_id": "qa-over-supplied-records", "key": "hcl2026", "title": "HCL", "sections": [ { "label": "Setting and seed", "text": "MuSiQue and HotpotQA within a textual task stream. A frozen model with editable task interface, memory, capabilities and routing." }, { "label": "Reported evolution", "text": "Sequential harness updates are tested on current tasks and historical anchors; the study contrasts strict retention with permissive adaptation." }, { "label": "Evidence and limits", "text": "Separate adaptation, validation and test examples support a continual-evolution experiment, but no final QA-specific executable diff is exposed in this record." } ], "sources": [ { "name": "HCL", "url": "https://arxiv.org/abs/2608.19013v2", "locator": "Sections 4.1 and 4.3.1; Appendix C.1" } ], "benchmarks": [ { "id": "musique", "role": "study" }, { "id": "hotpotqa", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "open-web-information-seeking--memevolve2025", "domain": "reasoning", "category": "Open-Web Information Seeking", "category_id": "open-web-information-seeking", "key": "memevolve2025", "title": "MemEvolve", "sections": [ { "label": "Setting and seed", "text": "TaskCraft; GAIA; WebWalkerQA; xBench-DeepSearch. AgentKB or a simple few-shot experience memory." }, { "label": "Reported evolution", "text": "Search changes memory encoding, retrieval and management programs. From a lightweight AgentKB-style seed without its offline knowledge base, Cerebra adds reusable tool and knowledge distillation and working-memory maintenance; other reported branches include Riva and Lightweight." }, { "label": "Evidence and limits", "text": "WebWalkerQA and xBench use architectures evolved on TaskCraft. Their evaluations are frozen transfer evidence, not independent in-benchmark evolution runs." } ], "sources": [ { "name": "MemEvolve", "url": "https://openreview.net/forum?id=qpkG0eKx4v", "locator": "Section 4; Section 9.1; Figures 8–10" } ], "benchmarks": [ { "id": "taskcraft", "role": "study" }, { "id": "gaia", "role": "study" }, { "id": "webwalkerqa", "role": "transfer" }, { "id": "xbench-deepsearch", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "open-web-information-seeking--harnessx2026", "domain": "reasoning", "category": "Open-Web Information Seeking", "category_id": "open-web-information-seeking", "key": "harnessx2026", "title": "HarnessX", "sections": [ { "label": "Setting and seed", "text": "GAIA. HarnessX task configuration with prompts, processors, slots and tool resources." }, { "label": "Reported evolution", "text": "The GAIA evolution analysis reports shipped prompt, processor, configuration and tool edits, addressing blocked sources alongside reasoning and parsing failures." }, { "label": "Evidence and limits", "text": "Lever-level statistics do not identify one universally retained retrieval tool. The same task pool is evaluated during search; this supports benchmark adaptation rather than a sealed generalization test." } ], "sources": [ { "name": "HarnessX", "url": "https://arxiv.org/abs/2606.14249v3", "locator": "Experimental setup; Appendix 12.1" } ], "benchmarks": [ { "id": "gaia", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "open-web-information-seeking--liteevo2026", "domain": "reasoning", "category": "Open-Web Information Seeking", "category_id": "open-web-information-seeking", "key": "liteevo2026", "title": "LiteEvo", "sections": [ { "label": "Setting and seed", "text": "GAIA. A neutral observation–action prompt, default runner settings and no custom processors." }, { "label": "Reported evolution", "text": "Trajectory analysis proposes prompt fragments, knowledge, processors and settings; replay removes inert or invalid processors before the composer assembles candidates." }, { "label": "Evidence and limits", "text": "GAIA is evaluated transductively. Appendix C.4 documents answer-copy exposure despite blocking known mirrors, including a returned processor's guidance about dataset mirrors. The score does not establish clean unseen-question generalization." } ], "sources": [ { "name": "LiteEvo", "url": "https://arxiv.org/abs/2609.33146v1", "locator": "Sections 3.1–4; Appendix C.4" } ], "benchmarks": [ { "id": "gaia", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "open-web-information-seeking--harnessevolve2026", "domain": "reasoning", "category": "Open-Web Information Seeking", "category_id": "open-web-information-seeking", "key": "harnessevolve2026", "title": "HarnessEvolve", "sections": [ { "label": "Setting and seed", "text": "SearchQA. OpenClaw with an unoptimized skill and tool configuration." }, { "label": "Reported evolution", "text": "Failed executions and validated answer-assisted reference trajectories drive persistent skill-directory and harness updates. The resulting SearchQA skills are frozen for cross-framework evaluation." }, { "label": "Evidence and limits", "text": "The SearchQA scores and transfer establish task-level evolution evidence, but the OfficeQA or enterprise SQL example must not be relabeled as a SearchQA-specific final edit." } ], "sources": [ { "name": "HarnessEvolve", "url": "https://arxiv.org/abs/2609.00829v1", "locator": "Sections 3–4; Tables 2–3" } ], "benchmarks": [ { "id": "searchqa", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "browser-mediated-interaction--skillweaver2025", "domain": "interactive", "category": "Browser-Mediated Interaction", "category_id": "browser-mediated-interaction", "key": "skillweaver2025", "title": "SkillWeaver", "sections": [ { "label": "Setting and seed", "text": "WebArena websites and held-out web tasks. A web agent using low-level browser actions." }, { "label": "Reported evolution", "text": "Practice discovers and refines executable Playwright APIs. In the published delete_merged_branches case, a global textbox selector matches both GitLab search and the confirmation field, causing a strict-mode error. The repaired API scopes the textbox and confirmation button to the deletion dialog; the agent patches its persistent knowledge base." }, { "label": "Evidence and limits", "text": "Functions persist beyond their discovery tasks. This is component-level skill improvement, not self-rewriting of the outer optimizer; dynamic-site skills can fail when realistic test inputs are unavailable." } ], "sources": [ { "name": "SkillWeaver", "url": "https://arxiv.org/abs/2504.07079v1", "locator": "Sections 2.2–2.3; Appendix D.1.2–D.2" } ], "benchmarks": [ { "id": "webarena", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "browser-mediated-interaction--wei2026evo", "domain": "interactive", "category": "Browser-Mediated Interaction", "category_id": "browser-mediated-interaction", "key": "wei2026evo", "title": "Evo-Harness", "sections": [ { "label": "Setting and seed", "text": "WebArena-Infinity. Solver without an evolved skill library." }, { "label": "Reported evolution", "text": "Online reflection is compiled into reusable general and task-type skills. Published web cases include disabling an existing auto label, preserving corporate email domains, and archiving qualifying read-important emails." }, { "label": "Evidence and limits", "text": "The main sequential protocol learns between batches. Frozen transfer is evaluated separately; the web examples are advisory procedures, not verified new browser APIs." } ], "sources": [ { "name": "Evo-Harness", "url": "https://arxiv.org/abs/2608.15071v2", "locator": "Sections 3–4; Appendix A; Table 6" } ], "benchmarks": [ { "id": "webarena-infinity", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "browser-mediated-interaction--liteevo2026", "domain": "interactive", "category": "Browser-Mediated Interaction", "category_id": "browser-mediated-interaction", "key": "liteevo2026", "title": "LiteEvo", "sections": [ { "label": "Setting and seed", "text": "WebShop. A neutral turn-protocol prompt and action–observation chat loop, with no initial processors or benchmark-specific prompt content." }, { "label": "Reported evolution", "text": "Repeated failure analysis is converted into reusable prompt, knowledge, processor and configuration components, with candidate selection under the WebShop task evaluator." }, { "label": "Evidence and limits", "text": "The study reports an inductive WebShop configuration, but the concrete login processor described for AppWorld is not a WebShop discovery. A complete selected WebShop diff is not established here." } ], "sources": [ { "name": "LiteEvo", "url": "https://arxiv.org/abs/2609.33146v1", "locator": "Section 3.1; Appendix B.1–B.2; benchmark-specific analysis" } ], "benchmarks": [ { "id": "webshop", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "browser-mediated-interaction--harnessx2026", "domain": "interactive", "category": "Browser-Mediated Interaction", "category_id": "browser-mediated-interaction", "key": "harnessx2026", "title": "HarnessX", "sections": [ { "label": "Setting and seed", "text": "WebShop. Composable HarnessX web-task configuration." }, { "label": "Reported evolution", "text": "Reported prompt/configuration edits reduce search and pagination loops; advisory processors address navigation. The added color-matching tool has zero measured hit-rate in the reported run." }, { "label": "Evidence and limits", "text": "Search repeatedly evaluates the full selected task pool. Aggregate improvement and edit counts do not establish unseen-site transfer or the causal effect of one component." } ], "sources": [ { "name": "HarnessX", "url": "https://arxiv.org/abs/2606.14249v3", "locator": "Experimental setup; Appendix Additional Results, WebShop" } ], "benchmarks": [ { "id": "webshop", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "record_screened", "selection_protocol": null, "controlled_variables": null, "signals": [ { "id": "unused-color-tool", "outcome": "not_assessed", "retention": "not_assessed", "attribution": "usage_observation", "components": [ "tools" ], "benchmark_ids": [ "webshop" ], "scope": "Color-matching tool in the reported WebShop run.", "summary": "Zero measured hit-rate is a usage observation, not an isolated demonstration of a performance regression or null effect.", "section": "Reported evolution", "evidence_quote": "The added color-matching tool has zero measured hit-rate in the reported run.", "source_index": 0, "primary_source": { "name": "HarnessX", "url": "https://arxiv.org/abs/2606.14249v3", "locator": "Experimental setup; Appendix Additional Results, WebShop" } } ] }, { "id": "application-and-service-workflows--liteevo2026", "domain": "interactive", "category": "Application and Service Workflows", "category_id": "application-and-service-workflows", "key": "liteevo2026", "title": "LiteEvo", "sections": [ { "label": "Setting and seed", "text": "AppWorld. The same neutral turn protocol with a code-action format; no initial processors or learned task components are supplied to the worker." }, { "label": "Reported evolution", "text": "The returned harness adds a worked login example and credential-field knowledge; a processor restates the login steps after two authorization errors. These components augment the initially neutral code-action loop." }, { "label": "Evidence and limits", "text": "The processor is a reported concrete returned component. Table 2 gives transductive AppWorld pass@2 of 0% with processors alone, 63.1% with the prompt and 67.7% for the full harness; this separates component bundles, not the login processor individually. The inductive protocol separately chooses on validation tasks and tests unseen tasks. Transfer of components evolved on other benchmarks is much weaker and must remain distinct." } ], "sources": [ { "name": "LiteEvo", "url": "https://arxiv.org/abs/2609.33146v1", "locator": "Sections 3.1–3.3; Appendix B.1–B.2; AppWorld evaluation" } ], "benchmarks": [ { "id": "appworld", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "application-and-service-workflows--scholarevolve2026", "domain": "interactive", "category": "Application and Service Workflows", "category_id": "application-and-service-workflows", "key": "scholarevolve2026", "title": "ScholarEvolve", "sections": [ { "label": "Setting and seed", "text": "AppWorld; τ²-Bench Telecom. Initial modular application/service agent." }, { "label": "Reported evolution", "text": "Validation selects combinations of research-derived tool-interface, context, skill, memory and reasoning modules. Table 8 identifies different selected modules for each backbone and benchmark; final AppWorld excerpts show their implementations." }, { "label": "Evidence and limits", "text": "Paper-derived module names identify design provenance, not proof that the complete source method was reproduced. Periodic lifelong refresh is distinct from the main finite module-search experiment." } ], "sources": [ { "name": "ScholarEvolve", "url": "https://arxiv.org/abs/2609.40169v1", "locator": "Section 4; Appendix A; Appendix D.2, Table 8 and final-module listings" } ], "benchmarks": [ { "id": "appworld", "role": "study" }, { "id": "tau2-bench-telecom", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "application-and-service-workflows--wei2026evo", "domain": "interactive", "category": "Application and Service Workflows", "category_id": "application-and-service-workflows", "key": "wei2026evo", "title": "Evo-Harness", "sections": [ { "label": "Setting and seed", "text": "τ-bench. Solver without an evolved skill library." }, { "label": "Reported evolution", "text": "Batch-level feedback supports addition, merging and revision of reusable service-task guidance, evaluated in the paper's online harness-learning protocol." }, { "label": "Evidence and limits", "text": "The service experiment qualifies as persistent skill evolution, but the detailed web examples above cannot be assumed to be its retained service policies." } ], "sources": [ { "name": "Evo-Harness", "url": "https://arxiv.org/abs/2608.15071v2", "locator": "Sections 3–4; benchmark and skill analyses" } ], "benchmarks": [ { "id": "tau-bench", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "application-and-service-workflows--harnessx2026", "domain": "interactive", "category": "Application and Service Workflows", "category_id": "application-and-service-workflows", "key": "harnessx2026", "title": "HarnessX", "sections": [ { "label": "Setting and seed", "text": "τ³-Bench. HarnessX dialogue and tool-use configuration." }, { "label": "Reported evolution", "text": "The reported service runs ship prompt rules and processors addressing policy ordering and premature actions. No tool edits are shipped in these runs." }, { "label": "Evidence and limits", "text": "This is a distinct experiment from WebShop, not an additional paper. Repeated search-pool evaluation and separate co-training experiments must remain distinct from frozen-model generalization." } ], "sources": [ { "name": "HarnessX", "url": "https://arxiv.org/abs/2606.14249v3", "locator": "Experimental setup; Appendix Additional Results, τ³-Bench" } ], "benchmarks": [ { "id": "tau3-bench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "application-and-service-workflows--rrsi2026", "domain": "interactive", "category": "Application and Service Workflows", "category_id": "application-and-service-workflows", "key": "rrsi2026", "title": "RRSI", "sections": [ { "label": "Setting and seed", "text": "Harvey LAB; transfer to JobBench, GDPval and APEX-Agents. A study-specific ReAct loop over an MCP gateway with a dynamic toolbelt and ReSum-style context management." }, { "label": "Reported evolution", "text": "Regularized edits optimize the workspace harness on Harvey LAB, and the resulting harness is evaluated on a pristine in-domain split and three unseen professional-work suites." }, { "label": "Evidence and limits", "text": "The paper's detailed accepted-edit table contains coding and engineering examples, not a Harvey-specific diff. We record workspace evolution and transfer without importing those other domains' mechanisms." } ], "sources": [ { "name": "RRSI", "url": "https://arxiv.org/abs/2609.24972v3", "locator": "Section 4.1; Appendix A; Appendix E" } ], "benchmarks": [ { "id": "harvey-lab", "role": "study" }, { "id": "jobbench", "role": "transfer" }, { "id": "gdpval", "role": "transfer" }, { "id": "apex-agents", "role": "transfer" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "game-playing--autoharness2026", "domain": "interactive", "category": "Game Playing", "category_id": "game-playing", "key": "autoharness2026", "title": "AutoHarness", "sections": [ { "label": "Setting and seed", "text": "TextArena. A fixed LLM-action/retry scaffold with function interfaces for synthesizing action proposals and legality checks. The LLM without this harness is a comparison baseline." }, { "label": "Reported evolution", "text": "Environment feedback fills and refines the supplied propose_action and is_legal_action interfaces. In the verifier configuration, the existing loop checks LLM-proposed moves and returns an illegal-action warning before retrying. Appendix D.2 exposes chess coordinate parsing, king location and attack detection; a separate configuration synthesizes whole Python policies." }, { "label": "Evidence and limits", "text": "We count the verifier configuration as a harness around an LLM. Pure code policies replace task-time LLM execution and are a distinct boundary case; neither changes the outer search rule." } ], "sources": [ { "name": "AutoHarness", "url": "https://openreview.net/forum?id=g9rEYVNn5T", "locator": "Sections 3–4; Appendices C–D" } ], "benchmarks": [ { "id": "textarena", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "game-playing--continualharness2026", "domain": "interactive", "category": "Game Playing", "category_id": "game-playing", "key": "continualharness2026", "title": "Continual Harness", "sections": [ { "label": "Setting and seed", "text": "Pokémon Red and Emerald. A minimal screenshot/map/button interface with no curated skills or subagents." }, { "label": "Reported evolution", "text": "An automated Refiner revises prompts, subagents, executable skills and memory during a continuous run, using navigation loops and tool failures as feedback." }, { "label": "Evidence and limits", "text": "This record concerns the automated configuration. Historical Gemini Plays Pokémon milestones involve human refinement; detailed historical battle-agent changes cannot be attributed to the automated experiment. Model co-training is separate." } ], "sources": [ { "name": "Continual Harness", "url": "https://arxiv.org/abs/2605.09998v1", "locator": "Sections 2–4; Appendix C" } ], "benchmarks": [ { "id": "pok-mon-red-emerald", "role": "setting" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "game-playing--che2026demoevolve", "domain": "interactive", "category": "Game Playing", "category_id": "game-playing", "key": "che2026demoevolve", "title": "DemoEvolve", "sections": [ { "label": "Setting and seed", "text": "Liar's Dice; Balatro; Slay the Spire 2. Game-specific base harnesses with fixed environment interfaces." }, { "label": "Reported evolution", "text": "Selected Liar's Dice self-rollout harnesses parse game state, calculate claim probabilities and legal raises, and compare calling with raising. Appendix B gives simplified sketches of selected harnesses. Its OneCall-Wild1 variant uses a Bayesian escalation model; during deep same-face escalation, with at least two own contributions, it overrides raising when calling's expected value is higher by a margin. On Balatro and Slay the Spire 2, successful demonstrations supply a separate source of evidence for persistent harness revision." }, { "label": "Evidence and limits", "text": "Appendix G audits a selected Meta-Harness baseline whose proposed Balatro hook never reached the model. That inactive hook is negative evidence, not a successful DemoEvolve discovery; held-out game seeds are evaluated separately." } ], "sources": [ { "name": "DemoEvolve", "url": "https://arxiv.org/abs/2605.24539v2", "locator": "Sections 4.1–4.3; Appendices B, C and G" } ], "benchmarks": [ { "id": "liar-s-dice", "role": "study" }, { "id": "balatro", "role": "study" }, { "id": "slay-the-spire-2", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "game-playing--hcl2026", "domain": "interactive", "category": "Game Playing", "category_id": "game-playing", "key": "hcl2026", "title": "HCL", "sections": [ { "label": "Setting and seed", "text": "Minecraft 50-task curriculum. Static task interface and capability library around a frozen model." }, { "label": "Reported evolution", "text": "Experience supports capability and workflow revisions; retained skill tests gate later updates, extending reuse across crafting and dependent operations." }, { "label": "Evidence and limits", "text": "This is one sequential Minecraft experiment. It documents retention-based capability evolution, without establishing that its updater itself evolves or exposing every final skill diff." } ], "sources": [ { "name": "HCL", "url": "https://arxiv.org/abs/2608.19013v2", "locator": "Section 4.2.2; Appendix C.3" } ], "benchmarks": [ { "id": "minecraft-curriculum", "role": "setting" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "robot-manipulation--zetta2026", "domain": "interactive", "category": "Robot Manipulation", "category_id": "robot-manipulation", "key": "zetta2026", "title": "Zetta", "sections": [ { "label": "Setting and seed", "text": "LIBERO-Pro; RoboCasa. Frozen VLA policy with an initially unevolved critic/recovery harness." }, { "label": "Reported evolution", "text": "The stove-placement lineage adds object-relative pregrasp. After grasp failure it releases and restages, chooses a proposal different from remembered failed grasps, and checks stability within a retry budget. Placement lowers the object until support is detected, then releases and retreats until separation is confirmed." }, { "label": "Evidence and limits", "text": "Code critics propose interventions; the fixed orchestrator authorizes them and the official task predicate determines success. Section 4.3 applies the cumulative stove-task checkpoints to PnP-Sink, PnP-Cabinet and PnP-Toaster without further evolution: mean success rises from 64% to 84%. This supports reuse of the complete recovery stack, not each branch independently. Historical regression and held-out checks gate reusable updates." } ], "sources": [ { "name": "Zetta", "url": "https://arxiv.org/abs/2608.16590v1", "locator": "Sections 3–4; Appendix C" } ], "benchmarks": [ { "id": "libero-pro", "role": "study" }, { "id": "robocasa", "role": "study" }, { "id": "pnp-sink-cabinet-toaster", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "robot-manipulation--shaper2026", "domain": "interactive", "category": "Robot Manipulation", "category_id": "robot-manipulation", "key": "shaper2026", "title": "SHAPER", "sections": [ { "label": "Setting and seed", "text": "VLABench. Generic planner guidance and a context builder accumulating all prior rounds." }, { "label": "Reported evolution", "text": "The evolved skill emits actor-compatible primitive commands with exact entity names. The final context builder emphasizes current images and recent execution state, compresses older rounds and exposes stagnation." }, { "label": "Evidence and limits", "text": "VLA weights and low-level action generation remain fixed. The before/after artifacts support actor-aware command and context changes; ESI-Bench active-perception changes are a separate setting." } ], "sources": [ { "name": "SHAPER", "url": "https://arxiv.org/abs/2608.11350v2", "locator": "Section 4.2; Appendix A.1, Figures A1–A2" } ], "benchmarks": [ { "id": "vlabench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "robot-manipulation--harnesspai2026", "domain": "interactive", "category": "Robot Manipulation", "category_id": "robot-manipulation", "key": "harnesspai2026", "title": "HarnessPAI", "sections": [ { "label": "Setting and seed", "text": "LIBERO/Pro; RoboCasa; other embodied suites. Code-as-policy interface around fixed action primitives." }, { "label": "Reported evolution", "text": "Execution failures revise programs and become node-local skills. The retained align-object-to-basket-mouth skill diagnoses a 5.3 cm depth error caused by estimating the target from all basket-mask points. It changes basket_center_world to use medians of upper-rim points for both horizontal position and height, with an all-points fallback when fewer than five rim points remain. The existing placement servo is unchanged." }, { "label": "Evidence and limits", "text": "The program stays fixed within each rollout and evolves between rollouts. Post-training on collected expert trajectories is a separate weight-update experiment; the published skill is not an isolated causal ablation." } ], "sources": [ { "name": "HarnessPAI", "url": "https://arxiv.org/abs/2609.29166v1", "locator": "Sections 4.4–5.4; Appendix C" } ], "benchmarks": [ { "id": "libero-pro", "role": "study" }, { "id": "robocasa", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "robot-manipulation--robot_harness_distillation2026", "domain": "interactive", "category": "Robot Manipulation", "category_id": "robot-manipulation", "key": "robot_harness_distillation2026", "title": "Recursive Harness Distillation", "sections": [ { "label": "Setting and seed", "text": "SimplerEnv Bridge uses frozen GR00T; three real-world tasks use a frozen fine-tuned π₀.₅ policy. A light agent has an observation/intervention interface, and playbook refinement starts from teacher guidance distilled from initial development rollouts; no-playbook agents are comparison baselines." }, { "label": "Reported evolution", "text": "Strong-agent experience and recipient feedback revise a reusable intervention playbook: re-identify occluded objects, distinguish weak from empty grasps, inspect action mismatches, and verify support before release." }, { "label": "Evidence and limits", "text": "The text guides interventions through the existing policy. On the same 48 held-out Bridge instances, Luna succeeds at 43.8% without a playbook, 31.3% with the initial teacher-distilled playbook and 66.7% with the refined playbook. A Luna-derived refined playbook instead reaches 22.9%; stronger-teacher guidance is not interchangeable with self-refinement. These whole-playbook comparisons do not isolate individual rules or show that the distillation algorithm rewrites itself." } ], "sources": [ { "name": "Recursive Harness Distillation", "url": "https://arxiv.org/abs/2609.33378v1", "locator": "Sections 3.4, 4.3 and 5; Figures 6–7" } ], "benchmarks": [ { "id": "simplerenv-bridge", "role": "study" }, { "id": "real-world-manipulation", "role": "setting" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "video-question-answering--videorsi2026", "domain": "visual", "category": "Video Question Answering", "category_id": "video-question-answering", "key": "videorsi2026", "title": "Video-RSI", "sections": [ { "label": "Setting and seed", "text": "LVBench evolution; frozen evaluation on Video-MME, LongVideoBench, EgoSchema and MLVU. The seed already has global/local inspection, transcript/OCR access and observation memory." }, { "label": "Reported evolution", "text": "Retained revisions add frame-linked findings with uncertainty, relations over existing observations, selective 64-frame overviews and occurrence counting. Graph queries process no new frames; counting consolidates overlapping detections." }, { "label": "Evidence and limits", "text": "The full harness uses fewer frames and answers more accurately than the seed on the four reported suites. Table 2 ablates editor investigation and selection gates, not individual deployed tools. Frame savings exclude editor cost and are not total-compute savings." } ], "sources": [ { "name": "Video-RSI", "url": "https://arxiv.org/abs/2609.37950v1", "locator": "Sections 4.1, 4.3–4.5; Table 2" } ], "benchmarks": [ { "id": "lvbench", "role": "study" }, { "id": "video-mme", "role": "transfer" }, { "id": "longvideobench", "role": "transfer" }, { "id": "egoschema", "role": "transfer" }, { "id": "mlvu", "role": "transfer" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "video-question-answering--vidharness2026", "domain": "visual", "category": "Video Question Answering", "category_id": "video-question-answering", "key": "vidharness2026", "title": "VidHarness", "sections": [ { "label": "Setting and seed", "text": "LongVideoBench; Video-MME; Video-Holmes; transfer to Video-MMMU/MMVU. A task contract and video/transcript/VLM APIs for constructing executable video-agent programs; no separately named seed is established here." }, { "label": "Reported evolution", "text": "MCTS with multi-fidelity validation searches selective-observation programs; retained harnesses form budget-specialized choices in a mixture." }, { "label": "Evidence and limits", "text": "Search design and budget routing are specified by the method. The reviewed text documents evolution and validation trajectories more clearly than final per-tool diffs, so we do not invent a discovered frame-sampling rule." } ], "sources": [ { "name": "VidHarness", "url": "https://arxiv.org/abs/2609.38413v1", "locator": "Sections 3–4; Appendix D" } ], "benchmarks": [ { "id": "longvideobench", "role": "study" }, { "id": "video-mme", "role": "study" }, { "id": "video-holmes", "role": "study" }, { "id": "video-mmmu", "role": "transfer" }, { "id": "mmvu", "role": "transfer" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "temporal-grounding--videoevolve2026", "domain": "visual", "category": "Temporal Grounding", "category_id": "temporal-grounding", "key": "videoevolve2026", "title": "VideoEvolve", "sections": [ { "label": "Setting and seed", "text": "Charades-STA; ActivityNet; QVHighlights; ReXTime; TimeLens-Bench. The seed predicts one interval from 64 uniform frames and generic instructions." }, { "label": "Reported evolution", "text": "Instructions check repeated occurrences and event boundaries. Coarse sampling or unusual interval length triggers another prediction; substantial disagreement triggers a third. Start and end times are separately combined by medians." }, { "label": "Evidence and limits", "text": "All compared configurations use the same 64 frames; extra calls add computation. Figure 3 shows a recovered kettle interval. For the light switch, the instruction-only variant already moves the correct seed interval [0,1] to [1,3]; the three-call variant returns [1,4]. Extra calls therefore do not recover this instruction-level regression. These are cases; Table 3 separates code and instruction contributions across settings." } ], "sources": [ { "name": "VideoEvolve", "url": "https://arxiv.org/abs/2610.01766v1", "locator": "Sections 4.1, 4.4–4.5; Table 3; Figure 3" } ], "benchmarks": [ { "id": "charades-sta", "role": "study" }, { "id": "activitynet", "role": "study" }, { "id": "qvhighlights", "role": "study" }, { "id": "rextime", "role": "study" }, { "id": "timelens-bench", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "record_screened", "selection_protocol": null, "controlled_variables": null, "signals": [ { "id": "kettle-recovery", "outcome": "improvement", "retention": "not_assessed", "attribution": "illustrative_case", "components": [ "instructions", "control" ], "benchmark_ids": [], "scope": "Figure 3 kettle example; the frozen record does not assign this example to one of its five benchmark memberships.", "summary": "A recovered interval is reported for the kettle case, not a dataset-wide component effect.", "section": "Evidence and limits", "evidence_quote": "Figure 3 shows a recovered kettle interval.", "source_index": 0, "primary_source": { "name": "VideoEvolve", "url": "https://arxiv.org/abs/2610.01766v1", "locator": "Sections 4.1, 4.4–4.5; Table 3; Figure 3" } }, { "id": "light-switch-regression", "outcome": "regression", "retention": "not_assessed", "attribution": "illustrative_case", "components": [ "instructions", "control" ], "benchmark_ids": [], "scope": "Figure 3 light-switch example only; not assigned to a specific benchmark in this annotation.", "summary": "The instruction variant moves a correct interval to an incorrect one; extra calls do not recover it.", "section": "Evidence and limits", "evidence_quote": "Extra calls therefore do not recover this instruction-level regression.", "source_index": 0, "primary_source": { "name": "VideoEvolve", "url": "https://arxiv.org/abs/2610.01766v1", "locator": "Sections 4.1, 4.4–4.5; Table 3; Figure 3" } } ] }, { "id": "image-perception-and-question-answering--hcl2026", "domain": "visual", "category": "Image Perception and Question Answering", "category_id": "image-perception-and-question-answering", "key": "hcl2026", "title": "HCL", "sections": [ { "label": "Setting and seed", "text": "COCO detection/captioning; RefCOCO; VQAv2. Frozen VLM with editable external task interfaces, memory, capabilities and router." }, { "label": "Reported evolution", "text": "Sequential non-parametric updates balance learning the next visual task against preserving previously solved anchor examples." }, { "label": "Evidence and limits", "text": "Disabling memory updates mainly reduces captioning performance; fixing the router produces its largest task-level decline on VQAv2. Interface updates contribute most visibly to captioning and VQAv2. These ablations support different component roles, not a published final grounding-specific rule; variants follow different update trajectories. This is the same paper as the QA and Minecraft records." } ], "sources": [ { "name": "HCL", "url": "https://arxiv.org/abs/2608.19013v2", "locator": "Section 4.3.2; Appendix B.2, Table 10; Appendix C.2" } ], "benchmarks": [ { "id": "coco-detection-captioning", "role": "study" }, { "id": "refcoco", "role": "study" }, { "id": "vqav2", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "image-generation--autoref2026", "domain": "visual", "category": "Image Generation", "category_id": "image-generation", "key": "autoref2026", "title": "AutoRef", "sections": [ { "label": "Setting and seed", "text": "MultiBanana; OmniContext. Search starts from a one-call generator and GEMS." }, { "label": "Reported evolution", "text": "The selected lineage binds prompt elements to references, generates subject-led and background-led drafts, checks hard failures before order-swapped comparisons, and generates a third draft from reference-specific complaints. When hard-failure counts tie, the challenger replaces the incumbent only if it wins both presentation orders. This selector uses the target reasoning model; the search evaluator is separate." }, { "label": "Evidence and limits", "text": "Direct-edit variants leave the retained lineage. The fixed-three-image ablations replace removed drafts with another sample and replace failure-aware selection with Best-of-N. They support the combined selector rather than isolating order swapping. Harness-search cost is separate; generator weights stay fixed." } ], "sources": [ { "name": "AutoRef", "url": "https://arxiv.org/abs/2609.35530v1", "locator": "Sections 4–5 and 6.5; Table 4; Appendices D and H.2" } ], "benchmarks": [ { "id": "multibanana", "role": "study" }, { "id": "omnicontext", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "image-generation--comfyclaw2026", "domain": "visual", "category": "Image Generation", "category_id": "image-generation", "key": "comfyclaw2026", "title": "COMFYCLAW", "sections": [ { "label": "Setting and seed", "text": "GenEval 2; DPG-Bench; OneIG splits. A ComfyUI graph-construction agent starts with predefined tools and four general image-generation skills. The supplied refinement architecture includes a visual verifier; the Base comparison executes once without verifier feedback." }, { "label": "Reported evolution", "text": "Cross-run traces and feedback produce reusable SKILL.md procedures; reported examples include spatial/count binding, attribute binding and benchmark-dependent style or detail guidance." }, { "label": "Evidence and limits", "text": "Typed editing, invalid-edit rollback and the region verifier are supplied architecture. Skill-read counts establish consultation, not isolated causal quality. The reported workflow-edit distribution describes per-image actions; it does not by itself identify persistent edits to the harness." } ], "sources": [ { "name": "COMFYCLAW", "url": "https://arxiv.org/abs/2607.01709v1", "locator": "Sections 3.4–4; Table 2; Appendix C, Table 8" } ], "benchmarks": [ { "id": "geneval-2", "role": "study" }, { "id": "dpg-bench", "role": "study" }, { "id": "oneig", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "image-generation--evogen_harness2026", "domain": "visual", "category": "Image Generation", "category_id": "image-generation", "key": "evogen_harness2026", "title": "EvoGen-Harness", "sections": [ { "label": "Setting and seed", "text": "GenEval2; T2I-CompBench++; WISE. Frozen image generators wrapped by an initial multi-responsibility harness." }, { "label": "Reported evolution", "text": "Figure 6 illustrates localized counting-policy, spatial-composition, attribute-binding tool-knowledge and action–object memory changes." }, { "label": "Evidence and limits", "text": "Earlier Appendix B examples phrased as possible patches illustrate the edit space, rather than establish exact retained diffs. Trace, No-Patch and held-out gates are supplied evolution machinery. Figure 6 gives qualitative cases; aggregate scores do not isolate each patch. The experiments adapt a separate harness for each generator, rather than transfer one unchanged harness across them." } ], "sources": [ { "name": "EvoGen-Harness", "url": "https://arxiv.org/abs/2610.00383v1", "locator": "Sections 3–4; Appendices B.6–B.7" } ], "benchmarks": [ { "id": "geneval-2", "role": "study" }, { "id": "t2i-compbench", "role": "study" }, { "id": "wise", "role": "study" } ], "evidence_status": "Concrete artifact described", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "video-production--videoweaver2026", "domain": "visual", "category": "Video Production", "category_id": "video-production", "key": "videoweaver2026", "title": "VideoWeaver", "sections": [ { "label": "Setting and seed", "text": "Long-video production categories and held-out cases. Fixed foundation skills plus initial category composition and creator skills." }, { "label": "Reported evolution", "text": "Training feedback revises reusable composition skills and the instructions that create them; category-local creator skills are merged for unseen-category transfer." }, { "label": "Evidence and limits", "text": "Composition skills are frozen for held-out cases; the merged creator is frozen before receiving unseen-category skill queries and does not update on OOD execution feedback. Section 4.2 and Appendix C specify the update and merging procedures, but the reviewed text does not establish an exact final per-category instruction diff." } ], "sources": [ { "name": "VideoWeaver", "url": "https://arxiv.org/abs/2606.08091v2", "locator": "Section 4.2; Sections 5.2–5.3; Appendix C" } ], "benchmarks": [ { "id": "long-video-production-videoweaver", "role": "setting" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] }, { "id": "video-production--cineforge2026", "domain": "visual", "category": "Video Production", "category_id": "video-production", "key": "cineforge2026", "title": "CineForge", "sections": [ { "label": "Setting and seed", "text": "CineScope; AnimeShooter; ViStoryBench. CineForge-Produce starts with a fixed initial production policy and supplied production stages." }, { "label": "Reported evolution", "text": "Cross-story evidence produces bounded stage-local changes to a persistent textual production policy, admitted through fresh replay and, where needed, paired stochastic evaluation." }, { "label": "Evidence and limits", "text": "The permitted patch types in Appendix B.3 are not a list of exact retained edits. Three-round checkpoint evaluation is read-only on a fixed unseen split. Review-workload measurement overrides editable operational-review settings with a frozen audit reviewer, so lower workload does not result from simply weakening that reviewer." } ], "sources": [ { "name": "CineForge", "url": "https://arxiv.org/abs/2608.29621v1", "locator": "Sections 3.3 and 5.5; Appendices A.6, B.1.6 and B.3–B.8" } ], "benchmarks": [ { "id": "cinescope", "role": "study" }, { "id": "animeshooter", "role": "study" }, { "id": "vistorybench", "role": "study" } ], "evidence_status": "Update procedure / partial artifact", "annotation_status": "not_reviewed", "selection_protocol": null, "controlled_variables": null, "signals": [] } ] }