feat(m3): add capability_4 dataset, wire policy text, remove unlicensed legacy data

.gitignore

@@ -96,7 +96,8 @@ benchmarks/m3/data/*.zip
 benchmarks/m3/data/capability_*/
 benchmarks/m3/data/small_capability_*/
 
-# small_train.zip is the bundled default --m3-data dataset (issue #61), derived
-# from VAKRA (IBM Research) and distributed under CC BY-NC-SA 4.0 — see
+# small_train.zip and capability_4_multiturn_policy_sampled.zip are VAKRA-derived
+# --m3-data datasets distributed under CC BY-NC-SA 4.0 — see
 # benchmarks/m3/data/LICENSE and benchmarks/m3/data/NOTICE.
 !benchmarks/m3/data/small_train.zip
+!benchmarks/m3/data/capability_4_multiturn_policy_sampled.zip

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
     hooks:
       - id: check-added-large-files
         args: ['--maxkb=2000']
-        exclude: '^(uv\.lock|benchmarks/appworld/appworld/|vendor/|milvus_policies\.db|.*\.parquet)'
+        exclude: '^(uv\.lock|benchmarks/appworld/appworld/|vendor/|milvus_policies\.db|.*\.parquet|benchmarks/m3/data/(small_train|capability_4_multiturn_policy_sampled)\.zip$)'
       - id: check-yaml
       - id: check-toml
       - id: end-of-file-fixer

LICENSE

@@ -267,8 +267,9 @@ Licensed under Creative Commons Attribution-NonCommercial-ShareAlike 4.0
 International (CC BY-NC-SA 4.0)
 Source: https://github.com/IBM/vakra
 
-The file benchmarks/m3/data/small_train.zip and any data derived from it are
-NOT covered by the Apache License 2.0 above. They are licensed separately
-under CC BY-NC-SA 4.0 (NonCommercial, ShareAlike) — see
+The files benchmarks/m3/data/small_train.zip and
+benchmarks/m3/data/capability_4_multiturn_policy_sampled.zip, and any data
+derived from them, are NOT covered by the Apache License 2.0 above. They are
+licensed separately under CC BY-NC-SA 4.0 (NonCommercial, ShareAlike) — see
 benchmarks/m3/data/LICENSE and benchmarks/m3/data/NOTICE for the full license
 text and attribution.

benchmarks/m3/config/m3_registry_m3_data.yaml

@@ -9,6 +9,7 @@
 # Capabilities:
 #   task_id=2 → container `capability_2_dashboard_apis`
 #   task_id=3 → container `capability_3_multihop_reasoning`
+#   task_id=4 → container `capability_4_multiturn`
 #
 # Usage:
 #   ./benchmarks/m3/eval.sh --m3-data                        # bundled data/small_train.zip (default)
@@ -80,3 +81,62 @@ services:
           - "trains"
           - "university"
           - "world_development_indicators"
+
+  - m3_task_4:
+      transport: "stdio"
+      type: "mcp"
+      description: "M3 Task 4 environment"
+
+      command: "${CONTAINER_RUNTIME}"
+      args:
+        - "exec"
+        - "-i"
+        - "-e"
+        - "MCP_DOMAIN={domain}"
+        - "-e"
+        - "CAPABILITY_ID=4"
+        - "capability_4_multiturn"
+        - "python3"
+        - "/app/mcp_dispatch.py"
+
+      metadata:
+        task_id: 4
+        container: "capability_4_multiturn"
+        # Domains covered by capability_4_multiturn_policy_sampled.zip (36 total).
+        domains:
+          - "address"
+          - "airline"
+          - "app_store"
+          - "authors"
+          - "beer_factory"
+          - "bike_share_1"
+          - "book_publishing_company"
+          - "books"
+          - "cars"
+          - "codebase_comments"
+          - "coinmarketcap"
+          - "college_completion"
+          - "computer_student"
+          - "cookbook"
+          - "disney"
+          - "european_football_1"
+          - "food_inspection"
+          - "genes"
+          - "hockey"
+          - "ice_hockey_draft"
+          - "image_and_language"
+          - "law_episode"
+          - "menu"
+          - "mondial_geo"
+          - "music_tracker"
+          - "olympics"
+          - "professional_basketball"
+          - "restaurant"
+          - "sales_in_weather"
+          - "shakespeare"
+          - "soccer_2016"
+          - "student_loan"
+          - "talkingdata"
+          - "university"
+          - "world"
+          - "world_development_indicators"

benchmarks/m3/data/NOTICE

@@ -33,3 +33,42 @@ License:
 Disclaimer:
   Provided "AS-IS" with no warranties of any kind — see Section 5 of the
   CC BY-NC-SA 4.0 license in LICENSE.
+
+-----------------------------------------------------------------------
+
+capability_4_multiturn_policy_sampled.zip — VAKRA-derived M3 evaluation data
+=============================================================================
+
+`capability_4_multiturn_policy_sampled.zip` is a 300-sample subset (36
+domains, for capability_4_multiturn) extracted and repackaged from the VAKRA
+benchmark dataset.
+
+Source / creators:
+  VAKRA: A Benchmark for Evaluating Multi-Hop, Multi-Source Tool-Calling in AI
+  Agents. Ankita Rajaram Naik, Anupama Murthi, Benjamin Elder, Siyu Huo,
+  Praveen Venkateswaran, Danish Contractor (IBM Research), 2026.
+  https://github.com/IBM/vakra
+  https://huggingface.co/datasets/ibm-research/VAKRA
+
+Sampling balance:
+  - Samples with a policy (non-empty additional_instructions): 150
+  - Samples without a policy: 150
+  - Single-turn dialogues: 133
+  - Multi-turn dialogues: 167
+  See benchmarks/m3/eval_config.toml (cap4_* keys) for the corresponding
+  --eval-key sample-id lists, generated by
+  benchmarks/m3/scripts/generate_policy_turn_keys.py.
+
+Modifications from the original VAKRA dataset:
+  capability_4_multiturn_policy_sampled.zip contains a repackaged subset of
+  the original VAKRA task data (input/output JSON for the domains listed
+  above), with macOS filesystem metadata (__MACOSX/, .DS_Store) removed.
+
+License:
+  Licensed under CC BY-NC-SA 4.0, same terms as small_train.zip above — see
+  the LICENSE file in this directory and the "License" section above for
+  full text and the Apache 2.0 incompatibility note.
+
+Disclaimer:
+  Provided "AS-IS" with no warranties of any kind — see Section 5 of the
+  CC BY-NC-SA 4.0 license in LICENSE.