feat(core): automatically count and store meta for tool call count

libs/core/langchain_core/tracers/__init__.py

@@ -15,6 +15,10 @@
     )
     from langchain_core.tracers.schemas import Run
     from langchain_core.tracers.stdout import ConsoleCallbackHandler
+    from langchain_core.tracers.utils import (
+        count_tool_calls_in_run,
+        store_tool_call_count_in_run,
+    )
 
 __all__ = (
     "BaseTracer",
@@ -25,6 +29,8 @@
     "Run",
     "RunLog",
     "RunLogPatch",
+    "count_tool_calls_in_run",
+    "store_tool_call_count_in_run",
 )
 
 _dynamic_imports = {
@@ -36,6 +42,8 @@
     "RunLogPatch": "log_stream",
     "Run": "schemas",
     "ConsoleCallbackHandler": "stdout",
+    "count_tool_calls_in_run": "utils",
+    "store_tool_call_count_in_run": "utils",
 }
 
 

libs/core/langchain_core/tracers/core.py

@@ -22,6 +22,7 @@
     LLMResult,
 )
 from langchain_core.tracers.schemas import Run
+from langchain_core.tracers.utils import store_tool_call_count_in_run
 
 if TYPE_CHECKING:
     from collections.abc import Coroutine, Sequence
@@ -283,6 +284,7 @@ def _complete_llm_run(self, response: LLMResult, run_id: UUID) -> Run:
                     )
         llm_run.end_time = datetime.now(timezone.utc)
         llm_run.events.append({"name": "end", "time": llm_run.end_time})
+        store_tool_call_count_in_run(llm_run)
 
         return llm_run
 

libs/core/langchain_core/tracers/utils.py

@@ -0,0 +1,68 @@
+"""Utility functions for working with `Run` objects and tracers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from langchain_core.tracers.schemas import Run
+
+
+def count_tool_calls_in_run(run: Run) -> int:
+    """Count tool calls in a `Run` object by examining messages.
+
+    Args:
+        run: The `Run` object to examine.
+
+    Returns:
+        The total number of tool calls found in the run's messages.
+    """
+    tool_call_count = 0
+
+    def _count_tool_calls_in_messages(messages: list) -> int:
+        count = 0
+        for msg in messages:
+            if hasattr(msg, "tool_calls"):
+                tool_calls = getattr(msg, "tool_calls", [])
+                count += len(tool_calls)
+            elif isinstance(msg, dict) and "tool_calls" in msg:
+                tool_calls = msg.get("tool_calls", [])
+                count += len(tool_calls)
+        return count
+
+    # Check inputs for messages containing tool calls
+    inputs = getattr(run, "inputs", {})
+    if isinstance(inputs, dict) and "messages" in inputs:
+        messages = inputs["messages"]
+        if messages:
+            tool_call_count += _count_tool_calls_in_messages(messages)
+
+    outputs = getattr(run, "outputs", {})
+    if isinstance(outputs, dict) and "messages" in outputs:
+        messages = outputs["messages"]
+        if messages:
+            tool_call_count += _count_tool_calls_in_messages(messages)
+
+    return tool_call_count
+
+
+def store_tool_call_count_in_run(run: Run, *, always_store: bool = False) -> int:
+    """Count tool calls in a `Run` and store the count in run metadata.
+
+    Args:
+        run: The `Run` object to analyze and modify.
+        always_store: If `True`, always store the count even if `0`. If `False`, only
+            store when there are tool calls.
+
+    Returns:
+        The number of tool calls found and stored.
+    """
+    tool_call_count = count_tool_calls_in_run(run)
+
+    # Only store if there are tool calls or if explicitly requested
+    if tool_call_count > 0 or always_store:
+        if run.extra is None:
+            run.extra = {}
+        run.extra["tool_call_count"] = tool_call_count
+
+    return tool_call_count

libs/core/tests/unit_tests/tracers/test_automatic_metadata.py

@@ -0,0 +1,154 @@
+"""Test automatic tool call count storage in tracers."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, PropertyMock
+
+from langchain_core.messages import AIMessage
+from langchain_core.messages.tool import ToolCall
+from langchain_core.outputs import LLMResult
+from langchain_core.tracers.core import _TracerCore
+from langchain_core.tracers.schemas import Run
+
+
+class MockTracerCore(_TracerCore):
+    """Mock tracer core for testing LLM run completion."""
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def _persist_run(self, run: Run) -> None:
+        """Mock implementation of _persist_run."""
+
+
+def test_complete_llm_run_automatically_stores_tool_call_count() -> None:
+    """Test that `_complete_llm_run` automatically stores tool call count."""
+    tracer = MockTracerCore()
+
+    # Create a mock LLM run with tool calls
+    run = MagicMock(spec=Run)
+    run.id = "test-llm-run-id"
+    run.run_type = "llm"
+    run.extra = {}
+    run.outputs = {}
+    run.events = []
+    run.end_time = None
+
+    # Set up messages with tool calls in inputs
+    tool_calls = [
+        ToolCall(name="search", args={"query": "test"}, id="call_1"),
+        ToolCall(name="calculator", args={"expression": "2+2"}, id="call_2"),
+    ]
+    messages = [AIMessage(content="Test", tool_calls=tool_calls)]
+    run.inputs = {"messages": messages}
+
+    # Add run to tracer's run_map
+    tracer.run_map[str(run.id)] = run
+
+    # Create a mock LLMResult
+    response = MagicMock(spec=LLMResult)
+    response.model_dump.return_value = {"generations": [[]]}
+    response.generations = [[]]
+
+    # Complete the LLM run (this should trigger automatic metadata storage)
+    completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+    # Verify tool call count was automatically stored
+    assert "tool_call_count" in completed_run.extra
+    assert completed_run.extra["tool_call_count"] == 2
+
+
+def test_complete_llm_run_handles_no_tool_calls() -> None:
+    """Test that `_complete_llm_run` handles runs with no tool calls gracefully."""
+    tracer = MockTracerCore()
+
+    # Create a mock LLM run without tool calls
+    run = MagicMock(spec=Run)
+    run.id = "test-llm-run-id-no-tools"
+    run.run_type = "llm"
+    run.extra = {}
+    run.outputs = {}
+    run.events = []
+    run.end_time = None
+
+    # Set up messages without tool calls
+    messages = [AIMessage(content="No tools here")]
+    run.inputs = {"messages": messages}
+
+    # Add run to tracer's run_map
+    tracer.run_map[str(run.id)] = run
+
+    # Create a mock LLMResult
+    response = MagicMock(spec=LLMResult)
+    response.model_dump.return_value = {"generations": [[]]}
+    response.generations = [[]]
+
+    # Complete the LLM run
+    completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+    # Verify tool call count is not stored when there are no tool calls
+    assert "tool_call_count" not in completed_run.extra
+
+
+def test_complete_llm_run_handles_runs_without_messages() -> None:
+    """Test that `_complete_llm_run` handles runs without messages gracefully."""
+    tracer = MockTracerCore()
+
+    # Create a mock LLM run without messages
+    run = MagicMock(spec=Run)
+    run.id = "test-llm-run-id-no-messages"
+    run.run_type = "llm"
+    run.extra = {}
+    run.outputs = {}
+    run.events = []
+    run.end_time = None
+    run.inputs = {}
+
+    # Add run to tracer's run_map
+    tracer.run_map[str(run.id)] = run
+
+    # Create a mock LLMResult
+    response = MagicMock(spec=LLMResult)
+    response.model_dump.return_value = {"generations": [[]]}
+    response.generations = [[]]
+
+    # Complete the LLM run
+    completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+    # Verify tool call count is not stored when there are no messages
+    assert "tool_call_count" not in completed_run.extra
+
+
+def test_complete_llm_run_continues_on_metadata_error() -> None:
+    """Test that `_complete_llm_run` continues working if metadata storage fails."""
+    tracer = MockTracerCore()
+
+    # Create a mock LLM run that will cause an error in tool call counting
+    run = MagicMock(spec=Run)
+    run.id = "test-llm-run-id-error"
+    run.run_type = "llm"
+    run.extra = {}
+    run.outputs = {}
+    run.events = []
+    run.end_time = None
+
+    # Make the run.inputs property raise an error when accessed
+    type(run).inputs = PropertyMock(side_effect=RuntimeError("Simulated error"))
+
+    # Add run to tracer's run_map
+    tracer.run_map[str(run.id)] = run
+
+    # Create a mock LLMResult
+    response = MagicMock(spec=LLMResult)
+    response.model_dump.return_value = {"generations": [[]]}
+    response.generations = [[]]
+
+    # Complete the LLM run - this should raise an exception due to our mock error
+    # but that's expected behavior since the tool call counting failed
+    try:  # noqa: SIM105
+        tracer._complete_llm_run(response=response, run_id=run.id)
+        # If no exception is raised, then the implementation changed
+        # to be more defensive, which is fine
+    except RuntimeError:
+        # This is the expected behavior since we made inputs raise an error
+        pass

libs/core/tests/unit_tests/tracers/test_imports.py

@@ -2,13 +2,15 @@
 
 EXPECTED_ALL = [
     "BaseTracer",
+    "ConsoleCallbackHandler",
     "EvaluatorCallbackHandler",
     "LangChainTracer",
-    "ConsoleCallbackHandler",
+    "LogStreamCallbackHandler",
     "Run",
     "RunLog",
     "RunLogPatch",
-    "LogStreamCallbackHandler",
+    "count_tool_calls_in_run",
+    "store_tool_call_count_in_run",
 ]