fix: auto-disable unhealthy automations

migrations/versions/020_add_automation_disabled_reason.py

@@ -0,0 +1,102 @@
+"""Add automation disabled reason metadata.
+
+Revision ID: 020
+Revises: 019
+Create Date: 2026-08-20
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+
+revision: str = "020"
+down_revision: str = "019"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _is_sqlite() -> bool:
+    return op.get_bind().dialect.name == "sqlite"
+
+
+def upgrade() -> None:
+    op.add_column("automations", sa.Column("disabled_reason", sa.Text(), nullable=True))
+    op.add_column("automations", sa.Column("disabled_detail", sa.JSON(), nullable=True))
+    op.add_column(
+        "automations",
+        sa.Column("disabled_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_table(
+        "automation_disable_events",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("automation_id", sa.Uuid(), nullable=False),
+        sa.Column("run_id", sa.Uuid(), nullable=True),
+        sa.Column("reason", sa.Text(), nullable=False),
+        sa.Column("detail", sa.JSON(), nullable=True),
+        sa.Column("source", sa.String(length=64), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["automation_id"], ["automations.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["run_id"], ["automation_runs.id"], ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_automation_disable_events_automation_id",
+        "automation_disable_events",
+        ["automation_id"],
+    )
+    op.create_index(
+        "ix_automation_disable_events_run_id",
+        "automation_disable_events",
+        ["run_id"],
+    )
+    op.create_index(
+        "ix_automation_disable_events_created_at",
+        "automation_disable_events",
+        ["created_at"],
+    )
+
+    if _is_sqlite():
+        return
+
+    op.execute(
+        "COMMENT ON COLUMN automations.disabled_reason IS "
+        "'Human-readable reason an automation is currently disabled.'"
+    )
+    op.execute(
+        "COMMENT ON COLUMN automations.disabled_detail IS "
+        "'Structured metadata for current automation disabled state.'"
+    )
+    op.execute(
+        "COMMENT ON TABLE automation_disable_events IS "
+        "'Historical records for automation auto-disable decisions.'"
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_automation_disable_events_created_at",
+        table_name="automation_disable_events",
+    )
+    op.drop_index(
+        "ix_automation_disable_events_run_id",
+        table_name="automation_disable_events",
+    )
+    op.drop_index(
+        "ix_automation_disable_events_automation_id",
+        table_name="automation_disable_events",
+    )
+    op.drop_table("automation_disable_events")
+    op.drop_column("automations", "disabled_at")
+    op.drop_column("automations", "disabled_detail")
+    op.drop_column("automations", "disabled_reason")

openhands/automation/config.py

@@ -449,6 +449,8 @@ class ServiceSettings(BaseSettings):
         AUTOMATION_DISPATCHER_INTERVAL_SECONDS: Dispatcher poll interval (default: 10)
         AUTOMATION_DISPATCHER_BATCH_SIZE: Dispatcher batch size (default: 10)
         AUTOMATION_WATCHDOG_INTERVAL_SECONDS: Watchdog poll interval (default: 60)
+        AUTOMATION_FAILURE_DISABLE_THRESHOLD: Consecutive permanent failures before
+            auto-disabling an automation (default: 3, <=0 disables auto-disable)
 
         # API pagination
         AUTOMATION_API_DEFAULT_PAGE_SIZE: Default page size (default: 50)
@@ -532,6 +534,7 @@ class ServiceSettings(BaseSettings):
     dispatcher_interval_seconds: int = 10
     dispatcher_batch_size: int = 10
     watchdog_interval_seconds: int = 60
+    failure_disable_threshold: int = 3
 
     # How long an accepted event stays in `integration_events`. It bounds two
     # things: the dedupe window (a redelivery older than this is indistinguishable

openhands/automation/dispatcher.py

@@ -65,6 +65,9 @@
 )
 from openhands.automation.utils.time import utcnow
 from openhands.automation.utils.timeout import resolve_automation_timeout_seconds
+from openhands.automation.utils.unhealthy import (
+    maybe_disable_unhealthy_automation_after_run,
+)
 
 
 logger = logging.getLogger("automation.dispatcher")
@@ -127,8 +130,13 @@ async def _poll_pending_runs(
     """
     select_query = (
         select(AutomationRun)
+        .join(AutomationRun.automation)
         .options(selectinload(AutomationRun.automation))
-        .where(AutomationRun.status == AutomationRunStatus.PENDING)
+        .where(
+            AutomationRun.status == AutomationRunStatus.PENDING,
+            Automation.enabled.is_(True),
+            Automation.deleted_at.is_(None),
+        )
         .order_by(AutomationRun.created_at.asc())
         .limit(batch_size)
     )
@@ -216,6 +224,22 @@ async def _fail(
             error,
             status_detail=status_detail,
         )
+        automation_disabled = disable
+        if disable:
+            automation_disabled = await disable_automation(
+                session_factory,
+                automation.id,
+                error,
+                disabled_detail={"status_detail": status_detail}
+                if status_detail is not None
+                else None,
+                run_id=run.id,
+            )
+        elif status_detail is not None:
+            automation_disabled = await maybe_disable_unhealthy_automation_after_run(
+                session_factory,
+                automation.id,
+            )
         await capture_automation_event(
             "automation_run_failed",
             automation=automation,
@@ -224,11 +248,9 @@ async def _fail(
             properties={
                 "trigger_source": "dispatcher",
                 "failure_kind": "dispatch_error",
-                "automation_disabled": disable,
+                "automation_disabled": automation_disabled,
             },
         )
-        if disable:
-            await disable_automation(session_factory, automation.id, error)
 
     # 1. Calculate effective timeout (doesn't depend on ctx). This same value
     # drives both the bash command timeout and the watchdog cleanup deadline.

openhands/automation/models.py

@@ -97,6 +97,13 @@ class Automation(Base):
     # Whether the automation is enabled (can be triggered)
     enabled: Mapped[bool] = mapped_column(default=True, nullable=False, index=True)
 
+    # Current disabled-state metadata. AutomationDisableEvent keeps history.
+    disabled_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
+    disabled_detail: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    disabled_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+
     # Soft delete timestamp (NULL = not deleted)
     deleted_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True, index=True
@@ -128,6 +135,11 @@ class Automation(Base):
     runs: Mapped[list["AutomationRun"]] = relationship(
         "AutomationRun", back_populates="automation", cascade="all, delete-orphan"
     )
+    disable_events: Mapped[list["AutomationDisableEvent"]] = relationship(
+        "AutomationDisableEvent",
+        back_populates="automation",
+        cascade="all, delete-orphan",
+    )
 
 
 class AutomationRun(Base):
@@ -206,6 +218,7 @@ class AutomationRun(Base):
         server_default=text("CURRENT_TIMESTAMP"),
         nullable=False,
     )
+
     started_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )
@@ -230,6 +243,41 @@ class AutomationRun(Base):
     )
 
 
+class AutomationDisableEvent(Base):
+    """Historical record of an automation being disabled."""
+
+    __tablename__ = "automation_disable_events"
+
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
+    automation_id: Mapped[uuid.UUID] = mapped_column(
+        Uuid,
+        ForeignKey("automations.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    run_id: Mapped[uuid.UUID | None] = mapped_column(
+        Uuid,
+        ForeignKey("automation_runs.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    reason: Mapped[str] = mapped_column(Text, nullable=False)
+    detail: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    source: Mapped[str] = mapped_column(String(64), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        server_default=text("CURRENT_TIMESTAMP"),
+        nullable=False,
+        index=True,
+    )
+
+    automation: Mapped["Automation"] = relationship(
+        "Automation",
+        back_populates="disable_events",
+    )
+    run: Mapped["AutomationRun | None"] = relationship("AutomationRun")
+
+
 class TarballUpload(Base):
     """A tarball upload for automation code.