fix(cloud-agent-next): recover cold attach after idle stop

packages/cloud-agent-sdk/src/normalizer.test.ts

@@ -1812,9 +1812,32 @@ describe('normalize', () => {
       });
     });
 
-    it('defaults error when missing', () => {
+    it('uses attach_exhausted as the error when error is missing', () => {
       const result = normalize(
-        createRaw('cloud.message.failed', { messageId: 'msg', delivery: 'sent' })
+        createRaw('cloud.message.failed', {
+          messageId: 'msg',
+          delivery: 'sent',
+          reason: 'attach_exhausted',
+        })
+      );
+      expect(result).toEqual({
+        type: 'cloud.message.failed',
+        messageId: 'msg',
+        executionId: undefined,
+        delivery: 'sent',
+        error: 'attach_exhausted',
+        reason: 'execution',
+        attempts: undefined,
+      });
+    });
+
+    it('defaults error for internal disconnect reasons when error is missing', () => {
+      const result = normalize(
+        createRaw('cloud.message.failed', {
+          messageId: 'msg',
+          delivery: 'sent',
+          reason: 'control_disconnected',
+        })
       );
       expect(result).toEqual({
         type: 'cloud.message.failed',

packages/cloud-agent-sdk/src/normalizer.ts

@@ -583,7 +583,11 @@ function normalizeInnerEvent(eventType: string, data: unknown): NormalizedEvent
       const reason: 'interrupted' | 'exhausted' | 'execution' =
         rawReason === 'interrupted' ? 'interrupted' : attempts != null ? 'exhausted' : 'execution';
       const error =
-        r.data.error !== undefined ? extractErrorMessage(r.data.error) : 'Message delivery failed';
+        r.data.error !== undefined
+          ? extractErrorMessage(r.data.error)
+          : rawReason === 'attach_exhausted'
+            ? rawReason
+            : 'Message delivery failed';
       return {
         type: 'cloud.message.failed',
         messageId,

services/cloud-agent-next/src/persistence/SandboxControl.ts

@@ -1024,6 +1024,19 @@ export class SandboxControl extends DurableObject<Env> {
       !matchesRoute(route) ||
       !socket
     ) {
+      const guard =
+        physical.state !== 'running'
+          ? 'physical_not_running'
+          : physical.stopTombstone
+            ? 'physical_stopping'
+            : !runtime
+              ? 'runtime_not_ready'
+              : physical.providerRef !== runtime.providerInstanceId
+                ? 'provider_mismatch'
+                : !matchesRoute(route)
+                  ? 'route_mismatch'
+                  : 'socket_not_ready';
+      logControlDiagnostic('worktree_changes_not_ready', { guard }, 'warn');
       return errorResponse(crypto.randomUUID(), 'not_ready', 'Worktree is not attached and ready');
     }
     if (

services/cloud-agent-next/src/sandbox-control/diagnostics.test.ts

@@ -1,5 +1,9 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
-import { CONTROL_DIAGNOSTIC_COALESCE_LIMIT, logControlDiagnostic } from './diagnostics.js';
+import {
+  CONTROL_DIAGNOSTIC_COALESCE_LIMIT,
+  diagnosticCause,
+  logControlDiagnostic,
+} from './diagnostics.js';
 import { logger } from '../logger.js';
 
 describe('logControlDiagnostic', () => {
@@ -68,3 +72,10 @@ describe('logControlDiagnostic', () => {
     expect(withFields).toHaveBeenCalledTimes(1);
   });
 });
+
+describe('diagnosticCause', () => {
+  it('sanitizes and bounds unknown causes', () => {
+    expect(diagnosticCause('untrusted cause/value')).toBe('untrusted_cause_value');
+    expect(diagnosticCause('x'.repeat(129))).toHaveLength(128);
+  });
+});

services/cloud-agent-next/src/sandbox-control/diagnostics.ts

@@ -69,7 +69,9 @@ export function diagnosticEventType(value: string): string {
 }
 
 export function diagnosticCause(value: string): string {
-  return CAUSES.has(value) ? value.replaceAll(' ', '_') : 'other';
+  return CAUSES.has(value)
+    ? value.replaceAll(' ', '_')
+    : value.replace(/[^a-zA-Z0-9_.:-]/g, '_').slice(0, 128);
 }
 
 const DELTA_PROGRESS_EVENTS = new Set([

services/cloud-agent-next/src/sandbox-session/SandboxSession.ts

@@ -175,6 +175,7 @@ import {
   matchesSessionMessageReplay,
   nextQueuedMessageId,
   recordAcceptedMessageActivity,
+  releaseCompletedRetryableAttach,
   releaseUnadmittedWaitingMessages,
   resolveSessionMessageIntent,
   streamCloudStatus,
@@ -2362,6 +2363,19 @@ export class SandboxSession extends DurableObject<Env> {
     const queued = assigned.messages.find(message => message.messageId === messageId);
     const deadlineAt = queued?.deliveryDeadlineAt;
     if (!queued || deadlineAt === undefined) return;
+    if (Date.now() >= deadlineAt && !queued.operations?.prompt?.dispatched) {
+      await this.failDelivery(
+        messageId,
+        'preparation_timeout',
+        queued.wrapperInstanceId,
+        queued.deliveryRetryScope
+      );
+      return;
+    }
+    if (queued.retryNotBefore !== undefined && queued.retryNotBefore > Date.now()) {
+      await this.armQueueRetry(Math.min(deadlineAt, queued.retryNotBefore));
+      return;
+    }
     const provider = getSandboxProvider(metadata);
     const acquisition =
       provider === 'cloudflare' ? { id: assigned.attemptId, deadlineAt } : undefined;
@@ -2480,6 +2494,12 @@ export class SandboxSession extends DurableObject<Env> {
                   ?.dispatched === true
               )
                 return;
+              if (
+                operation === 'session.attach' &&
+                current.operations?.retiredAttach &&
+                sameSessionOperation(current.operations.retiredAttach.authorization, authorization)
+              )
+                return;
               throw new Error('Session operation scope changed');
             },
             defer: pending => this.ctx.waitUntil(pending),
@@ -2685,7 +2705,7 @@ export class SandboxSession extends DurableObject<Env> {
             ? { preparation: { attemptId: recorder.attemptId, triggerMessageId: messageId } }
             : {}),
         };
-        phase = 'attach';
+        phase = needsPreparation ? 'preparing' : 'attach';
         await wait(() =>
           control.attachSession({
             ...(metadata.workspace?.worktreeId
@@ -2862,21 +2882,41 @@ export class SandboxSession extends DurableObject<Env> {
     const message = this.queuedMessage(messageId, epoch, wrapperInstanceId);
     if (!message) return;
     const rejection = error instanceof ControlRequestError && error.code !== 'runtime_unhealthy';
-    const scope = rejection && !message.unresolvedDispatch ? 'message' : 'runtime';
+    const completedAttachFailure =
+      message.operations?.attach?.dispatched === true &&
+      message.operations.attach.result?.ok === false;
+    const retryableCompletedAttach =
+      phase !== 'prompt' && rejection && isRetryableDeliveryError(error) && completedAttachFailure;
+    const scope =
+      phase === 'attach'
+        ? retryableCompletedAttach
+          ? 'message'
+          : 'runtime'
+        : rejection && !message.unresolvedDispatch
+          ? 'message'
+          : 'runtime';
     if (Date.now() >= deadlineAt) {
       await this.failDelivery(messageId, 'preparation_timeout', wrapperInstanceId, scope);
       return;
     }
     const busy = rejection && error.code === 'session_busy';
+    const retryNotBefore = Math.min(deadlineAt, Date.now() + QUEUE_RETRY_MS);
+    const released = retryableCompletedAttach
+      ? releaseCompletedRetryableAttach(this.loadMessages(), messageId, retryNotBefore)
+      : this.loadMessages();
     const updated =
       phase === 'preparing' || busy
         ? undefined
-        : incrementDeliveryFailure(this.loadMessages(), messageId, phase);
-    const messages = (updated?.messages ?? this.loadMessages()).map(
+        : incrementDeliveryFailure(released, messageId, phase);
+    const messages = (updated?.messages ?? released).map(
       (message): MessageRecord =>
         message.messageId === messageId ? { ...message, deliveryRetryScope: scope } : message
     );
     if (!this.saveMessages(messages, epoch)) return;
+    if (retryableCompletedAttach && !updated?.exhausted) {
+      await this.armQueueRetry(retryNotBefore);
+      return;
+    }
     if (isRetryableDeliveryError(error) && !updated?.exhausted) {
       await this.armQueueRetry(Math.min(deadlineAt, Date.now() + QUEUE_RETRY_MS));
       return;

services/cloud-agent-next/src/sandbox-session/control-dispatch.test.ts

@@ -74,18 +74,22 @@ describe('controlRequestResult', () => {
 
 describe('deliveryErrorLogFields', () => {
   it.each(['session_busy', 'not_ready', 'runtime_unhealthy'])(
-    'logs only the allowlisted %s code and retry classification',
+    'logs the public message with the allowlisted %s code and retry classification',
     code => {
       const error = Object.assign(
-        new ControlRequestError({ code, message: 'sensitive-message', retryable: true }),
+        new ControlRequestError({ code, message: 'Public control error', retryable: true }),
         {
           cause: 'sensitive-cause',
           stack: 'sensitive-stack',
           auth: 'sensitive-auth',
           env: 'sensitive-env',
         }
       );
-      expect(deliveryErrorLogFields(error)).toEqual({ errorCode: code, retryable: true });
+      expect(deliveryErrorLogFields(error)).toEqual({
+        errorCode: code,
+        errorMessage: 'Public control error',
+        retryable: true,
+      });
     }
   );
 
@@ -94,11 +98,15 @@ describe('deliveryErrorLogFields', () => {
       deliveryErrorLogFields(
         new ControlRequestError({
           code: 'sensitive-untrusted-code',
-          message: 'sensitive-message',
+          message: 'Public control error',
           retryable: false,
         })
       )
-    ).toEqual({ errorCode: 'unknown_control_error', retryable: false });
+    ).toEqual({
+      errorCode: 'unknown_control_error',
+      errorMessage: 'Public control error',
+      retryable: false,
+    });
   });
 
   it.each([false, true])(

services/cloud-agent-next/src/sandbox-session/control-dispatch.ts

@@ -72,6 +72,7 @@ export function deliveryErrorLogFields(error: unknown) {
       error instanceof ControlRequestError
         ? (controlErrorCodes.find(code => code === error.code) ?? 'unknown_control_error')
         : 'transport_or_internal_error',
+    ...(error instanceof ControlRequestError ? { errorMessage: error.message } : {}),
     retryable: isRetryableDeliveryError(error),
   };
 }

services/cloud-agent-next/src/sandbox-session/recovery/failed-snapshot-uses-terminal-failure.test.ts

@@ -22,6 +22,7 @@ describe('failed snapshot', () => {
           delivery: 'queued',
           accepted: false,
           reason: 'environment_failed',
+          error: 'environment_failed',
           timestamp: 99,
         },
       },
@@ -35,6 +36,7 @@ describe('failed snapshot', () => {
           delivery: 'sent',
           accepted: true,
           reason: 'environment_failed',
+          error: 'environment_failed',
           timestamp: 20,
         },
       },