Repository navigation
Expand file tree
/
Copy pathcodec_agents.py
More file actions
1389 lines (1226 loc) · 65.9 KB
/
Copy pathcodec_agents.py
File metadata and controls
1389 lines (1226 loc) · 65.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
CODEC Agents — Local multi-agent framework
Replaces CrewAI with ~300 lines. Zero external dependencies.
Uses CODEC skills as tools + Qwen 3.6 35B with thinking mode.
"""
import asyncio
import contextvars
import json
import os
import re
import time
from dataclasses import dataclass, field
from datetime import datetime
from typing import Callable, Dict, List, Optional
import hashlib
import logging
import threading
import httpx
from codec_audit import audit as _audit_core
from codec_hooks import HookVeto, run_with_hooks
from codec_llm_proxy import llm_queue, Priority
import codec_fence
import codec_models
log = logging.getLogger("codec_agents")
class _ToolNote(str):
"""A message CODEC writes in place of a tool result (timeout, plugin veto).
It is CODEC's own text, so Agent.run does not fence it (P-021)."""
# ── Tool-call validation ──
_VALID_TOOL_NAME_RE = re.compile(r'^[A-Za-z0-9_.\-]+$')
_MAX_TOOL_NAME_LEN = 100
_MAX_TOOL_INPUT_LEN = 50000
# L1 / SR-61: per-tool wall-clock budget. A skill tool with no internal timeout
# (input(), a no-timeout network call, a deadlock) would otherwise hang the
# agent — and its default-thread-pool worker — forever. wait_for returns control
# to the agent as a recoverable error string. (Residual: the abandoned worker
# thread can't be killed and stays parked until the blocking call returns — a
# generous budget keeps that rare; tools doing real work finish well inside it.)
_TOOL_CALL_TIMEOUT_SECONDS = 120
# ── CONFIG ──
from codec_config_path import config_path # where config.json is (CODEC_CONFIG_PATH in tests)
try:
from codec_config import SKILLS_DIR, USER_SKILLS_DIR
except ImportError:
SKILLS_DIR = os.path.expanduser("~/.codec/skills")
USER_SKILLS_DIR = None
DB_PATH = os.path.expanduser("~/.codec/memory.db")
def _cfg():
try:
with open(config_path()) as f:
return json.load(f)
except Exception as e:
log.warning("Config load failed: %s", e)
return {}
def _qwen_base():
# A-12 (PR-3E-async): base URL (no /chat/completions) for codec_llm.acall.
# (Replaced the old _qwen_url() — both its callers now use codec_llm, which
# appends /chat/completions itself.)
return _cfg().get("llm_base_url", "http://localhost:8083/v1")
# A pinned-model override for crews that must NOT run on the chat picker's
# active model (a voice/style fine-tune confabulates on factual research).
# run_crew sets it for the duration of a pinned run; None everywhere else, so
# chat and every other path keep using cfg:llm_model unchanged.
_MODEL_OVERRIDE: contextvars.ContextVar = contextvars.ContextVar(
"codec_model_override", default=None)
# Deep Research is restricted to strong general models. 35B is the default;
# 27B is user-selectable. Everything else (voice fine-tunes, 4B/8B) is rejected
# — research on a small/voice model is exactly what produced the bad report.
DEEP_RESEARCH_MODELS = dict(codec_models.RESEARCH_MODELS) # ids live in codec_models (P-057)
DEEP_RESEARCH_DEFAULT = DEEP_RESEARCH_MODELS["35b"]
def resolve_deep_research_model(requested) -> str:
"""Map a UI selection to an allowed research model id. Accepts a short key
('35b'/'27b') or a full model id; anything not in the allowlist (incl. None
or a chat-picker voice model) falls back to the 35B default."""
if not requested:
return DEEP_RESEARCH_DEFAULT
r = str(requested).strip()
if r.lower() in DEEP_RESEARCH_MODELS:
return DEEP_RESEARCH_MODELS[r.lower()]
if r in DEEP_RESEARCH_MODELS.values():
return r
return DEEP_RESEARCH_DEFAULT
def _qwen_model():
return _MODEL_OVERRIDE.get() or codec_models.resolve("agent", _cfg())
# PR-2B-2 (D-15): Keychain-aware getter (cfg→Keychain migration + env fallback).
def _serper_api_key() -> str:
try:
from codec_config import get_serper_api_key
return get_serper_api_key()
except Exception:
return os.environ.get("SERPER_API_KEY", "")
# L1 / SR-61: dropped the eager module global `SERPER_API_KEY = _serper_api_key()`
# — it was never read (web_search routes through codec_search.search, which
# fetches the key itself) yet did a Keychain shellout at every import. The
# getter above stays for callers that want a live read.
# ── HTTP connection pools (reuse TCP connections across calls) ──
_HTTP_HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}
_sync_http = httpx.Client(timeout=30, follow_redirects=True, headers=_HTTP_HEADERS)
# Deep Research sends very large prompts (14k+ tokens). On a 27B at ~13 tok/s a
# single call measured 336s (78s prefill + 258s decode) — the old 180s ceiling
# aborted it while the model was still generating, surfacing as
# "async LLM call failed" after the run had already burned 12 minutes.
# Operator-tunable via ~/.codec/config.json:agents.llm_timeout_s.
def _agent_llm_timeout():
try:
import json as _j
with open(config_path()) as _f:
_c = _j.load(_f).get('agents', {})
return float(_c.get('llm_timeout_s', 900))
except Exception:
return 900.0
_async_http = httpx.AsyncClient(timeout=_agent_llm_timeout())
# ── AUDIT LOGGER ──
# Crew/agent events route through codec_audit.audit() — writes to
# ~/.codec/audit.log via the unified envelope (schema:1) per
# docs/PHASE1-STEP1-DESIGN.md. The legacy duplicate audit.log writer
# (formerly _AUDIT_LOG_PATH at this position) is gone: one writer, one
# rotation, one threading.Lock. See codec_audit.py for the actual write.
# Correlation-id propagation: a top-level operation (Crew.run, Agent.run when
# called outside a crew) sets _correlation_id_var; nested emits inherit it
# automatically. Using contextvars keeps the ID intact across asyncio task
# boundaries and run_in_executor calls. See design §1.4.
#
# A5 / SR-5: the canonical home for this contextvar moved to codec_audit so
# downstream readers (codec_ask_user, codec_observer, codec_triggers) can
# import it without dragging codec_agents into a cycle. Re-exported here for
# back-compat with any external importer that grabbed
# `codec_agents._correlation_id_var` directly.
from codec_audit import (
_correlation_id_var as _correlation_id_var, # noqa: F401 — re-export
_new_correlation_id as _new_correlation_id, # noqa: F401 — re-export
)
def _audit(event_type: str, **kwargs):
"""Shim over codec_audit.audit() for crew/agent runtime events.
Translates the historic crew kwargs (`elapsed`, free-form keys) into the
unified envelope. Pulls correlation_id from the contextvar when not
passed explicitly. Never raises.
"""
elapsed = kwargs.pop("elapsed", None)
duration_ms = kwargs.pop("duration_ms", None)
if duration_ms is None and isinstance(elapsed, (int, float)):
duration_ms = float(elapsed) * 1000.0
cid = kwargs.pop("correlation_id", None) or _correlation_id_var.get()
tool = kwargs.pop("tool", "") or ""
agent = kwargs.pop("agent", None)
outcome = kwargs.pop("outcome", "ok")
error_type = kwargs.pop("error_type", None)
error = kwargs.pop("error", None)
extra = {k: v for k, v in kwargs.items() if v is not None}
# `elapsed` survives as-is in extra (analyzer may want the integer-second form)
if elapsed is not None and "elapsed" not in extra:
extra["elapsed"] = elapsed
try:
_audit_core(
tool=tool,
event=event_type,
source="codec-agents",
outcome=outcome,
duration_ms=duration_ms,
agent=agent,
transport="crew",
error_type=error_type,
error=error,
correlation_id=cid,
extra=extra or None,
)
except Exception as e:
log.debug("Audit emit failed (event=%s): %s", event_type, e)
# Captures the last Google Docs URL created — fallback if Writer forgets to echo it
_last_gdoc_url: Optional[str] = None
# Google Docs rate-limit / dedup state
_gdoc_lock = threading.Lock()
_gdoc_created: Dict[str, float] = {} # title_hash → timestamp
_GDOC_COOLDOWN_SEC = 60 # minimum seconds between docs with same title
# ═══════════════════════════════════════════════════════════════
# TOOL
# ═══════════════════════════════════════════════════════════════
@dataclass
class Tool:
name: str
description: str
fn: Callable
def run(self, input_str: str) -> str:
try:
result = self.fn(input_str)
text = str(result) if result else "No output."
if len(text) > 10000:
text = text[:10000] + f"\n\n[TRUNCATED: output was {len(text)} chars, showing first 10000]"
return text
except Exception as e:
return f"Tool error ({self.name}): {e}"
# ═══════════════════════════════════════════════════════════════
# BUILT-IN TOOLS
# ═══════════════════════════════════════════════════════════════
def _web_search(query: str) -> str:
"""Search via DuckDuckGo (free, no key) or Serper if configured in ~/.codec/config.json."""
import sys as _sys
_sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from codec_search import search, format_results
results = search(query.strip(), max_results=10)
return format_results(results, max_snippets=10)
def _web_fetch(url: str) -> str:
try:
# Fix #7 (H1) + re-audit N3: SSRF guard BEFORE the request AND on every
# redirect hop. The fetched text is returned to the agent/LLM, so a read
# of an internal/metadata host is an exfil path; _sync_http defaults to
# follow_redirects=True, which would reach an internal target via a 302
# the guard never saw — so we follow redirects manually here.
# P-008: the body is streamed and the whole fetch stops near 2 MB / 15 s.
import codec_ssrf
from urllib.parse import urljoin
cur = url.strip()
budget = codec_ssrf.FetchBudget()
try:
for _ in range(6): # initial request + up to 5 redirects
codec_ssrf.validate_url(cur)
with _sync_http.stream("GET", cur, follow_redirects=False,
timeout=budget.timeout()) as r:
if r.is_redirect and r.headers.get("location"):
cur = urljoin(cur, r.headers["location"])
continue
status = r.status_code
if status < 400:
body, _cut = codec_ssrf.read_capped(r.iter_bytes(), budget)
text = body.decode(r.encoding or "utf-8", errors="replace")
break
else:
return "Fetch error: blocked URL (too many redirects)"
except codec_ssrf.SSRFError as e:
return f"Fetch error: blocked URL ({e})"
except TimeoutError:
return "Fetch error: the page took longer than 15 s"
if status in (401, 403):
return f"Blocked by site (HTTP {status}). Site requires JavaScript or blocks automated access."
if status >= 400:
return f"HTTP error {status} fetching {url}"
text = re.sub(r'<script[^>]*>[\s\S]*?</script>', '', text)
text = re.sub(r'<style[^>]*>[\s\S]*?</style>', '', text)
text = re.sub(r'<[^>]+>', ' ', text)
text = re.sub(r'\s+', ' ', text).strip()
if len(text) > 10000:
return text[:10000] + f"\n\n[TRUNCATED: page was {len(text)} chars, showing first 10000]"
return text
except Exception as e:
return f"Fetch error: {e}"
def _file_read(path: str) -> str:
path = path.strip()
if path.startswith("~/"):
path = os.path.expanduser(path)
elif not path.startswith("/"):
path = os.path.join(os.path.expanduser("~/codec-workspace"), path)
# Resolve symlinks and .. to prevent traversal
path = os.path.realpath(path)
home = os.path.realpath(os.path.expanduser("~"))
if not path.startswith(home):
return "Error: cannot read files outside home directory."
try:
with open(path, "r", errors="ignore") as f:
content = f.read()
if len(content) > 10000:
return content[:10000] + f"\n\n[TRUNCATED: file was {len(content)} chars, showing first 10000]"
return content
except Exception as e:
return f"File read error: {e}"
def _file_write(input_str: str) -> str:
path = ""
content = ""
for line in input_str.split("\n"):
if line.lower().startswith("path:"):
path = line.split(":", 1)[1].strip()
elif line.lower().startswith("content:"):
content = input_str.split("content:", 1)[1].strip()
break
if not path:
lines = input_str.strip().split("\n", 1)
path = lines[0].strip()
content = lines[1] if len(lines) > 1 else ""
workspace = os.path.expanduser("~/codec-workspace")
os.makedirs(workspace, exist_ok=True)
if not path.startswith("/"):
path = os.path.join(workspace, path)
# Resolve symlinks and .. to prevent traversal
path = os.path.realpath(path)
# P-006: only the allowed write folders, no scripts or launch files.
import codec_write_policy
ok, reason = codec_write_policy.check_write(path)
if not ok:
return f"Error: {reason}"
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
f.write(content)
return f"Written {len(content)} chars to {path}"
except Exception as e:
return f"File write error: {e}"
def _google_docs_create(input_str: str) -> str:
"""Create a richly styled Google Doc — reuses codec_gdocs.create_google_doc().
Rate-limited: blocks duplicate titles within 60 seconds."""
global _last_gdoc_url
title = "CODEC Report"
content = input_str
if "title:" in input_str.lower():
for line in input_str.split("\n"):
if line.lower().startswith("title:"):
title = line.split(":", 1)[1].strip()
elif line.lower().startswith("content:"):
content = input_str.split("content:", 1)[1].strip()
break
# Dedup: reject same title within cooldown period
title_hash = hashlib.sha256(title.encode()).hexdigest()[:16]
with _gdoc_lock:
now = time.time()
last_created = _gdoc_created.get(title_hash, 0)
if now - last_created < _GDOC_COOLDOWN_SEC:
remaining = int(_GDOC_COOLDOWN_SEC - (now - last_created))
return (f"Rate-limited: a Google Doc titled '{title}' was created {int(now - last_created)}s ago. "
f"Wait {remaining}s or use a different title. Last URL: {_last_gdoc_url or 'unknown'}")
try:
import sys as _sys
_dash = os.path.dirname(os.path.abspath(__file__))
if _dash not in _sys.path:
_sys.path.insert(0, _dash)
from codec_gdocs import create_google_doc
doc_url = create_google_doc(title, content)
if doc_url:
_last_gdoc_url = doc_url
_gdoc_created[title_hash] = now
return f"Google Doc created: {doc_url}"
return "Google Docs error: doc creation returned None"
except Exception as e:
return f"Google Docs error: {e}"
# ── shell_execute: macOS sandbox + consent (P-030 part 2) ──
# Owner decision D-17, 29 Oct 2026 (docs/CREW-SHELL-SANDBOX-DESIGN.md). The
# crew / custom-agent shell runs `/bin/sh -c <cmd>` inside sandbox-exec with a
# profile from codec_sandbox's template: no network, the home folder's secret
# folders unreadable, writes only in the output folder and /private/tmp.
# Commands codec_config.is_dangerous flags also need the owner's OK through
# codec_consent. Where sandbox-exec does not exist the tool refuses: it never
# runs a command unsandboxed.
_SHELL_SANDBOX_EXEC = "/usr/bin/sandbox-exec"
_SHELL_TIMEOUT_SECONDS = 30
# The approval card shows the first 200 characters of the request
# (codec_consent.local_consent_ok), so a flagged command must fit in them.
_SHELL_CARD_CHARS = 200
_SHELL_PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin"
_SHELL_OUTPUT_SUBDIR = os.path.join("codec-workspace", "agent-shell")
_SHELL_DENIED_HINT = ("\n[sandbox] Something was refused: shell commands here have no network, "
"cannot read secret folders and write only in {out} and /tmp.")
def _shell_sandbox_available() -> bool:
return os.path.exists(_SHELL_SANDBOX_EXEC)
def _shell_refused(reason: str, cmd: str) -> None:
"""One audit line for a refusal by the tool itself. Lengths only, never
the command text (P-030 part 4)."""
_audit("shell_blocked", tool="shell_execute", outcome="denied",
reason=reason, cmd_chars=len(cmd))
def _shell_is_flagged(cmd: str) -> bool:
try:
from codec_config import is_dangerous
return bool(is_dangerous(cmd))
except Exception:
return True # cannot check: treat as flagged (asks, never runs blind)
def _run_shell_sandboxed(cmd: str, home: str, output_dir: str, profile_dir: str):
"""Run `/bin/sh -c cmd` under sandbox-exec with a codec_sandbox profile.
Returns (stdout, stderr, timed_out). Raises on a setup failure."""
import signal
import subprocess
import tempfile
from codec_sandbox import _write_sandbox_profile
os.makedirs(profile_dir, exist_ok=True)
# One profile file per call, in a folder the sandbox cannot write
# (~/.codec by default): no race between two agents, no edit by a command.
fd, profile_path = tempfile.mkstemp(prefix="crew_shell-", suffix=".sb", dir=profile_dir)
os.close(fd)
try:
_write_sandbox_profile(allow_network=False, home=home, profile_path=profile_path,
output_dir=output_dir, allow_shell=True)
env = {"PATH": _SHELL_PATH, "HOME": home, "TMPDIR": "/private/tmp",
"LANG": "en_US.UTF-8", "TERM": "dumb"}
p = subprocess.Popen([_SHELL_SANDBOX_EXEC, "-f", profile_path, "/bin/sh", "-c", cmd],
cwd=output_dir, env=env, stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
errors="replace", start_new_session=True)
try:
out, err = p.communicate(timeout=_SHELL_TIMEOUT_SECONDS)
return out, err, False
except subprocess.TimeoutExpired:
try:
os.killpg(p.pid, signal.SIGKILL) # the command and everything it started
except OSError:
pass
try:
out, err = p.communicate(timeout=5)
except subprocess.TimeoutExpired: # a child that left the group holds the pipes
out, err = "", ""
return out, err, True
finally:
try:
os.unlink(profile_path)
except OSError:
pass
def _shell_execute(cmd: str, *, home: str = None, output_dir: str = None,
profile_dir: str = None) -> str:
"""The crew / custom-agent shell tool. `home`, `output_dir` and
`profile_dir` default to the real home, ~/codec-workspace/agent-shell and
~/.codec; the real-sandbox tests pass temp folders instead."""
cmd = (cmd or "").strip()
if not cmd:
_shell_refused("empty", cmd)
return "BLOCKED: no command given."
if not _shell_sandbox_available():
_shell_refused("no_sandbox", cmd)
return ("BLOCKED: shell_execute runs commands only inside the macOS sandbox "
f"({_SHELL_SANDBOX_EXEC}), which this machine does not have. Command not run.")
home = os.path.realpath(home or os.path.expanduser("~"))
output_dir = os.path.realpath(output_dir or os.path.join(home, _SHELL_OUTPUT_SUBDIR))
profile_dir = profile_dir or os.path.join(home, ".codec")
if _shell_is_flagged(cmd):
import codec_consent
if not codec_consent.gate_enabled():
_shell_refused("consent_gate_off", cmd)
return ("BLOCKED: dangerous command pattern detected and the consent gate is off "
"(CONSENT_GATE_ENABLED=false), so it cannot be approved. Command not executed.")
if len(cmd) > _SHELL_CARD_CHARS:
_shell_refused("too_long_to_show", cmd)
return (f"BLOCKED: this command is flagged as dangerous and is longer than the "
f"{_SHELL_CARD_CHARS} characters the approval card can show. Command not executed.")
# One audit line (local_consent_decision, asked_from=crew) either way.
if not codec_consent.local_consent_ok("shell_execute", cmd, "crew"):
return ("BLOCKED: this command is flagged as dangerous and needs the owner's OK; "
"it was declined or not answered. Command not executed.")
# Print command for transparency before execution
log.info(f"[shell_execute] Running in the sandbox: {cmd[:200]}")
try:
stdout, stderr, timed_out = _run_shell_sandboxed(cmd, home, output_dir, profile_dir)
except Exception as e:
_shell_refused("sandbox_setup_failed", cmd)
return f"BLOCKED: the sandbox could not be set up ({type(e).__name__}). Command not executed."
if timed_out:
return f"Command timed out ({_SHELL_TIMEOUT_SECONDS}s); it and everything it started were stopped."
out = stdout or ""
if len(out) > 5000:
out = out[:5000] + f"\n[TRUNCATED: stdout was {len(stdout)} chars]"
if stderr:
err = stderr
if len(err) > 2000:
err = err[:2000] + f"\n[TRUNCATED: stderr was {len(stderr)} chars]"
out += "\nSTDERR: " + err
if "Operation not permitted" in stderr:
out += _SHELL_DENIED_HINT.format(out=output_dir)
return out or "(no output)"
BUILTIN_TOOLS = [
Tool("web_search", "Search Google for any query. Input: search query string.", _web_search),
Tool("web_fetch", "Fetch and read a web page. Input: URL string.", _web_fetch),
Tool("file_read", "Read a file from disk. Input: file path.", _file_read),
Tool("file_write", "Write a file. Input: 'path: /path\\ncontent: text'", _file_write),
Tool("google_docs_create","Create a Google Doc. Input: 'title: Title\\ncontent: body text'", _google_docs_create),
Tool("shell_execute", "Run a shell command in a macOS sandbox: no network, secret folders unreadable, "
"writes only in ~/codec-workspace/agent-shell (the working folder) and /tmp. "
"Dangerous commands need the owner's OK. Input: cmd", _shell_execute),
]
# ═══════════════════════════════════════════════════════════════
# SKILL LOADER (lazy via SkillRegistry)
# ═══════════════════════════════════════════════════════════════
from codec_skill_registry import SkillRegistry
_agents_registry = SkillRegistry(SKILLS_DIR, USER_SKILLS_DIR)
def _make_lazy_fn(registry: "SkillRegistry", skill_name: str):
"""Return a callable that lazy-loads the skill module on first call."""
def _lazy_run(input_str: str) -> str:
from codec_skill_switches import is_off, off_message
if is_off(skill_name): # P3.6: switched off on the Skills page
return off_message(skill_name)
mod = registry.load(skill_name)
if mod is None or not hasattr(mod, "run"):
return f"Skill '{skill_name}' could not be loaded."
return mod.run(input_str)
return _lazy_run
def load_skill_tools() -> List[Tool]:
"""Scan skills and return Tool objects with lazy-loaded run functions.
Only metadata is parsed at startup (via AST); the actual module
import happens on first invocation of each tool.
"""
_agents_registry.scan()
tools = []
for name in _agents_registry.names():
desc = _agents_registry.get_description(name)
tools.append(Tool(
name=name,
description=desc,
fn=_make_lazy_fn(_agents_registry, name),
))
print(f"[Agents] Registered {len(tools)} skill tools (lazy)")
return tools
def get_all_tools() -> List[Tool]:
return BUILTIN_TOOLS + load_skill_tools()
# ── Custom-agent tool lists (P-030 part 3, owner decision D-17) ──
# A saved custom agent may name only tools CODEC knows: the built-in tools and
# the scanned skills (the same list /api/agents/tools shows). Saving refuses an
# unknown name; an agent saved before this check still loads, with its unknown
# names left out and one audit line (once per agent and dropped set, per process).
_DROPPED_AUDITED: set = set()
_DROPPED_LOCK = threading.Lock()
def known_tool_names() -> set:
_agents_registry.scan()
return {t.name for t in BUILTIN_TOOLS} | set(_agents_registry.names())
def known_tools_only(agent_id, tools, known: Optional[set] = None) -> List[str]:
"""Return the names in `tools` that CODEC knows, in their saved order."""
if known is None:
known = known_tool_names()
raw = tools if isinstance(tools, list) else ([] if tools in (None, "") else [tools])
kept = [t for t in raw if isinstance(t, str) and t in known]
dropped = len(raw) - len(kept)
if dropped:
key = (str(agent_id), tuple(sorted(repr(t)[:120] for t in raw if t not in kept)))
with _DROPPED_LOCK:
first = key not in _DROPPED_AUDITED
_DROPPED_AUDITED.add(key)
if first:
_audit("custom_agent_tools_dropped", outcome="warning",
agent_id=str(agent_id)[:100], dropped=dropped, kept=len(kept))
return kept
# ═══════════════════════════════════════════════════════════════
# AGENT
# ═══════════════════════════════════════════════════════════════
@dataclass
class Agent:
name: str
role: str
tools: List[Tool] = field(default_factory=list)
max_tool_calls: int = 5
thinking: bool = False # Keep off by default — adds latency; crews can override
verbose: bool = True
# Phase 1 Step 3 §2.2 stuck-detection ring buffer.
# Per-agent: each Agent instance tracks its own (tool_name, args_hash)
# window of last M=5 calls. When the same key appears N=3 times,
# _handle_stuck() fires (warn first, escalate at N+2 = 5).
# Defaults loaded from ~/.codec/config.json: stuck.{repeat_threshold,
# window, escalation_action} on first Agent.run call. Cached as
# instance attrs to avoid re-reading config every loop iteration.
_recent_calls: List[tuple] = field(default_factory=list, repr=False)
_stuck_warned_keys: set = field(default_factory=set, repr=False)
_stuck_escalated_keys: set = field(default_factory=set, repr=False)
# ── A-7 (PR-3D-a): ReAct-loop helpers extracted from run() ──────────────
@staticmethod
def _parse_action(text: str) -> tuple:
"""Pure parse of the ReAct text protocol. Returns (tool, final_text):
tool = (name, input) if a well-formed TOOL:/INPUT: block exists, else None
final_text = text after the LAST 'FINAL:' (stripped) if present, else None
Both may be set; run() applies TOOL-before-FINAL precedence with its own
tool-budget state (unchanged)."""
m = re.search(r'TOOL:\s*(\S+)\s*\nINPUT:\s*([\s\S]*?)(?=\nTOOL:|\nFINAL:|$)', text)
tool = (m.group(1).strip(), m.group(2).strip()) if m else None
final_text = text.rsplit("FINAL:", 1)[1].strip() if "FINAL:" in text else None
return tool, final_text
@staticmethod
def _validate_tool_call(tool_name: str, tool_input: str):
"""Return a rejection message to feed back to the LLM if the tool call is
malformed, else None. Pure — the caller does the logging + message append."""
if not tool_name:
return "Empty tool name rejected. Try again or use FINAL:."
if len(tool_name) > _MAX_TOOL_NAME_LEN:
return "Tool name too long (max 100 chars). Try again or use FINAL:."
if not _VALID_TOOL_NAME_RE.match(tool_name):
return (f"Tool name '{tool_name[:60]}' contains invalid characters. "
f"Only alphanumeric, underscore, hyphen, and dot are allowed. "
f"Try again or use FINAL:.")
if len(tool_input) > _MAX_TOOL_INPUT_LEN:
return (f"Tool input too long ({len(tool_input)} chars, max "
f"{_MAX_TOOL_INPUT_LEN}). Try again or use FINAL:.")
return None
async def _execute_tool_with_hooks(self, tool, tool_name: str, tool_input: str) -> str:
"""Run `tool` through run_with_hooks in a worker thread, propagating
contextvars (incl. _correlation_id_var) so audits fired inside the tool
inherit the agent/crew cid (asyncio doesn't copy them automatically).
Applies the Step-2 veto contract. Returns the tool result string. Stuck
detection is applied by the caller AFTER the tool_result audit, to keep
that audit's result_len reporting the pre-stuck length."""
loop = asyncio.get_event_loop()
_agent_cid = _correlation_id_var.get() or _new_correlation_id()
_agent_name = self.name
_tool_name_local = tool_name
_tool_input_local = tool_input
_real_tool = tool
def _run_tool_with_hooks():
def _inner(t, _c):
# P-021: memory writes made by this tool call are agent-origin.
with codec_fence.caller("agent"):
return _real_tool.run(t)
return run_with_hooks(
tool_name=_tool_name_local,
task=_tool_input_local,
context="",
transport="crew",
agent=_agent_name,
correlation_id=_agent_cid,
invoke=_inner,
)
ctx = contextvars.copy_context()
try:
result = await asyncio.wait_for(
loop.run_in_executor(None, ctx.run, _run_tool_with_hooks),
timeout=_TOOL_CALL_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError:
# L1 / SR-61: surface as a tool-error string so the agent's ReAct
# loop can recover (try a different tool / FINAL) instead of hanging.
_audit("tool_result", tool=tool_name, agent=self.name, outcome="timeout",
error=f"tool exceeded {_TOOL_CALL_TIMEOUT_SECONDS}s budget")
return _ToolNote(f"Tool '{tool_name}' timed out after "
f"{_TOOL_CALL_TIMEOUT_SECONDS}s. Try a different approach or "
f"give your FINAL answer.")
if isinstance(result, HookVeto):
result = _ToolNote(f"Tool '{tool_name}' was vetoed by plugin "
f"'{result.plugin_name}': {result.reason}")
return result
async def run(self, task: str, context: str = "", callback: Optional[Callable] = None) -> str:
# Inherit correlation_id from the surrounding Crew if there is one;
# otherwise (e.g. run_custom_agent — solo agent path) generate our own.
# We don't reset the token: the asyncio.Task that owns this context goes
# away after the request, and any nested tool calls inside this same
# agent run should see the same cid (that's the whole point).
if _correlation_id_var.get() is None:
_correlation_id_var.set(_new_correlation_id())
self._gdoc_url = None # Capture real URL from google_docs_create tool
tool_desc = "\n".join(f" - {t.name}: {t.description}" for t in self.tools) or " (no tools)"
system = f"""{self.role}
You have access to these tools:
{tool_desc}
To use a tool, respond EXACTLY in this format (nothing else on those lines):
TOOL: tool_name
INPUT: the input for the tool
To give your final answer, respond EXACTLY in this format:
FINAL: your complete answer here
Rules:
- Use tools to gather information you need.
- You may use up to {self.max_tool_calls} tool calls total.
- Think step by step before choosing a tool.
- After each tool result, decide: need more info → another TOOL, or ready → FINAL.
- ALWAYS end with FINAL: when you have enough information."""
messages = [{"role": "system", "content": system}]
if context:
messages.append({"role": "user", "content": f"Context from previous step:\n{context}"})
messages.append({"role": "user", "content": f"Your task:\n{task}"})
tool_calls_made = 0
last_response = ""
import codec_llm
for _ in range(self.max_tool_calls + 3):
# A-12 (PR-3E-async): codec_llm.acall (async, raise_on_error) replaces
# the inline _async_http.post + parse. Queue (MEDIUM) + the reused
# client stay here; the except keeps the "LLM error" early-exit.
await llm_queue.acquire(Priority.MEDIUM)
try:
response = await codec_llm.acall(
messages, base_url=_qwen_base(), model=_qwen_model(),
max_tokens=4000, temperature=0.7, enable_thinking=self.thinking,
http=_async_http, raise_on_error=True,
)
except Exception as e:
return f"LLM error: {e}"
finally:
await llm_queue.release(Priority.MEDIUM)
# Strip thinking tags (codec_llm already strips; kept harmless).
response = re.sub(r'<think>[\s\S]*?</think>', '', response).strip()
last_response = response
if self.verbose:
print(f"[{self.name}] {response[:200]}…")
# ── Parse the ReAct protocol (A-7: extracted to _parse_action) ──
# TOOL is checked before FINAL so a response with both doesn't loop.
parsed_tool, final = self._parse_action(response)
# FINAL answer (final = text after the LAST 'FINAL:'; skips quoted prompt).
if final is not None and not (parsed_tool and tool_calls_made < self.max_tool_calls):
# Guard: if agent has google_docs_create but never called it, reject the FINAL
has_docs_tool = any(t.name == "google_docs_create" for t in self.tools)
called_docs = any("google_docs_create" in str(m_msg.get("content", "")) for m_msg in messages if m_msg["role"] == "user" and "Tool result from" in str(m_msg.get("content", "")))
if has_docs_tool and not called_docs and tool_calls_made == 0:
messages.append({"role": "assistant", "content": response})
messages.append({"role": "user", "content": (
"REJECTED: You must use the google_docs_create tool BEFORE giving a FINAL answer. "
"Do NOT invent URLs. Call the tool now with your content."
)})
continue
# Guard: replace any fabricated Google Docs URL with the real one
if self._gdoc_url and "docs.google.com" in final:
final = re.sub(
r'https://docs\.google\.com/document/d/[A-Za-z0-9_-]+(?:/edit)?(?:\?[^\s)]*)?',
self._gdoc_url,
final
)
log.info(f"[{self.name}] Replaced fabricated URL with real: {self._gdoc_url}")
elif self._gdoc_url and "docs.google.com" not in final:
# LLM didn't even include a URL — append it
final = f"{final}\n{self._gdoc_url}"
log.info(f"[{self.name}] Appended real URL to FINAL: {self._gdoc_url}")
if callback:
await _safe_cb(callback, {"agent": self.name, "status": "complete", "preview": final[:200]})
return final
# TOOL call (parsed_tool from _parse_action above)
if parsed_tool and tool_calls_made < self.max_tool_calls:
tool_name, tool_input = parsed_tool
# ── Input validation (A-7: extracted to _validate_tool_call) ──
rejection = self._validate_tool_call(tool_name, tool_input)
if rejection:
log.warning("Rejected malformed tool call: %s", tool_name[:120])
messages.append({"role": "assistant", "content": response})
messages.append({"role": "user", "content": rejection})
continue
tool = next((t for t in self.tools if t.name == tool_name), None)
if tool:
if callback:
await _safe_cb(callback, {
"agent": self.name, "status": "tool_call",
"tool": tool_name, "input": tool_input[:100]
})
if self.verbose:
print(f"[{self.name}] → {tool_name}({tool_input[:80]}…)")
_audit("tool_call", agent=self.name, tool=tool_name,
input=tool_input[:200])
# A-7: executor + run_with_hooks + Step-2 veto extracted to
# _execute_tool_with_hooks (propagates contextvars/cid into the
# worker thread; veto string becomes the tool result per §4.4).
result = await self._execute_tool_with_hooks(tool, tool_name, tool_input)
tool_calls_made += 1
_audit("tool_result", agent=self.name, tool=tool_name,
result_len=len(result))
# P-021: a tool's output is data for the model, never instructions.
# Fenced before the stuck banner, so CODEC's own banners and the
# owner's escalation answer stay outside the fence.
if not isinstance(result, _ToolNote):
result = codec_fence.fence(result, f"Result of the {tool_name} tool")
# Phase 1 Step 3 §2.2 — stuck detection.
# Run in the executor so the worker thread can call
# ask_user.ask() synchronously without blocking the
# event loop on escalation. The helper returns a
# (possibly modified) result string with a warning
# banner injected, OR an escalation answer string if
# the user told the agent how to proceed.
if _stuck_enabled():
stuck_ctx = contextvars.copy_context()
result = await asyncio.get_event_loop().run_in_executor(
None, stuck_ctx.run,
self._handle_stuck_post_tool, tool_name,
tool_input, result)
# Capture real Google Docs URL from tool result
if tool_name == "google_docs_create" and "docs.google.com" in result:
url_match = re.search(r'https://docs\.google\.com/document/d/[A-Za-z0-9_-]+/edit', result)
if url_match:
self._gdoc_url = url_match.group(0)
log.info(f"[{self.name}] Captured real GDoc URL: {self._gdoc_url}")
if self.verbose:
print(f"[{self.name}] ← {result[:150]}…")
messages.append({"role": "assistant", "content": response})
messages.append({
"role": "user",
"content": (
f"Tool result from {tool_name}:\n{result}\n\n"
f"Continue. Use another TOOL or respond with FINAL: "
f"({self.max_tool_calls - tool_calls_made} tool calls remaining)."
)
})
else:
messages.append({"role": "assistant", "content": response})
messages.append({
"role": "user",
"content": f"Tool '{tool_name}' not found. Available: {', '.join(t.name for t in self.tools)}. Try again or use FINAL:."
})
else:
# No TOOL/FINAL — treat as final
if callback:
await _safe_cb(callback, {"agent": self.name, "status": "complete", "preview": response[:200]})
return response
return last_response
# ── Phase 1 Step 3 §2.2 — stuck detection ──────────────────────────
def _handle_stuck_post_tool(self, tool_name: str, tool_input: str,
result: str) -> str:
"""Called from Agent.run after each tool result. Records the call
in the per-agent ring buffer, detects N=3 / N+2=5 repeats, emits
stuck_warning / stuck_escalated audit events, and either injects
a soft warning into the result OR invokes ask_user for explicit
user direction.
Runs in a worker thread (via run_in_executor wrapping in
Agent.run) so that ask_user.ask()'s threading.Event.wait()
doesn't block the asyncio event loop.
Returns the (possibly modified) result string the agent's ReAct
loop will see as the tool result.
"""
try:
window, threshold, escalation_action = _load_stuck_config()
args_hash = hashlib.sha1(
(tool_input or "").encode("utf-8", errors="replace")
).hexdigest()[:8]
key = (tool_name, args_hash)
self._recent_calls.append(key)
if len(self._recent_calls) > window:
self._recent_calls = self._recent_calls[-window:]
repeat_count = self._recent_calls.count(key)
cid = _correlation_id_var.get()
if repeat_count >= threshold + 2 and key not in self._stuck_escalated_keys:
# Escalation: invoke ask_user (synchronously — we're in
# a worker thread). Per §2.3.
self._stuck_escalated_keys.add(key)
action = escalation_action
from codec_audit import log_event as _le
try:
_le(
"stuck_escalated", "codec-agents",
f"Agent {self.name} stuck calling {tool_name}",
extra={"tool": tool_name,
"repeat_count": repeat_count,
"agent": self.name,
"action": action},
outcome="warning", level="warning",
tool=tool_name,
correlation_id=cid,
)
except Exception as e:
log.warning("[stuck] escalation audit failed: %s", e)
if action == "abort":
raise RuntimeError(
f"Stuck-abort: agent '{self.name}' called {tool_name} "
f"{repeat_count} times with the same args.")
if action == "warn_only":
return result + (
f"\n\n[STUCK ESCALATED] Agent has called {tool_name} "
f"{repeat_count} times with the same args; warn_only "
f"mode — proceed with caution.")
# Default: ask_user
try:
from codec_ask_user import ask
user_directive = ask(
question=(
f"Agent '{self.name}' has called {tool_name} "
f"{repeat_count} times with the same args and keeps "
f"getting the same result. How should I proceed?"
),
options=["Try a different approach", "Abandon the task",
"Continue anyway"],
agent=self.name,
asked_from="crew",
)
except Exception as e:
log.warning("[stuck] ask_user invoke failed: %s", e)
user_directive = "(ask_user failed — agent should self-recover)"
return result + (
f"\n\n[STUCK — user said]: {user_directive}\n"
f"Adjust your strategy based on this directive.")
if repeat_count >= threshold and key not in self._stuck_warned_keys:
# Soft warning: inject a banner into the result. The LLM
# will see this and (hopefully) try a different tool.
self._stuck_warned_keys.add(key)
from codec_audit import log_event as _le
try:
_le(
"stuck_warning", "codec-agents",
f"Agent {self.name} repeating {tool_name}",
extra={"tool": tool_name,
"repeat_count": repeat_count,
"agent": self.name},
outcome="warning", level="warning",
tool=tool_name,
correlation_id=cid,
)
except Exception as e:
log.warning("[stuck] warning audit failed: %s", e)
return result + (
f"\n\n⚠ [STUCK WARNING] You've called {tool_name} "
f"{repeat_count} times with the same args. Try a "
f"different tool, different inputs, or wrap up with "
f"FINAL: — repeating won't help.")
return result
except Exception as e:
log.warning("[stuck] handler failed (non-fatal): %s", e)
return result
def _stuck_enabled() -> bool:
"""Read STUCK_DETECTION_ENABLED env var. Default true."""
val = (os.environ.get("STUCK_DETECTION_ENABLED") or "true").strip().lower()
return val not in ("false", "0", "no", "off")
def _load_stuck_config() -> tuple: