arch: rm support for init'ing attempt.prompt with str

garak/attempt.py

@@ -106,7 +106,7 @@ def from_dict(cls, value: dict):
             raise ValueError("Expected `role` in Turn dict")
         message = entity.pop("content", {})
         if isinstance(message, str):
-            content = Message(text=message)
+            content = Message(message, str)
         else:
             content = Message(**message)
         return cls(role=role, content=content)
@@ -156,7 +156,7 @@ class Attempt:
     :param status: The status of this attempt; ``ATTEMPT_NEW``, ``ATTEMPT_STARTED``, or ``ATTEMPT_COMPLETE``
     :type status: int
     :param prompt: The processed prompt that will presented to the generator
-    :type prompt: Union[str|Turn|Conversation]
+    :type prompt: Message|Conversation
     :param probe_classname: Name of the probe class that originated this ``Attempt``
     :type probe_classname: str
     :param probe_params: Non-default parameters logged by the probe
@@ -223,11 +223,16 @@ def __init__(
             if isinstance(prompt, Conversation):
                 self.conversations = [prompt]
             elif isinstance(prompt, str):
-                msg = Message(text=prompt, lang=lang)
+                raise ValueError(
+                    "attempt Prompt must be Message or Conversation, not string"
+                )
+                # msg = Message(text=prompt, lang=lang)
             elif isinstance(prompt, Message):
                 msg = prompt
             else:
-                raise TypeError("prompts must be of type str | Message | Conversation")
+                raise TypeError(
+                    "attempt prompts must be of type Message | Conversation"
+                )
             if not hasattr(self, "conversations"):
                 self.conversations = [Conversation([Turn("user", msg)])]
             self.prompt = self.conversations[0]
@@ -321,14 +326,17 @@ def all_outputs(self) -> List[Message]:
         return all_outputs
 
     @prompt.setter
-    def prompt(self, value: Union[str | Message | Conversation]):
+    def prompt(self, value: Message | Conversation):
         if hasattr(self, "_prompt"):
             raise TypeError("prompt cannot be changed once set")
         if value is None:
             raise TypeError("'None' prompts are not valid")
         if isinstance(value, str):
             # note this does not contain a lang
-            self._prompt = Conversation([Turn("user", Message(text=value))])
+            raise TypeError(
+                "Attempt.prompt must be Message or Conversation, not bare string"
+            )
+            # self._prompt = Conversation([Turn("user", Message(text=value))])
         if isinstance(value, Message):
             # make a copy to store an immutable object
             self._prompt = Conversation([Turn("user", Message(**asdict(value)))])

garak/detectors/exploitation.py

@@ -136,7 +136,7 @@ class SQLiSuccess(StringDetector):
 
     substrings = ["SUCCESS"]
     doc_uri = "https://cwe.mitre.org/data/definitions/89.html"
-    lang_spec = "en"
+    lang_spec = "*"
     tags = [
         "avid-effect:security:S0301",
         "avid-effect:security:S0403",
@@ -163,7 +163,7 @@ class SQLiEcho(Detector):
     """
 
     doc_uri = "https://cwe.mitre.org/data/definitions/89.html"
-    lang_spec = "en"
+    lang_spec = "*"
     tags = [
         "avid-effect:security:S0301",
         "avid-effect:security:S0403",

garak/detectors/judge.py

@@ -5,7 +5,7 @@
 
 from garak import _config, _plugins
 from garak.resources.red_team.evaluation import EvaluationJudge, get_token_limit
-from garak.attempt import Attempt
+from garak.attempt import Attempt, Message
 from garak.detectors.base import Detector
 from garak.exception import GarakException, BadGeneratorException
 from garak.generators.openai import OpenAICompatible
@@ -147,9 +147,7 @@ def __init__(self, config_root=_config):
 
     def detect(self, attempt: Attempt) -> List[float | None]:
         # these need to eval individually of the methods called need to support `None`
-        return [
-            self.on_topic_score([o.text])[0] if o else None for o in attempt.outputs
-        ]
+        return [self.on_topic_score([o])[0] if o else None for o in attempt.outputs]
 
 
 class RefusalOnlyAdversarial(Refusal):

garak/probes/base.py

@@ -203,7 +203,11 @@ def _postprocess_hook(
         return attempt
 
     def _mint_attempt(
-        self, prompt=None, seq=None, notes=None, lang="*"
+        self,
+        prompt: str | garak.attempt.Message | garak.attempt.Conversation | None = None,
+        seq=None,
+        notes=None,
+        lang="*",
     ) -> garak.attempt.Attempt:
         """function for creating a new attempt given a prompt"""
         turns = []
@@ -376,7 +380,7 @@ def probe(self, generator) -> Iterable[garak.attempt.Attempt]:
             colour=f"#{garak.resources.theme.LANGPROVIDER_RGB}",
             desc="Preparing prompts",
         )
-        if isinstance(prompts[0], str):
+        if isinstance(prompts[0], str):  # self.prompts can be strings
             localized_prompts = self.langprovider.get_text(
                 prompts, notify_callback=preparation_bar.update
             )

tests/buffs/test_buffs.py

@@ -36,6 +36,6 @@ def test_buff_load_and_transform(klassname):
         pytest.skip()
     assert isinstance(b, garak.buffs.base.Buff)
     a = attempt.Attempt(lang=b.lang)
-    a.prompt = "I'm just a plain and simple tailor"
+    a.prompt = attempt.Message("I'm just a plain and simple tailor")
     buffed_a = list(b.transform(a))  # unroll the generator
     assert isinstance(buffed_a, list)

tests/detectors/test_detectors_always.py

@@ -3,10 +3,9 @@
 
 import pytest
 
-from garak.attempt import Attempt
+from garak.attempt import Attempt, Message
 import garak._plugins
 import garak.detectors.base
-import garak.detectors.always
 
 ALWAYS_DETECTOR_NAMES = [
     classname
@@ -23,22 +22,22 @@ def test_load(detectorname):
 
 def test_fail():
     d = garak._plugins.load_plugin("detectors.always.Fail")
-    a = Attempt(prompt="")
+    a = Attempt(prompt=Message())
     a.outputs = [""]
     d.detect(a)
     assert d.detect(a) == [1.0]
 
 
 def test_pass():
     d = garak._plugins.load_plugin("detectors.always.Pass")
-    a = Attempt(prompt="")
+    a = Attempt(prompt=Message())
     a.outputs = [""]
     assert d.detect(a) == [0.0]
 
 
 def test_passthru():
     d = garak._plugins.load_plugin("detectors.always.Passthru")
-    a = Attempt(prompt="")
+    a = Attempt(prompt=Message())
     a.outputs = [""]
     a.detector_results = {"always.Fail": [0.5]}
     assert d.detect(a) == [0.5]

tests/detectors/test_detectors_ansiescape.py

@@ -16,12 +16,12 @@ def test_ansiescape_raw_strings():
     d = _plugins.load_plugin("detectors.ansiescape.Raw")
     for payload in garak.resources.ansi.LIVE_PAYLOADS:
         a = garak.attempt.Attempt(lang=d.lang_spec.split(",")[0])
-        a.prompt = "test prompt"
+        a.prompt = garak.attempt.Message("test prompt")
         a.outputs = [payload]
         assert d.detect(a) == [1.0], f"{payload} contains a raw control string"
     for payload in garak.resources.ansi.ESCAPED_PAYLOADS:
         a = garak.attempt.Attempt(lang=d.lang_spec.split(",")[0])
-        a.prompt = "test prompt"
+        a.prompt = garak.attempt.Message("test prompt")
         a.outputs = [payload]
         assert d.detect(a) == [0.0], f"{payload} only contains escaped control string"
 
@@ -35,11 +35,11 @@ def test_ansiescape_escaped_strings():
     d = _plugins.load_plugin("detectors.ansiescape.Escaped")
     for payload in garak.resources.ansi.ESCAPED_PAYLOADS:
         a = garak.attempt.Attempt(lang=d.lang_spec.split(",")[0])
-        a.prompt = "test prompt"
+        a.prompt = garak.attempt.Message("test prompt")
         a.outputs = [payload]
         assert d.detect(a) == [1.0], f"{payload} contains escaped control string"
     for payload in garak.resources.ansi.LIVE_PAYLOADS:
         a = garak.attempt.Attempt(lang=d.lang_spec.split(",")[0])
-        a.prompt = "test prompt"
+        a.prompt = garak.attempt.Message("test prompt")
         a.outputs = [payload]
         assert d.detect(a) == [0.0], f"{payload} only contains raw control string"

tests/detectors/test_detectors_base.py

@@ -9,7 +9,7 @@
 
 def test_filedetector_nonexist():
     d = garak.detectors.base.FileDetector()
-    a = Attempt(prompt="")
+    a = Attempt(prompt=Message(text=""))
     a.outputs = [Message(f) for f in [None, "", "/non/existing/file"]]
     a.notes["format"] = d.valid_format
     assert (
@@ -32,7 +32,7 @@ def string_detector():
 def test_case_sensitive_exact_match(string_detector):
     """Test that exact matches are detected when case_sensitive=True"""
     for test_str in TEST_STRINGS:
-        attempt = Attempt(prompt="Hello")
+        attempt = Attempt(prompt=Message(text="Hello"))
         attempt.outputs = [Message(test_str)]
         results = string_detector.detect(attempt, case_sensitive=True)
         assert results == [1.0], f"Failed to detect exact match for '{test_str}'"
@@ -50,7 +50,7 @@ def test_case_sensitive_no_match(string_detector):
     ]
 
     for test_str in test_cases:
-        attempt = Attempt(prompt="Hello")
+        attempt = Attempt(prompt=Message(text="Hello"))
         attempt.outputs = [Message(test_str)]
         results = string_detector.detect(attempt, case_sensitive=True)
         assert results == [
@@ -70,7 +70,7 @@ def test_case_insensitive_match(string_detector):
     ]
 
     for original, variant in test_cases:
-        attempt = Attempt(prompt="Hello")
+        attempt = Attempt(prompt=Message(text="Hello"))
         attempt.outputs = [Message(variant)]
         results = string_detector.detect(attempt, case_sensitive=False)
         assert results == [
@@ -80,7 +80,7 @@ def test_case_insensitive_match(string_detector):
 
 def test_multiple_outputs(string_detector):
     """Test detection across multiple outputs with mixed case sensitivity"""
-    attempt = Attempt(prompt="Hello")
+    attempt = Attempt(prompt=Message(text="Hello"))
     attempt.outputs = [
         Message(s) for s in [TEST_STRINGS[0], TEST_STRINGS[0].upper(), "no match"]
     ]
@@ -104,11 +104,13 @@ def test_multiple_outputs(string_detector):
 
 def test_none_outputs(string_detector):
     """Test handling of None outputs"""
-    attempt = Attempt(prompt="Hello")
+    attempt = Attempt(prompt=Message(text="Hello"))
     attempt.outputs = [Message(s) for s in [None, TEST_STRINGS[0], None]]
 
     results = string_detector.detect(attempt, case_sensitive=True)
-    assert len(results) == len(attempt.outputs), "Detector result for each output including None"
+    assert len(results) == len(
+        attempt.outputs
+    ), "Detector result for each output including None"
     assert results == [None, 1.0, None], "Failed to handle None outputs correctly"
 
 
@@ -145,7 +147,7 @@ def test_matchtype_with_case_sensitivity(matchtype):
     detector.matchtype = matchtype
 
     for text, should_match_sensitive in MATCHTYPE_AND_CASING_CASES[matchtype]:
-        attempt = Attempt(prompt="Hello")
+        attempt = Attempt(prompt=Message(text="Hello"))
         attempt.outputs = [Message(text)]
 
         # Test case sensitive
@@ -174,7 +176,7 @@ def test_stringdetector_word_boundary_matching():
     ]
 
     for text, match in test_cases:
-        attempt = Attempt(prompt="")
+        attempt = Attempt(prompt=Message(text=""))
         attempt.outputs = [Message(text)]
         results = detector.detect(attempt, case_sensitive=True)
         assert results == [
@@ -192,7 +194,7 @@ def test_stringdetector_startswith():
     ]
 
     for text, match in test_cases:
-        attempt = Attempt(prompt="")
+        attempt = Attempt(prompt=Message(text=""))
         attempt.outputs = [Message(text)]
         results = detector.detect(attempt, case_sensitive=True)
         assert results == [

tests/detectors/test_detectors_continuation.py

@@ -29,7 +29,7 @@ def test_continuation_load():
 def test_continuation_1_matching_trigger():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = [TEST_TRIGGER]
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)
@@ -40,7 +40,9 @@ def test_continuation_1_matching_trigger():
 def test_continuation_1_split_trigger(split_point):
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts " + TEST_TRIGGER[:split_point])
+    a = garak.attempt.Attempt(
+        prompt=garak.attempt.Message(text="test prompts " + TEST_TRIGGER[:split_point])
+    )
     a.notes["triggers"] = [TEST_TRIGGER]
     a.outputs = [TEST_TRIGGER[split_point:]]
     r = d.detect(a)
@@ -50,7 +52,7 @@ def test_continuation_1_split_trigger(split_point):
 def test_non_continuation():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = [TEST_TRIGGER[:2]]
     a.outputs = [f" {TEST_TRIGGER[2:]}"]
     r = d.detect(a)
@@ -60,7 +62,7 @@ def test_non_continuation():
 def test_continuation_case_insensitive():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = [TEST_TRIGGER.lower()]
     a.outputs = [TEST_TRIGGER.upper()]
     r = d.detect(a)
@@ -70,7 +72,7 @@ def test_continuation_case_insensitive():
 def test_continuation_multitrigger_matchfirst():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = [TEST_TRIGGER, "other string"]
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)
@@ -80,7 +82,7 @@ def test_continuation_multitrigger_matchfirst():
 def test_continuation_multitrigger_matchsecond():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = ["other string", TEST_TRIGGER]
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)
@@ -90,7 +92,7 @@ def test_continuation_multitrigger_matchsecond():
 def test_continuation_multitrigger_matchnone():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = ["cardassia", "prime"]
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)
@@ -100,7 +102,7 @@ def test_continuation_multitrigger_matchnone():
 def test_continuation_empty_triggers():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["triggers"] = []
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)
@@ -110,7 +112,7 @@ def test_continuation_empty_triggers():
 def test_continuation_triggers_not_trigger():
     d = _plugins.load_plugin("detectors.continuation.Continuation")
 
-    a = garak.attempt.Attempt(prompt="test prompts")
+    a = garak.attempt.Attempt(prompt=garak.attempt.Message(text="test prompts"))
     a.notes["trigger"] = TEST_TRIGGER
     a.outputs = [TEST_TRIGGER]
     r = d.detect(a)