Merge pull request #56 from m-misiura/add_slow_tokenizer

Update transformers to transformers==4.57.1; add slow tokenizers

Author: m-misiura

detectors/huggingface/detector.py

@@ -72,7 +72,11 @@ def initialize_model(self, model_files_path):
         """
         Load and configure the model and tokenizer.
         """
-        self.tokenizer = AutoTokenizer.from_pretrained(model_files_path, use_fast=True)
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(model_files_path, use_fast=True)
+        except (ValueError, OSError, ImportError) as e:
+            logger.warning(f"Failed to load fast tokenizer: {e}. Falling back to slow tokenizer.")
+            self.tokenizer = AutoTokenizer.from_pretrained(model_files_path, use_fast=False)
         config = AutoConfig.from_pretrained(model_files_path)
         logger.info(f"Model Config: {config}")
 

detectors/huggingface/requirements.txt

@@ -1 +1,3 @@
-transformers==4.50.0
+transformers==4.57.1
+sentencepiece==0.2.1
+tiktoken==0.12.0