remove kobert tokenizer

philipithomas · philipithomas · commit fac8fead93b7 · 2025-02-28T13:34:08.000-08:00
diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -58,6 +58,9 @@
 
     # TODO: remove when https://github.com/huggingface/transformers/issues/28096 is addressed
     'RajuKandasamy/tamillama_tiny_30m',
+
+    # TODO: remove when KoBertTokenizer is properly supported
+    'monologg/kobert',
 ]
 
 MAX_TESTS = {

Original file line number	Diff line number	Diff line change
`@@ -58,6 +58,9 @@`
`58`	`58`
`59`	`59`	`# TODO: remove when https://github.com/huggingface/transformers/issues/28096 is addressed`
`60`	`60`	`'RajuKandasamy/tamillama_tiny_30m',`
	`61`	`+`
	`62`	`+ # TODO: remove when KoBertTokenizer is properly supported`
	`63`	`+ 'monologg/kobert',`
`61`	`64`	`]`
`62`	`65`
`63`	`66`	`MAX_TESTS = {`