remove Tokenizer in ernie

huggingface · Sep 5, 2022 · eb48d3d · eb48d3d
1 parent 9e93500
commit eb48d3d
Showing 1 changed file with 0 additions and 6 deletions.
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
@@ -145,10 +145,7 @@
     ],
     "models.ernie": [
         "ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP",
-        "BasicTokenizer",
         "ErnieConfig",
-
-        "WordpieceTokenizer",
     ],
     "models.bert_generation": ["BertGenerationConfig"],
     "models.bert_japanese": ["BertJapaneseTokenizer", "CharacterTokenizer", "MecabTokenizer"],
@@ -3014,10 +3011,7 @@
     )
     from .models.ernie import (
         ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP,
-        BasicTokenizer,
         ErnieConfig,
-
-        WordpieceTokenizer,
     )
     from .models.bert_generation import BertGenerationConfig
     from .models.bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer