From eb48d3dd39f5cdd23882870e24cb6346c45700e4 Mon Sep 17 00:00:00 2001
From: rightyonghu <rightyonghu@tencent.com>
Date: Thu, 1 Sep 2022 00:17:47 +0800
Subject: [PATCH] remove Tokenizer in ernie

---
 src/transformers/__init__.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index c42265698c347..5ac0edebc0055 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -145,10 +145,7 @@
     ],
     "models.ernie": [
         "ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP",
-        "BasicTokenizer",
         "ErnieConfig",
-       
-        "WordpieceTokenizer",
     ],
     "models.bert_generation": ["BertGenerationConfig"],
     "models.bert_japanese": ["BertJapaneseTokenizer", "CharacterTokenizer", "MecabTokenizer"],
@@ -3014,10 +3011,7 @@
     )
     from .models.ernie import (
         ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP,
-        BasicTokenizer,
         ErnieConfig,
-       
-        WordpieceTokenizer,
     )
     from .models.bert_generation import BertGenerationConfig
     from .models.bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer