Truong / MMAudio-main /transformers /utils /test_module /custom_tokenization_fast.py
Doaneer's picture
Upload 5065 files
e0be88b verified
raw
history blame contribute delete
193 Bytes
from transformers import BertTokenizerFast
from .custom_tokenization import CustomTokenizer
class CustomTokenizerFast(BertTokenizerFast):
slow_tokenizer_class = CustomTokenizer
pass