---
license: apache-2.0
library_name: tokenizers
tags:
  - tokenizer
language:
  - hi
  - as
  - mr
  - gu
  - pa
  - en
  - or
  - te
  - ta
  - ml
  - kn
  - bn
  - sd
  - ur
  - ne
  - ks
  - sa
  - gom
  - mai
  - mni
  - brx
  - doi
  - sat
Vocab_size: 
2,56,000
---