File size: 1,270 Bytes
4ed02d8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
import re
import unicodedata
from transformers import LlamaTokenizerFast
tokenizer = LlamaTokenizerFast.from_pretrained("local_tokenizer")
def fasttext_preprocess_func(content: str) -> str:
"""Fasttext preprocess function.
Args:
content (str): Content to process.
Returns:
str: Processed normalized content.
"""
# 1. remove multiple newlines
content = re.sub(r'\n{3,}', '\n\n', content)
# 2. lower the content
content = content.lower()
# 3. remove diacritics
content = ''.join(
c for c in unicodedata.normalize('NFKD', content)
if unicodedata.category(c) != 'Mn')
# 4. word segmentation
token_ids = tokenizer.encode(content, add_special_tokens=False)
single_text_list = []
for token_id in token_ids:
curr_text = tokenizer.decode([token_id])
single_text_list.append(curr_text)
content = ' '.join(single_text_list)
# 5. keep escape chars, \n, \t, \r -> \\n, \\t, \\r,
# which will saved as \n, \t, \r in txt file.
content = re.sub(r'\n', '\\\\n', content)
content = re.sub(r'\r', '\\\\r', content)
content = re.sub(r'\t', '\\\\t', content)
content = re.sub(r' +', ' ', content)
content = content.strip()
return content
|