File size: 12,880 Bytes
81a8221 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 |
# -*- encoding: utf-8 -*-
"""
@Auth: xuelyuxin.xlx
@Time: 2023/05/12 14:55:44
@Desc: 正则模块
`Normalizer.normalize_regular`输出内容只包含简体中文、英文、tts相关标点的文本
@Usge: null
"""
import re
from typing import List, Dict
from .utils.chronology import RE_DATE
from .utils.chronology import RE_DATE2
from .utils.chronology import RE_TIME, RE_TIME_2, RE_TIME_3
from .utils.chronology import RE_TIME_RANGE
from .utils.chronology import replace_date
from .utils.chronology import replace_date2
from .utils.chronology import replace_time, replace_time_nohour
from .utils.num import RE_DECIMAL_NUM
from .utils.num import RE_DEFAULT_NUM
from .utils.num import RE_FRAC
from .utils.num import RE_INTEGER
from .utils.num import RE_NUMBER
from .utils.num import RE_PERCENTAGE
from .utils.num import RE_POSITIVE_QUANTIFIERS
from .utils.num import RE_POSITIVE_QUANTIFIERS_2
from .utils.num import RE_RANGE
from .utils.num import RE_DIGITS
from .utils.num import RE_LICENSE_PLATE
from .utils.num import replace_default_num_with_altone, replace_default_num_without_altone
from .utils.num import replace_frac
from .utils.num import replace_negative_num
from .utils.num import replace_number
from .utils.num import replace_percentage
from .utils.num import replace_positive_quantifier
from .utils.num import replace_positive_quantifier_2
from .utils.num import replace_range
from .utils.num import replace_license_plate
from .utils.phonecode import RE_MOBILE_PHONE
from .utils.phonecode import RE_NATIONAL_UNIFORM_NUMBER
from .utils.phonecode import RE_TELEPHONE
from .utils.phonecode import replace_mobile
from .utils.phonecode import replace_phone
from .utils.quantifier import RE_TEMPERATURE
from .utils.quantifier import replace_temperature
from .utils.address import RE_ADDRESS_room, RE_ADDRESS
from .utils.address import replace_address_room, replace_address
from .utils.currency import RE_CURRENCY, RE_CURRENCY_2
from .utils.currency import replace_currency, replace_currency_2
from .utils.en_num import normalize_numbers as en_normalize_numbers
from .utils.string_operator import BLANK_CHAR, PUNC_MAP_OTHER2CN, PUNC_MAP_STANDARD, PUNC_STANDARD, REGEX_CN # noqa
from .utils.string_operator import StringOperator as stringop
def add_blank(match_obj):
return " ".join(list(match_obj.group(0)))
def convert_date(string):
nums = re.findall("[\d]+", string)
if len(nums) == 3:
year, month, day = [RE_NUMBER.sub(replace_number, w) for w in nums]
return f"{year}年{month}月{day}日"
elif len(nums) == 2:
month, day = [RE_NUMBER.sub(replace_number, w) for w in nums]
return f"{month}月{day}日"
else:
return string
class Normalizer:
"""文本正则
"""
@classmethod
def substitute(cls, pattern: re.Pattern, replace_func, text: str, trace: list):
for matchobj in pattern.finditer(text):
origin_word = matchobj.group(0)
new_word = replace_func(matchobj)
trace.append({"origin_word": origin_word, "new_word": new_word})
text = text.replace(origin_word, new_word)
return text
@classmethod
def preprocess(cls, text: str) -> str:
"""正则前的预处理
1. 繁体转简体
2. 过滤不影响正则的标点符号等，包括
a. 数字间的逗号
b. 空格（除英文之间外）
c. 空白字符\t\n\r\f
3. 英文转小写
"""
text = stringop.replace_F2H(text) # 统一转半角
text = stringop.delete_comma_in_number(text) # 去掉数字之间的逗号
# text = stringop.delete_space(text)
text = re.sub(rf"[{BLANK_CHAR}]", "，", text) # 去掉不影响正则的字符
# text = text.lower()
# 处理特殊符号
text = re.sub(r"㎡", "m²", text)
text = text.replace("㎡", "m²")
text = text.replace("cm²", "平方厘米")
text = text.replace("m²", "平方米")
# text = text.replace("<", "<")
# text = text.replace(">", ">")
# text = text.replace("&", "&")
# text = text.replace(""", "\"")
# text = text.replace("'", "'")
text = re.sub(r">(\d)", r"大于\1", text)
text = re.sub(r"<(\d)", r"小于\1", text)
text = re.sub(r"=", "等于", text)
text = re.sub(r"(?<=\d)ml(?![a-zA-Z])", "毫升", text)
text = re.sub(r"(?<=\d)mmHg(?![a-zA-Z])", "毫米汞柱", text)
text = re.sub(r"([0-9.]+元)(-)([0-9.]+元)", "\\1至\\3", text)
return text
@classmethod
def postprocess(cls, text: str, custom: List[Dict] = None) -> str:
"""正则后处理，只包括删除和替换操作
正则后的文本中的字符类型只包括：
1. 中文 2. 英文 3. 标点`，。！？`
"""
if custom is not None:
for map_dict in custom:
text = stringop.replace(text, map_dict)
return text
# 标点符号统一为中文标点`。！？，`
text = stringop.replace_punc_en2cn(text)
text = stringop.replace(text, PUNC_MAP_OTHER2CN)
# 进一步把中文标点统一为标准中文标点`。！？，`
text = stringop.replace(text, PUNC_MAP_STANDARD)
# 处理连续句号"。"
text = re.sub(r"。+", "。", text)
# 处理正则后的 "/"
text = re.sub("/", "每", text)
# 处理正则后的[~～]+
text = re.sub(r"～+", "～", text)
text = re.sub(r"~+", "~", text)
# text = re.sub(r"[~～](?=[0-9])", "至", text)
text = re.sub(r"[~～]", "。", text)
# 删除除了中文、英文、数字、标准中文标点、@break外的其他符号
text = stringop.delete(text, f"[^{PUNC_STANDARD}{REGEX_CN}A-Za-z0-9@]")
text = stringop.delete(text, "@(?!break)") # 删除@符号，`@break`除外
return text
@classmethod
def custom(cls, text: str, *, interpret_as: str) -> str:
text = cls.preprocess(text)
if not text:
return "" # 对于预处理后为空的字符串直接返回
text = cls.normalize_custom(text, interpret_as=interpret_as)
text = cls.postprocess(text)
return text
@classmethod
def regular(cls, text: str) -> str:
text = cls.preprocess(text)
if not text:
return "" # 对于预处理后为空的字符串直接返回
text = cls.normalize_regular(text)
text = cls.postprocess(text)
return text
@classmethod
def normalize_custom(cls, text: str, *, interpret_as: str) -> str:
"""指定正则
仅针对`interpret_as`进行正则匹配
Args:
interpret_as: string in ['cardinal', 'currency', 'digits', 'telephone', 'address', 'date', 'time', 'id']
"""
assert interpret_as in [
"cardinal",
"currency",
"digits",
"telephone",
"address",
"date",
"time",
"id",
"measure",
"punctuation"
], f"""
interpret_as:{interpret_as} not supported
""".strip()
if interpret_as == "cardinal":
text = text.replace(",", "")
text = RE_NUMBER.sub(replace_number, text) # 正负整数小数
text = RE_FRAC.sub(replace_frac, text)
text = RE_PERCENTAGE.sub(replace_percentage, text)
elif interpret_as == "currency":
text = RE_CURRENCY.sub(replace_currency, text)
text = RE_CURRENCY_2.sub(replace_currency_2, text)
text = text.replace(",", "")
text = RE_NUMBER.sub(replace_number, text) # 正负整数小数
text = RE_FRAC.sub(replace_frac, text)
elif interpret_as == "digits":
text = RE_DIGITS.sub(replace_default_num_without_altone, text)
elif interpret_as == "telephone":
text = RE_MOBILE_PHONE.sub(replace_mobile, text)
text = RE_TELEPHONE.sub(replace_phone, text)
text = RE_NATIONAL_UNIFORM_NUMBER.sub(replace_phone, text)
text = RE_DIGITS.sub(replace_default_num_with_altone, text)
elif interpret_as == "address":
text = text.replace("-", "杠")
text = RE_ADDRESS_room.sub(replace_address_room, text)
text = RE_ADDRESS.sub(replace_address, text)
elif interpret_as == "date":
text = RE_DATE.sub(replace_date, text) # 年月日
text = RE_DATE2.sub(replace_date2, text) # YY/MM/DD 或者 YY-MM-DD
text = convert_date(text)
text = text.replace("-", "至")
elif interpret_as == "time":
text = RE_TIME_RANGE.sub(replace_time, text) # 8:30-12:30
text = RE_TIME.sub(replace_time, text) # 12:30:58
elif interpret_as == "id":
text = RE_DIGITS.sub(replace_default_num_with_altone, text)
text = text.replace("_", "下划线").replace("-", "杠").upper()
text = re.sub("[a-zA-Z]+", add_blank, text)
elif interpret_as == "measure":
text = text.replace("㎡", "m²")
text = text.replace("cm²", "平方厘米")
text = text.replace("m²", "平方米")
text = text.replace("cm", "厘米")
text = text.replace("mm", "毫米")
text = text.replace("m", "米")
text = text.replace("kg", "千克")
text = text.replace("g", "克")
elif interpret_as == "punctuation":
text = re.sub("…+", "省略号", text)
text = re.sub("\"|“|”", "双引号", text)
text = re.sub("'|‘|’", "单引号", text)
text = re.sub("（|\\(", "左括号", text)
text = re.sub("）|\\)", "右括号", text)
text = re.sub("!|！", "叹号", text)
for sym, txt in zip(
['……', '…', '!', '"', '#', '$', '%', '&', '‘', '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_'], # noqa
['省略号', '省略号', '叹号', '双引号', '井号', 'dollar', '百分号', 'and', '单引号', '左括号', '右括号', '星号', '加号', '逗号', '杠', '点', '斜杠', '冒号', '分号', '小于', '等号', '大于', '问号', 'at', '左方括号', '反斜线', '右方括号', '脱字符', '下划线'] # noqa
):
text = text.replace(sym, txt)
return text
@classmethod
def normalize_regular(cls, text: str, is_en: bool = False, return_details: bool = False) -> str:
"""通用正则
包含了所有可能情况的正则匹配
输出的文本中只包含中文、英文、tts所需的标点("，。！？")
"""
trace = []
if is_en is True:
text = en_normalize_numbers(text)
text = text.replace(".", "。")
text = text.replace(",", "，")
else:
text = cls.substitute(RE_DATE, replace_date, text, trace)
text = cls.substitute(RE_DATE2, replace_date2, text, trace)
text = re.sub(r"(?<=[\d%])[-~](?=\d)", "至", text) # 先判断日期（2023-01-02），并跟减号负号区分
text = cls.substitute(RE_TIME_RANGE, replace_time, text, trace)
text = cls.substitute(RE_TIME, replace_time, text, trace)
text = cls.substitute(RE_CURRENCY, replace_currency, text, trace)
text = cls.substitute(RE_CURRENCY_2, replace_currency_2, text, trace)
text = cls.substitute(RE_TEMPERATURE, replace_temperature, text, trace)
text = cls.substitute(RE_LICENSE_PLATE, replace_license_plate, text, trace)
text = cls.substitute(RE_FRAC, replace_frac, text, trace)
text = cls.substitute(RE_PERCENTAGE, replace_percentage, text, trace)
text = cls.substitute(RE_MOBILE_PHONE, replace_mobile, text, trace)
text = cls.substitute(RE_TELEPHONE, replace_phone, text, trace)
text = cls.substitute(RE_NATIONAL_UNIFORM_NUMBER, replace_phone, text, trace)
text = cls.substitute(RE_RANGE, replace_range, text, trace)
text = cls.substitute(RE_INTEGER, replace_negative_num, text, trace)
text = cls.substitute(RE_DECIMAL_NUM, replace_number, text, trace)
text = cls.substitute(RE_POSITIVE_QUANTIFIERS_2, replace_positive_quantifier_2, text, trace)
text = cls.substitute(RE_POSITIVE_QUANTIFIERS, replace_positive_quantifier, text, trace)
text = cls.substitute(RE_DEFAULT_NUM, replace_default_num_with_altone, text, trace)
text = cls.substitute(RE_NUMBER, replace_number, text, trace)
return text, trace
|