mbart chunking
Browse files- chunking.py +4 -0
chunking.py
CHANGED
|
@@ -26,6 +26,10 @@ def get_max_word_length(target_languages: list[str]) -> int:
|
|
| 26 |
"sq": 140,
|
| 27 |
"sl": 140,
|
| 28 |
"cnr": 60,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
}
|
| 30 |
|
| 31 |
max_word_length = 700 # Default Chunk Size
|
|
|
|
| 26 |
"sq": 140,
|
| 27 |
"sl": 140,
|
| 28 |
"cnr": 60,
|
| 29 |
+
"nl": 50,
|
| 30 |
+
"pt": 50,
|
| 31 |
+
"lv": 50,
|
| 32 |
+
"ro": 50,
|
| 33 |
}
|
| 34 |
|
| 35 |
max_word_length = 700 # Default Chunk Size
|