File size: 1,863 Bytes
fe1e225 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | diff --git a/src/nlp/punctuation/punctuation_rules.py b/src/nlp/punctuation/punctuation_rules.py
index 0d44de2..43b59d8 100644
--- a/src/nlp/punctuation/punctuation_rules.py
+++ b/src/nlp/punctuation/punctuation_rules.py
@@ -60,7 +60,9 @@ def arabic_postprocessing(text: str) -> str:
# Only applies if a colon is actually present on the verb or the name
def _fix_misplaced(m):
verb, col1, name, col2 = m.groups()
- if col1 == ':' or col2 == ':':
+ if col1 == ':':
+ return f"{verb}: {name}"
+ if col2 == ':':
return f"{verb} {name}:"
return m.group(0)
@@ -87,9 +89,9 @@ def arabic_postprocessing(text: str) -> str:
return match.group(0)
if prev_word.startswith(('ال', 'لل', 'بال', 'فال', 'وال', 'كال')):
- return context + " "
+ return match.group(0) # Preserve the colon! Do not delete it.
- return match.group(0)
+ return context + ' '
text = re.sub(r'([^:]+)(:)', _colon_guard, text)
@@ -132,16 +134,11 @@ _EXCL_CUES = {'يا', 'ما', 'كم', 'لا', 'هل', 'أين', 'متى',
def _normalize_for_comparison(text: str) -> str:
"""
Normalize Arabic for safe comparison.
- Prevents false rejection from hamza/alef/ya variants.
+ Only removes diacritics to prevent punctuation model from stripping harakat.
+ Does NOT fold hamza/ya/ta-marbuta to ensure we catch spelling regressions!
"""
# Remove diacritics
text = re.sub(r'[\u064B-\u0652]', '', text)
- # Fold hamza/alef variants: أ إ آ → ا
- text = re.sub(r'[أإآ]', 'ا', text)
- # Fold ya: ى → ي
- text = text.replace('ى', 'ي')
- # Fold ta marbuta: ة → ه (comparison only)
- text = text.replace('ة', 'ه')
return text
|