| diff --git a/src/nlp/punctuation/punctuation_rules.py b/src/nlp/punctuation/punctuation_rules.py |
| index 0d44de2..43b59d8 100644 |
| --- a/src/nlp/punctuation/punctuation_rules.py |
| +++ b/src/nlp/punctuation/punctuation_rules.py |
| @@ -60,7 +60,9 @@ def arabic_postprocessing(text: str) -> str: |
| # Only applies if a colon is actually present on the verb or the name |
| def _fix_misplaced(m): |
| verb, col1, name, col2 = m.groups() |
| - if col1 == ':' or col2 == ':': |
| + if col1 == ':': |
| + return f"{verb}: {name}" |
| + if col2 == ':': |
| return f"{verb} {name}:" |
| return m.group(0) |
| |
| @@ -87,9 +89,9 @@ def arabic_postprocessing(text: str) -> str: |
| return match.group(0) |
| |
| if prev_word.startswith(('ال', 'لل', 'بال', 'فال', 'وال', 'كال')): |
| - return context + " " |
| + return match.group(0) # Preserve the colon! Do not delete it. |
| |
| - return match.group(0) |
| + return context + ' ' |
| |
| text = re.sub(r'([^:]+)(:)', _colon_guard, text) |
| |
| @@ -132,16 +134,11 @@ _EXCL_CUES = {'يا', 'ما', 'كم', 'لا', 'هل', 'أين', 'متى', |
| def _normalize_for_comparison(text: str) -> str: |
| """ |
| Normalize Arabic for safe comparison. |
| - Prevents false rejection from hamza/alef/ya variants. |
| + Only removes diacritics to prevent punctuation model from stripping harakat. |
| + Does NOT fold hamza/ya/ta-marbuta to ensure we catch spelling regressions! |
| """ |
| # Remove diacritics |
| text = re.sub(r'[\u064B-\u0652]', '', text) |
| - # Fold hamza/alef variants: أ إ آ → ا |
| - text = re.sub(r'[أإآ]', 'ا', text) |
| - # Fold ya: ى → ي |
| - text = text.replace('ى', 'ي') |
| - # Fold ta marbuta: ة → ه (comparison only) |
| - text = text.replace('ة', 'ه') |
| return text |
| |
| |
|
|