File size: 1,863 Bytes
fe1e225
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
diff --git a/src/nlp/punctuation/punctuation_rules.py b/src/nlp/punctuation/punctuation_rules.py
index 0d44de2..43b59d8 100644
--- a/src/nlp/punctuation/punctuation_rules.py
+++ b/src/nlp/punctuation/punctuation_rules.py
@@ -60,7 +60,9 @@ def arabic_postprocessing(text: str) -> str:
     # Only applies if a colon is actually present on the verb or the name
     def _fix_misplaced(m):
         verb, col1, name, col2 = m.groups()
-        if col1 == ':' or col2 == ':':
+        if col1 == ':':
+            return f"{verb}: {name}"
+        if col2 == ':':
             return f"{verb} {name}:"
         return m.group(0)
         
@@ -87,9 +89,9 @@ def arabic_postprocessing(text: str) -> str:
             return match.group(0)
             
         if prev_word.startswith(('ال', 'لل', 'بال', 'فال', 'وال', 'كال')):
-            return context + " " 
+            return match.group(0)  # Preserve the colon! Do not delete it.
             
-        return match.group(0)
+        return context + ' '
         
     text = re.sub(r'([^:]+)(:)', _colon_guard, text)
     
@@ -132,16 +134,11 @@ _EXCL_CUES = {'يا', 'ما', 'كم', 'لا', 'هل', 'أين', 'متى',
 def _normalize_for_comparison(text: str) -> str:
     """
     Normalize Arabic for safe comparison.
-    Prevents false rejection from hamza/alef/ya variants.
+    Only removes diacritics to prevent punctuation model from stripping harakat.
+    Does NOT fold hamza/ya/ta-marbuta to ensure we catch spelling regressions!
     """
     # Remove diacritics
     text = re.sub(r'[\u064B-\u0652]', '', text)
-    # Fold hamza/alef variants: أ إ آ → ا
-    text = re.sub(r'[أإآ]', 'ا', text)
-    # Fold ya: ى → ي
-    text = text.replace('ى', 'ي')
-    # Fold ta marbuta: ة → ه (comparison only)
-    text = text.replace('ة', 'ه')
     return text