diff --git a/src/nlp/grammar/grammar_rules.py b/src/nlp/grammar/grammar_rules.py index 141fb50..1f80154 100644 --- a/src/nlp/grammar/grammar_rules.py +++ b/src/nlp/grammar/grammar_rules.py @@ -77,31 +77,8 @@ class ArabicGrammarGuard: return " ".join(corrected_tokens) def smart_asmaa_khamsa_fix(self, text): - tokens = simple_word_tokenize(text) - disambig_tokens = self.mle.disambiguate(tokens) - corrected_tokens = [] - verb_seen = False - - for i, token_info in enumerate(disambig_tokens): - word = tokens[i] - - pos_tag = token_info.analyses[0].analysis.get('pos', 'unknown') if token_info.analyses else 'unknown' - - if pos_tag == 'verb': - verb_seen = True - corrected_tokens.append(word) - continue - - is_asmaa = any(word.startswith(root) or word.startswith('أ' + root[1:]) for root in self.asmaa_khamsa_roots if len(root)>1) - - if is_asmaa and len(word) >= 3: - if verb_seen: - word = word.replace('ا', 'و').replace('ي', 'و') - verb_seen = False - - corrected_tokens.append(word) - - return " ".join(corrected_tokens) + # Bug 1.1: Disabled blind replacement of ا and ي with و as it corrupts object position + return text def _apply_jazm_to_verb(self, word, token_info): # 1. Handle Af'al Khamsa using camel_tools analysis @@ -182,7 +159,7 @@ class ArabicGrammarGuard: disambig_tokens = self.mle.disambiguate(tokens) nasb_particles = ['أن', 'ان', 'لن', 'كي', 'لكي', 'حتى', 'حتي', 'إذن', 'اذا'] - jazm_particles = ['لم', 'لما', 'لا'] + jazm_particles = ['لم', 'لما'] corrected_tokens = [] @@ -196,12 +173,16 @@ class ArabicGrammarGuard: if i > 0: prev_word = tokens[i-1] - if prev_word in nasb_particles or word.startswith('ل'): + if prev_word in nasb_particles: is_nasb_context = True - if prev_word in jazm_particles or word.startswith('ل') or word.startswith('ول'): + if prev_word in jazm_particles: is_jazm_context = True + # Lam Al-Ta'leel / Lam Al-Amr logic was flawed (triggering for ANY word starting with ل) + # Ensure it only applies to actual verbs starting with 'لي' or 'لت' etc. + if (word.startswith('ل') or word.startswith('ول') or word.startswith('فل')) and len(word) >= 4 and word[1] in ['ي', 'ت', 'ن', 'أ']: + is_nasb_context = True - is_present_tense = word.startswith('ي') or word.startswith('ت') or word.startswith('ن') or word.startswith('أ') + is_present_tense = word.startswith('ي') or word.startswith('ت') or word.startswith('ن') or word.startswith('أ') or word.startswith('لي') or word.startswith('لت') or word.startswith('ولي') or word.startswith('فلي') if (pos_tag == 'verb' or is_present_tense) and (is_nasb_context or is_jazm_context): if is_jazm_context: word = self._apply_jazm_to_verb(word, token_info) @@ -223,9 +204,14 @@ class ArabicGrammarGuard: # When a feminine noun is followed by a masculine adjective, add ة # e.g. السيارة جميل → السيارة جميلة words = text.split() + KNOWN_MASC_TA_MARBUTA = {'خليفة', 'أسامة', 'حمزة', 'طلحة', 'معاوية', 'عبيدة', 'قضاة', 'دعاة', 'رماة', 'حماة'} for i in range(len(words) - 1): noun = words[i] adj = words[i + 1] + # Strip prefixes like 'ال' to check root + clean_noun = noun[2:] if noun.startswith('ال') else noun + if clean_noun in KNOWN_MASC_TA_MARBUTA: + continue is_fem_noun = (noun in KNOWN_FEMININE_NOUNS or (noun.endswith('ة') and len(noun) >= 3) or (noun.startswith('ال') and noun.endswith('ة'))) @@ -250,6 +236,9 @@ class ArabicGrammarGuard: 'النيران', 'نيران', 'الألوان', 'ألوان', 'البلدان', 'بلدان', 'الأوطان', 'أوطان', 'الأبدان', 'أبدان', 'الأركان', 'أركان', 'الفرسان', 'فرسان', 'الغزلان', 'غزلان', 'القضبان', 'قضبان', + 'فرعون', 'قانون', 'القانون', 'كانون', 'قارون', 'طاعون', + 'عربون', 'هارون', 'زيدون', 'معجون', 'مجنون', 'زيتون', 'صابون', + 'الفرعون', 'قوانين' } def fix_prepositions_advanced(self, text): @@ -261,31 +250,36 @@ class ArabicGrammarGuard: stem = m.group(2) suffix = m.group(3) full_word = stem + suffix - # Skip words in blocklist (root nouns, not duals) - if full_word in self._PREP_BLOCKLIST: - return m.group(0) # return unchanged - # Skip ال-prefixed words ending in ان — almost always root nouns - if stem.startswith('ال') and suffix == 'ان': - return m.group(0) # return unchanged - return f'{prep} {stem}ين' - - text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{4,})(ون|ان)\b', _prep_replace, text) + + # Use camel-tools disambiguation to determine if it's really a dual/plural + tokens = simple_word_tokenize(full_word) + disambig_tokens = self.mle.disambiguate(tokens) + if disambig_tokens and disambig_tokens[0].analyses: + num = disambig_tokens[0].analyses[0].analysis.get('num', 's') + # Only apply ين suffix if the word is actually Dual or Plural + if num in ['d', 'p']: + return f'{prep} {stem}ين' + return m.group(0) + + text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{3,})(ون|ان)\b', _prep_replace, text) # (وبالمبرمجون) -> (وبالمبرمجين) # FIX-33b: Same blocklist protection as first regex def _attached_prep_replace(m): - prefix = m.group(1) # وب، ب، فب، ول، ل، etc. + prefix = m.group(1) stem = m.group(2) suffix = m.group(3) - full_word = 'ال' + stem + suffix # reconstruct with ال for blocklist check - if full_word in self._PREP_BLOCKLIST: - return m.group(0) - # Words ending in ان with 4+ char stems are almost always root nouns - if suffix == 'ان': - return m.group(0) - return f'{prefix}ال{stem}ين' + full_word = 'ال' + stem + suffix + + tokens = simple_word_tokenize(full_word) + disambig_tokens = self.mle.disambiguate(tokens) + if disambig_tokens and disambig_tokens[0].analyses: + num = disambig_tokens[0].analyses[0].analysis.get('num', 's') + if num in ['d', 'p']: + return f'{prefix}ال{stem}ين' + return m.group(0) - text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{4,})(ون|ان)\b', _attached_prep_replace, text) + text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{3,})(ون|ان)\b', _attached_prep_replace, text) # (ولمهندسون) -> (ولمهندسين) # FIX-33b: Same protection — reconstruct full word for blocklist @@ -321,8 +315,8 @@ class ArabicGrammarGuard: if word == 'ذو': return 'ذا' if word == 'فو': return 'فا' if word == 'حمو': return 'حما' - if word.endswith('ون'): return word[:-2] + 'ين' - if word.endswith('ان'): return word[:-2] + 'ين' + if word.endswith('ون') and num == 'p' and word not in ('قانون', 'فرعون', 'كانون', 'معجون', 'طاعون', 'مجنون'): return word[:-2] + 'ين' + if word.endswith('ان') and num == 'd' and word not in ('امتحان', 'إنسان', 'ميدان', 'سلطان', 'شيطان'): return word[:-2] + 'ين' elif target_case == 'n': # Marfoo' if word in ('أبا', 'أبي'): return 'أبو' if word in ('أخا', 'أخي'): return 'أخو' @@ -481,30 +475,18 @@ class ArabicGrammarGuard: # FIX-08: Expanded feminine plurals 'المهندسات', 'مهندسات', 'الطبيبات', 'طبيبات', 'اللاعبات', 'لاعبات', 'الممثلات', 'ممثلات', - 'الشركات', 'شركات', 'الجامعات', 'جامعات', - 'المدارس', 'مدارس', 'المستشفيات', 'مستشفيات', - 'الحكومات', 'حكومات', 'المنظمات', 'منظمات', - 'الطائرات', 'طائرات', 'السيارات', 'سيارات', } - if noun_word in KNOWN_PLURALS_MASC: + if noun_num == 'd': + pass # Dual noun, skip plural logic to prevent corruption + elif noun_word in KNOWN_PLURALS_MASC: is_plural_masc = True elif noun_word in KNOWN_PLURALS_FEM: is_plural_fem = True - elif noun_word.endswith('ون') or noun_word.endswith('ين'): - # Sound masculine plural — but only if 4+ chars (avoid short words) - if len(noun_word) >= 5: - is_plural_masc = True - elif noun_word.endswith('ات') and len(noun_word) >= 5: - is_plural_fem = True - # FIX-08: Broken plural heuristic — common patterns - elif noun_num == 'p': - # Trust POS tagger when it says plural AND word is long enough - if len(noun_word) >= 4: - if noun_gen == 'f': - is_plural_fem = True - else: - is_plural_masc = True + elif (noun_word.endswith('ون') or (noun_word.endswith('ين') and noun_num == 'p')) and len(noun_word) >= 5: + # Bug 1.14: Check noun_num == 'p' to prevent duals from triggering plural logic + is_plural_masc = True + # Bug 1.2: Disabled POS tagger heuristic for broken plurals and generic 'ات' ending because it forces 'ن' on non-human plurals is_singular_fem = False if not is_plural_masc and not is_plural_fem: @@ -574,7 +556,8 @@ class ArabicGrammarGuard: return word # إن وأخواتها - text = re.sub(r'\b(إن|أن|كأن|لكن|لعل|ليت|ان|كان)\s+(أبوك|ابوك|أخوك|اخوك|ذو|فوك)\b', + # Bug 1.13: removed كان from this regex because it caused كان أخوك -> كان أخاك + text = re.sub(r'\b(إن|أن|كأن|لكن|لعل|ليت|ان)\s+(أبوك|ابوك|أخوك|اخوك|ذو|فوك)\b', lambda m: f"{m.group(1)} {_add_hamza(m.group(2)).replace('و', 'ا')}", text) # الأفعال المتعدية (Object position) @@ -596,19 +579,19 @@ class ArabicGrammarGuard: # FIX-PC010: Add targeted safe regex for Nasb/Jazm particles + verb # Only match clear present tense verbs starting with ي/ت/ن/أ and ending in ون - text = re.sub(r'\b(أن|ان|لن|كي|حتى|لم|لما)\s+([يتا][\u0600-\u06FF]{2,})ون\b', + text = re.sub(r'\b(أن|ان|لن|كي|حتى|لم|لما)\s+([يتاأن][\u0600-\u06FF]{2,})ون\b', r'\1 \2وا', text) return text def fix_conditional_sentences(self, text): - conditional_particles = {'إن', 'ان', 'من', 'ما', 'متى', 'متي', 'مهما', 'أينما', 'حيثما', 'أيان', 'ايان', 'كيفما', 'أنى', 'اني'} + conditional_particles = {'إن', 'ان', 'متى', 'متي', 'مهما', 'أينما', 'حيثما', 'أيان', 'ايان', 'كيفما', 'أنى', 'اني'} tokens = simple_word_tokenize(text) disambig_tokens = self.mle.disambiguate(tokens) corrected_tokens = list(tokens) # Lookahead for 2nd person context - has_2nd_person_context = any(t.endswith('كم') or t.endswith('كمو') or t.startswith('ت') for t in tokens) + has_2nd_person_context = False in_cond = False verbs_jazmed = 0 @@ -634,10 +617,6 @@ class ArabicGrammarGuard: # Apply jazm using the comprehensive camel_tools helper word = self._apply_jazm_to_verb(word, token_info) - # Fix pronoun mismatch if 2nd person context exists - if has_2nd_person_context and word.startswith('ي') and (word.endswith('وا') or word.endswith('ا') or word.endswith('ي')): - word = 'ت' + word[1:] - corrected_tokens[i] = word # Increment jazmed verbs counter (handles both فعل الشرط and جواب الشرط) verbs_jazmed += 1 @@ -714,8 +693,8 @@ class ArabicGrammarGuard: else: corrected_tokens[i+1] = base_adj + ('ان' if is_nom else 'ين') - # Plural Human Adjective Agreement - elif w1_num == 'p' and w1_pos in ['noun', 'unknown']: + # Plural Human Adjective Agreement (Sound Plurals Only) + elif w1_num == 'p' and w1_pos in ['noun', 'unknown'] and (w1.endswith('ون') or w1.endswith('ين') or w1.endswith('ات')): base_adj = None for suffix in ['ان', 'ين', 'تان', 'تين', 'ة', 'ون', 'ات', 'ين', '']: stem = w2[:-len(suffix)] if suffix else w2 @@ -726,9 +705,11 @@ class ArabicGrammarGuard: if base_adj: if w1.endswith('ون') or w1.endswith('ين') or w1_gen == 'm': is_nom = w1.endswith('ون') - corrected_tokens[i+1] = base_adj + ('ون' if is_nom else 'ين') + if not w2.endswith('ان') and not w2.endswith('ين') and not w2.endswith('ات'): + corrected_tokens[i+1] = base_adj + ('ون' if is_nom else 'ين') elif w1.endswith('ات') or w1_gen == 'f': - corrected_tokens[i+1] = base_adj + 'ات' + if not w2.endswith('ان') and not w2.endswith('ين') and not w2.endswith('ات'): + corrected_tokens[i+1] = base_adj + 'ات' return " ".join(corrected_tokens) @@ -866,8 +847,9 @@ class ArabicGrammarGuard: } words = text.split() for i, w in enumerate(words): - if w in _ALWAYS_TANWEEN: - words[i] = _ALWAYS_TANWEEN[w] + clean_w = w.rstrip('.,،؛;:!؟?()[]{}«»"\'…') + if clean_w in _ALWAYS_TANWEEN: + words[i] = _ALWAYS_TANWEEN[clean_w] + w[len(clean_w):] return ' '.join(words) def fix_initial_hamza(self, text): @@ -920,16 +902,27 @@ class ArabicGrammarGuard: _PRONOUN_SUFFIXES = {'ه', 'ها', 'ك', 'كم', 'كن', 'هم', 'هن', 'ني', 'نا'} words = text.split() for i, w in enumerate(words): - if w in _HAMZA_FIXES: - words[i] = _HAMZA_FIXES[w] + clean_w = w.rstrip('.,،؛;:!؟?()[]{}«»"\'…') + if clean_w in _HAMZA_FIXES: + words[i] = _HAMZA_FIXES[clean_w] + w[len(clean_w):] continue # إنّ/أنّ: kasra at sentence start, fathah mid-sentence _is_sent_start = (i == 0) or (words[i-1][-1] in '.؟!؛' if words[i-1] else False) - if _is_sent_start and w in _INNA_SENTENCE_INITIAL: - words[i] = _INNA_SENTENCE_INITIAL[w] + if _is_sent_start and clean_w in _INNA_SENTENCE_INITIAL: + words[i] = _INNA_SENTENCE_INITIAL[clean_w] + w[len(clean_w):] continue - if not _is_sent_start and w in _ANNA_MID_SENTENCE: - words[i] = _ANNA_MID_SENTENCE[w] + if not _is_sent_start and clean_w in _ANNA_MID_SENTENCE: + if clean_w == 'ان': + # Bug 1.5: Added تقول and all variants to Qawl check + if i > 0 and words[i-1].rstrip('.,،؛;:!؟?()[]{}«»"\'…') in ('قال', 'قالت', 'يقول', 'يقولون', 'قلت', 'قلنا', 'تقول', 'يقولوا'): + words[i] = 'إن' + w[len(clean_w):] + else: + pass # leave it alone to avoid breaking conditional + else: + if i > 0 and words[i-1].rstrip('.,،؛;:!؟?()[]{}«»"\'…') in ('قال', 'قالت', 'يقول', 'يقولون', 'قلت', 'قلنا', 'تقول', 'يقولوا'): + words[i] = _INNA_SENTENCE_INITIAL[clean_w] + w[len(clean_w):] + else: + words[i] = _ANNA_MID_SENTENCE[clean_w] + w[len(clean_w):] continue for stem, fixed in _HAMZA_STEMS.items(): if w.startswith(stem) and len(w) > len(stem):