tokenizer.py

import re

def custom_tokenizer(text):
    # Lowercase the text
    text = text.lower()

    # Remove punctuation
    text = re.sub(r'[^\w\s]', '', text)

    # Split the text into tokens
    tokens = text.split()

    # Add special tokens for the question
    tokens.insert(0, '<question>')
    tokens.append('<endquestion>')

    return tokens