preprocessor.py

#%%
"""
Generated files/folders (depending on the flags set on __main__) summary:
    `Folders`
      - `tmp/Initial`       > Contains files generated by initial hardline filtering
      - `tmp/pattern`       > Contains files generated by character map replacements
      - `tmp/patch`         > Contains files generated by replacing patches of identical non-bangla texts on both sides
      - `tmp/transilaterate`> Contains files generated by transliterating dangling characters on bangla side
      - `Final`             > Contains files generated by applying previous transformations and some final postprocessing
    `Files(prefixes)`
      - `saved`             > Transformed Lines that pass hardline filtering after applying transformation 
      - `savedOriginal`     > Original Lines that pass hardline filtering after applying transformation
      - `filtered`          > Lines that don't pass hardline filtering at a stage
      - `cleaned`           > Lines that pass hardline filtering at a stage (Initial/Previous passing Lines + saved)
"""

import re
import os
import difflib
import time
from subprocess import check_output
from aksharamukha import transliterate
from itertools import chain
import shutil
import sys
import uuid
import multiprocessing
from multiprocessing import Pool
import argparse
import glob
from tqdm import tqdm

def globalize(func):
    def result(*args, **kwargs):
        return func(*args, **kwargs)
    result.__name__ = result.__qualname__ = uuid.uuid4().hex
    setattr(sys.modules[result.__module__], result.__name__, result)
    return result

REPLACE_UNICODE_PUNCTUATION = [
    (u"\u09F7", u"\u0964"),
    (u"，", u","),
    (u"、", u","),
    (u"”", u'"'),
    (u"“", u'"'),
    (u"∶", u":"),
    (u"：", u":"),
    (u"？", u"?"),
    (u"《", u'"'),
    (u"》", u'"'),
    (u"）", u")"),
    (u"！", u"!"),
    (u"（", u"("),
    (u"；", u";"),
    (u"」", u'"'),
    (u"「", u'"'),
    (u"０", u"0"),
    (u"１", u'1'),
    (u"２", u"2"),
    (u"３", u"3"),
    (u"４", u"4"),
    (u"５", u"5"),
    (u"６", u"6"),
    (u"７", u"7"),
    (u"８", u"8"),
    (u"９", u"9"),
    (u"～", u"~"),
    (u"’", u"'"),
    (u"…", u"..."),
    (u"━", u"-"),
    (u"〈", u"<"),
    (u"〉", u">"),
    (u"【", u"["),
    (u"】", u"]"),
    (u"％", u"%"),
]

NORMALIZE_UNICODE = [
    ('\u00AD', ''),
    ('\u09AF\u09BC', '\u09DF'),
    ('\u09A2\u09BC', '\u09DD'),
    ('\u09A1\u09BC', '\u09DC'),
    ('\u09AC\u09BC', '\u09B0'),
    ('\u09C7\u09BE', '\u09CB'),
    ('\u09C7\u09D7', '\u09CC'),
    ('\u0985\u09BE', '\u0986'),
    ('\u09C7\u0981\u09D7', '\u09CC\u0981'),
    ('\u09C7\u0981\u09BE', '\u09CB\u0981'),
    ('\u09C7([^\u09D7])\u09D7', "\g<1>\u09CC"),
    ('\\xa0', ' '),
    ('\u200B', u''),  
    ('\u2060', u''),
    (u'„', r'"'),
    (u'“', r'"'),
    (u'”', r'"'),
    (u'–', r'-'),
    (u'—', r' - '),
    (r' +', r' '),
    (u'´', r"'"),
    (u'([a-zA-Z])‘([a-zA-Z])', r"\g<1>'\g<2>"),
    (u'([a-zA-Z])’([a-zA-Z])', r"\g<1>'\g<2>"),
    (u'‘', r"'"),
    (u'‚', r"'"),
    (u'’', r"'"),
    (u'´´', r'"'),
    (u'…', r'...'),
]

FRENCH_QUOTES = [
    (u'\u00A0«\u00A0', r'"'),
    (u'«\u00A0', r'"'),
    (u'«', r'"'),
    (u'\u00A0»\u00A0', r'"'),
    (u'\u00A0»', r'"'),
    (u'»', r'"'),
]

SUBSTITUTIONS = [NORMALIZE_UNICODE, FRENCH_QUOTES, REPLACE_UNICODE_PUNCTUATION]
SUBSTITUTIONS = list(chain(*SUBSTITUTIONS))

BANGLA_CHARS = (
    r'['
    r'\u0981-\u0983'
    r'\u0985-\u098B'
    r'\u098F-\u0990'
    r'\u0993-\u09A8'
    r'\u09AA-\u09B0'
    r'\u09B2'
    r'\u09B6-\u09B9'
    r'\u09BC'
    r'\u09BE-\u09C3'
    r'\u09C7-\u09C8'
    r'\u09CB-\u09CC'
    r'\u09CE'
    r'\u09D7'
    r'\u09DC-\u09DD'
    r'\u09DF'
    r'\u09E6-\u09EF'
    r'\u09F3'
    r'\u0964'
    r']'
)

NEUTRAL_CHARS = (
    r'['
    r'\s'
    r'\u09CD'
    r'\u0021-\u002F'
    r'\u003A-\u0040'
    r'\u005B-\u0060'
    r'\u007B-\u007E'
    r'\u00A0'
    r'\u00A3'
    r'\u00B0'
    r'\u2000-\u2014'
    r'\u2018-\u201D'
    r'\u2028-\u202F'
    r'\u2032-\u2033'
    r'\u2035-\u2036'
    r'\u2060-\u206F'
    r']'
)

ENGLISH_CHARS = (
    r'[a-zA-Z0-9]'
)

NON_BANGLA_PATCH = (
    r'['
    r'^'
    r'\u0981-\u0983'
    r'\u0985-\u098B'
    r'\u098F-\u0990'
    r'\u0993-\u09A8'
    r'\u09AA-\u09B0'
    r'\u09B2'
    r'\u09B6-\u09B9'
    r'\u09BC'
    r'\u09BE-\u09C3'
    r'\u09C7-\u09C8'
    r'\u09CB-\u09CC'
    r'\u09CE'
    r'\u09D7'
    r'\u09DC-\u09DD'
    r'\u09DF'
    r'\u09E6-\u09EF'
    r'\u09F3'
    r'\u0964'
    r'\s'
    r'\u09CD'
    r'\u0021-\u002F'
    r'\u003A-\u0040'
    r'\u005B-\u0060'
    r'\u007B-\u007E'
    r'\u00A0'
    r'\u00A3'
    r'\u00B0'
    r'\u2000-\u2014'
    r'\u2018-\u201D'
    r'\u2028-\u202F'
    r'\u2032-\u2033'
    r'\u2035-\u2036'
    r'\u2060-\u206F'
    r']'
    r'+'
)

NON_ENGLISH_PATCH = (
    r'['
    r'^'
    r'a-z'
    r'A-Z'
    r'0-9'
    r'\s'
    r'\u09CD'
    r'\u0021-\u002F'
    r'\u003A-\u0040'
    r'\u005B-\u0060'
    r'\u007B-\u007E'
    r'\u00A0'
    r'\u00A3'
    r'\u00B0'
    r'\u2000-\u2014'
    r'\u2018-\u201D'
    r'\u2028-\u202F'
    r'\u2032-\u2033'
    r'\u2035-\u2036'
    r'\u2060-\u206F'
    r']'
    r'+'
)

WHITESPACE_PUNCTATION = (
    r'[\(\[\{'
    r'\u0021-\u0027'
    r'\u002A-\u002F'
    r'\u003A-\u0040'
    r'\u005C'
    r'\u005E-\u0060'
    r'\u007C'
    r'\u007E'
    r'\u02B9-\u02DD'
    r'\u09F7'
    r'\u0964'
    r'\u0965'
    r'\u2010-\u201F'
    r'\s\t'
    r'\)\]\}]'
    r'+'
)

INCLUSIVE_NON_BANGLA_PATCH = (
    r'[\(\[\{'
    r'a-zA-Z'
    r'\u00A1-\u00AC'
    r'\u00AE-\u02FF'
    r'\u0300-\u07BF'
    r'\u0900'
    r'\u0904-\u094D'
    r'\u094E-\u0950'
    r'\u0955-\u0963'
    r'\u0966-\u097F'
    r'\u0A00-\u1FFF'
    r'\u2020-\u2027'
    r'\u2030-\u2031'
    r'\u203B-\u205E'
    r'\u2070-\uFE4F'
    r'\uFE70-\uFEFF'
    r'\uFF21-\uFF3A'
    r'\uFF41-\uFF5A'
    r'\uFF5F-\uFFEF'
    r'\uFFF9-\uFFFF'
    r'\u0021-\u0027'
    r'\u002A-\u002F'
    r'\u003A-\u0040'
    r'\u005C'
    r'\u005E-\u0060'
    r'\u007C'
    r'\u007E'
    r'\u02B9-\u02DD'
    r'\u2010-\u201F'
    r'0-9'
    r'\s\t'
    r'\)\]\}]'
    '+'
)

BRACKETED_SPANS = (
    r'[\(\[\{]'
    r'['
    r'\u0021-\u0027'
    r'\u002A-\u002F'
    r'\u003A-\u0040'
    r'\u005C'
    r'\u005E-\u0060'
    r'\u007C'
    r'\u007E'
    r'\u02B9-\u02DD'
    r'\u2010-\u201F'
    r'\s\t'
    r']'
    r'*'
    r'[\)\]\}]'
)

def normalize(text):
    for regexp, replacement in SUBSTITUTIONS:
        text = re.sub(regexp, replacement, text, flags=re.UNICODE)
    
    text = re.sub(r'\s+', ' ', text)
    return text.strip()

def readFile(filename):
    with open(filename) as f:
        lines = [line.strip() for line in f.readlines()]
        return lines

def readFilePair(bnFile, enFile):
    bnLines = readFile(bnFile)
    enLines = readFile(enFile)
    
    return zip(bnLines, enLines)

def readReplacePatterns(filename):
    """
        Patterns should have the following form in each line:
        `Pattern:enReplacement(:optional bnReplacement)`

        Both Pattern and Replacement can contain arbitrary no of spaces.
        Be careful not to place unnecessary spaces.
        In absence of bnReplacement, bnReplacement = enReplacement
    """
    enPatternMap, bnPatternMap = {}, {}

    with open(filename) as f:
        for line in f.readlines():
            try:
                splitLine = line.rstrip('\n').split(":")
                pattern = splitLine[0]
                enReplacement = splitLine[1]
                if len(splitLine) == 3:
                    bnPatternMap[pattern] = splitLine[2]
                else:
                    bnPatternMap[pattern] = enReplacement
                enPatternMap[pattern] = enReplacement
            except:
                continue
    
    return enPatternMap, bnPatternMap

def hasNonBangla(line):
    return len(line) - countBanglaChars(line) - countNeutralChars(line)

def hasNonEnglish(line):
    return len(line) - countEnglishChars(line) - countNeutralChars(line)

def hasOOV(line):
    return len(line) - countBanglaChars(line) - countEnglishChars(line) - countNeutralChars(line)

def countBanglaChars(line):
    chars = re.findall(
        BANGLA_CHARS,
        line,
        flags=re.UNICODE
    )
    return len(chars)

def countEnglishChars(line):
    chars = re.findall(
        ENGLISH_CHARS,
        line,
        flags=re.UNICODE
    )
    return len(chars)

def countNeutralChars(line):
    chars = re.findall(
        NEUTRAL_CHARS,
        line,
        flags=re.UNICODE
    )
    return len(chars)

def getNonBanglaPatches(line):
    return re.findall(
        NON_BANGLA_PATCH,
        line,
        flags=re.UNICODE
    )

def getNonEnglishPatches(line):
    return re.findall(
        NON_ENGLISH_PATCH,
        line,
        flags=re.UNICODE
    )

def replaceEmojis(*lines):
    outputLines = []
    for line in lines:
        line = re.sub(
            r'\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]',
            "",
            line,
            flags=re.UNICODE
        )
        line = re.sub(r'[\U00010000-\U0010ffff]', "", line, flags=re.UNICODE)
        outputLines.append(line)

    return tuple(outputLines)

def isValidProcessedPair(bnLine, enLine):
    if bnLine.strip() == "" or enLine.strip() == "":
        return False 

    # check if either side contains only punctuations and whitespaces
    
    if (
            re.sub(WHITESPACE_PUNCTATION, "", bnLine.strip(), flags=re.UNICODE) == "" or 
            re.sub(WHITESPACE_PUNCTATION, "", enLine.strip(), flags=re.UNICODE) == ""
        ):
        return False

    return True
   
def writeFilePairs(dir, validPairs, foreignPairs, savedPairs=None):
    os.makedirs(dir, exist_ok=True)

    with open(os.path.join(dir, "cleaned.bn"), 'w') as bn, \
        open(os.path.join(dir, "cleaned.en"), 'w') as en:
        for bnLine, enLine in validPairs:
            print(bnLine, file=bn)
            print(enLine, file=en)

    with open(os.path.join(dir, "filtered.bn"), 'w') as bn, \
        open(os.path.join(dir, "filtered.en"), 'w') as en:
        for bnLine, enLine in foreignPairs:
            print(bnLine, file=bn)
            print(enLine, file=en)
    
    if savedPairs:
        with open(os.path.join(dir, "savedOrignal.bn"), 'w') as bnO, \
            open(os.path.join(dir, "savedOrignal.en"), 'w') as enO, \
            open(os.path.join(dir, "saved.bn"), 'w') as bnS, \
            open(os.path.join(dir, "saved.en"), 'w') as enS:
            for bnOriginal, enOriginal, bnSaved, enSaved in savedPairs:
                print(bnOriginal, file=bnO)
                print(enOriginal, file=enO)
                print(bnSaved, file=bnS)
                print(enSaved, file=enS)

def convertNumerals(bnLine, enLine):
    newBnLine, newEnLine = bnLine, enLine

    for enNumeral in re.findall(r'[0-9]+', newBnLine, flags=re.UNICODE):
        newBnLine = newBnLine.replace(enNumeral, transliterate.process('RomanReadable', 'Bengali', enNumeral), 1)

    for bnNumeral in re.findall(r'[০-৯]+', newEnLine, flags=re.UNICODE):
        newEnLine = newEnLine.replace(bnNumeral, transliterate.process('Bengali', 'RomanReadable', bnNumeral), 1)

    return newBnLine, newEnLine 

def applyPatterns(line, patternMap):
    """
    Returns:
    patternFound(bool) : Whether any of the patterns were found in the line
    line(str) : Transformed line using pattern replacements
    """
    patternFound = False
    orignalLine = line
    for pattern, replacement in patternMap.items():
        line = line.replace(pattern, replacement)
        if orignalLine != line:
            patternFound = True

    return patternFound, line

def patternHandler(bnLine, enLine, verbose):
    _, newEnLine = applyPatterns(enLine, enPatternMap)
    _, newBnLine = applyPatterns(bnLine, bnPatternMap)

    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            patternHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                patternHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            patternHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def patchHandler(bnLine, enLine, verbose):
    """
    finds continous patches of non bangla text on bangla side and removes
    a found patch from both sides if it is also present in english side  
    """

    minPatchLength = 2 # this shouldn't be too small
    newBnLine, newEnLine = bnLine, enLine
    nonBanglaPatches = re.findall(
        INCLUSIVE_NON_BANGLA_PATCH,
        newBnLine,
        flags=re.UNICODE
    )

    for patch in nonBanglaPatches:
        patch = patch.strip()

        # ignore whitespace only patches
        if patch == "":
            continue
        # ignore number only patches
        try:
            float(patch)
            continue
        except:
            pass
        # patch shouldnt end in starting braces
        if patch[-1] in ['[', '{', '(']:
            patch = patch[:-1].strip()
        # patch shouldnt start with ending braces/common punctuations
        if patch and patch[0] in [']', '}', ')', ',', '.']:
            patch = patch[1:].strip()

        # should the patch length be counted with spaces included??
        # should all matching patches be removed in english side?
        if (
                len(patch) >= minPatchLength and
                (
                    patch in newEnLine or
                    patch.upper() in newEnLine or 
                    patch.lower() in newEnLine or
                    patch.capitalize() in newEnLine
                )
            ):
            newBnLine = newBnLine.replace(patch, "").strip()
            newEnLine = newEnLine.replace(
                patch, ""
            ).replace(
                patch.lower(), ""
            ).replace(
                patch.upper(), ""
            ).replace(
                patch.capitalize(), ""
            ).strip()


    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            patchHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                patchHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            patchHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def alternatePatchHandler(bnLine, enLine, verbose):
    """
    finds common patches in both lines and removes a patch
    from bothsides if the found patch is non bangla
    """

    def isValidPatch(patch):
        patch = re.sub(
            INCLUSIVE_NON_BANGLA_PATCH,
            "",
            patch,
            flags=re.UNICODE
        ).strip()
        return patch == ""

    minPatchLength = 2 # this shouldn't be too small
    newBnLine, newEnLine = bnLine, enLine

    matcher = difflib.SequenceMatcher(None, newBnLine, newEnLine)
    potentialPatches = [newBnLine[match.a : match.a + match.size] for match in matcher.get_matching_blocks() if match.size > 0]

    for patch in potentialPatches:
        patch = patch.strip()

        # ignore whitespace only patches
        if patch == "":
            continue
        # ignore number only patches
        try:
            float(patch)
            continue
        except:
            pass

        # patch shouldnt end in starting braces
        if patch[-1] in ['[', '{', '(']:
            patch = patch[:-1].strip()
        # patch shouldnt start with ending braces/common punctuations
        if patch and patch[0] in [']', '}', ')', ',', '.']:
            patch = patch[1:].strip()

        # should the patch length be counted with spaces included??
        # should all matching patches be removed in english side?
        if (
                isValidPatch(patch) and
                len(patch) >= minPatchLength and
                (
                    patch in newEnLine or
                    patch.upper() in newEnLine or 
                    patch.lower() in newEnLine or
                    patch.capitalize() in newEnLine
                )
            ):
            newBnLine = newBnLine.replace(patch, "").strip()
            newEnLine = newEnLine.replace(
                patch, ""
            ).replace(
                patch.lower(), ""
            ).replace(
                patch.upper(), ""
            ).replace(
                patch.capitalize(), ""
            ).strip()

    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            alternatePatchHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                alternatePatchHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            alternatePatchHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def bpediaPatchHandler(bnLine, enLine, verbose):
    """
    removes non bangla patches from first-bracketed spans on bangla side.
    Specific to Banglapedia.
    """
    newBnLine, newEnLine = bnLine, enLine
    bracketedSpans = re.findall(r'\([^\)]*?\)', newBnLine, flags=re.UNICODE)

    for span in bracketedSpans:
        if hasNonBangla(span):
            # if span contains no bangla text, remove it
            if not countBanglaChars(span):
                newBnLine = newBnLine.replace(span, "").strip()
            else:
                nonBanglaPatches = re.findall(
                    INCLUSIVE_NON_BANGLA_PATCH,
                    span,
                    flags=re.UNICODE
                )
                newSpan = span
                for patch in nonBanglaPatches:
                    if isValidProcessedPair(patch, patch):
                        newSpan = newSpan.replace(patch.strip(), "")

                newBnLine = newBnLine.replace(span, newSpan.strip()).strip()

    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            bpediaPatchHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                bpediaPatchHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            bpediaPatchHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def transliterateHandler(bnLine, enLine, verbose):
    newBnLine, newEnLine = bnLine, enLine
    # convert numerals to appropriate script
    newBnLine, newEnLine = convertNumerals(newBnLine, newEnLine)

    charConvertMap = {
        'a' : 'এ',
        'b' : 'বি',
        'c' : 'সি',
        'd' : 'ডি',
        'e' : 'ই',
        'f' : 'এফ',
        'g' : 'জি',
        'h' : 'এইচ',
        'i' : 'আই',
        'j' : 'জে',
        'k' : 'কে',
        'l' : 'এল',
        'm' : 'এম',
        'n' : 'এন',
        'o' : 'ও',
        'p' : 'পি',
        'q' : 'কিউ',
        'r' : 'আর',
        's' : 'এস',
        't' : 'টি',
        'u' : 'ইউ',
        'v' : 'ভি',
        'w' : 'ডব্লিউ',
        'x' : 'এক্স',
        'y' : 'ওআই',
        'z' : 'জেড',
    }

    def customReplace(match):
        caughtChar = match.group(1).strip()
        return charConvertMap[caughtChar.lower()]

    newBnLine = re.sub(r'(\b[a-zA-Z]\b)', customReplace, newBnLine, flags=re.UNICODE)

    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            transliterateHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                transliterateHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            transliterateHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def ratioHandler(bnLine, enLine, verbose):
    newBnLine, newEnLine = bnLine, enLine

    bnLowerThresh = .001
    bnUpperThresh = 10.0

    enLowerThresh = .001
    enUpperThresh = 10.0


    bnNativeChars = countBanglaChars(bnLine)
    bnForeignChars = len(bnLine) - bnNativeChars - countNeutralChars(bnLine)
    bnRatio = bnForeignChars/bnNativeChars
    
    enNativeChars = countEnglishChars(enLine)
    enForeignChars = len(enLine) - enNativeChars - countNeutralChars(enLine)
    enRatio = enForeignChars/enNativeChars

    if verbose:
        # if there is foreign text in the linepair after applying replacements
        if (
                bnRatio >= bnUpperThresh or
                enRatio >= enUpperThresh or
                bnRatio <= bnLowerThresh or
                enRatio <= enLowerThresh or
                countBanglaChars(enLine) or countEnglishChars(bnLine)
            ):
            ratioHandler.foreignPairs.append((newBnLine, newEnLine))
        else:
            # if the original linepair had foreign characters
            if bnForeignChars or enForeignChars:
                ratioHandler.savedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            ratioHandler.validPairs.append((newBnLine, newEnLine))

    return newBnLine, newEnLine

def cleanSentencePairs(linePairs, options, output_dir, verbose=True):
    for option, choice in options.items():
        if choice:
            globals()[f'{option}Handler'].validPairs = multiprocessing.Manager().list()
            globals()[f'{option}Handler'].foreignPairs = multiprocessing.Manager().list()
            globals()[f'{option}Handler'].savedPairs = multiprocessing.Manager().list()
    

    global finalValidPairs, finalForeignPairs, finalSavedPairs, initialValidPairs, initialForeignPairs

    finalValidPairs, finalForeignPairs, finalSavedPairs = (
        multiprocessing.Manager().list(), 
        multiprocessing.Manager().list(),
        multiprocessing.Manager().list()
    )
    initialValidPairs, initialForeignPairs = (
        multiprocessing.Manager().list(),
        multiprocessing.Manager().list()
    )

    @globalize
    def processPair(bnLine, enLine):
        validPair = False

        if bnLine == enLine or (not countBanglaChars(bnLine) or not countEnglishChars(enLine)):
            return

        if hasNonBangla(bnLine) or hasNonEnglish(enLine):
            initialForeignPairs.append((bnLine, enLine))
        else:
            initialValidPairs.append((bnLine, enLine))
            validPair = True

        # apply all selected transformations
        newBnLine, newEnLine = bnLine, enLine

        newBnLine = normalize(newBnLine).strip()
        newEnLine = normalize(newEnLine).strip()
        newBnLine, newEnLine = replaceEmojis(newBnLine, newEnLine)

        # remove unnecessary bracketed spans; this needs to be done first for patch removal to work
        newBnLine = re.sub(
            BRACKETED_SPANS,
            "",
            newBnLine,
            flags=re.UNICODE
        ) 
        newEnLine = re.sub(
            BRACKETED_SPANS,
            "",
            newEnLine,
            flags=re.UNICODE
        ) 
        
        for option, choice in options.items():
            if choice and not validPair:
                newBnLine, newEnLine = globals()[f'{option}Handler'](newBnLine, newEnLine, verbose)
    
        # do some final postprocessing
        if not isValidProcessedPair(newBnLine, newEnLine):
            return

        # remove unnecessary bracketed spans (after patch deletion)
        newBnLine = re.sub(
            BRACKETED_SPANS,
            "",
            newBnLine,
            flags=re.UNICODE
        ) 
        newEnLine = re.sub(
            BRACKETED_SPANS,
            "",
            newEnLine,
            flags=re.UNICODE
        )

        newBnLine = re.sub(r'\s+', ' ', newBnLine)
        newEnLine = re.sub(r'\s+', ' ', newEnLine)
    

        # if there is foreign text in the linepair after applying transformations
        if hasNonBangla(newBnLine) or hasNonEnglish(newEnLine):
            finalForeignPairs.append((bnLine, enLine))
        else:
            # if the original linepair had foreign characters
            if hasNonBangla(bnLine) or hasNonEnglish(enLine):
                finalSavedPairs.append((bnLine, enLine, newBnLine, newEnLine))

            finalValidPairs.append((newBnLine, newEnLine))

    print('\tStarting processes...')
    with Pool() as pool:
        pool.starmap(processPair, linePairs)

    print('\tWriting logs and outputs...')
    # write the final and temporary filepairs to appropriate directories
    writeFilePairs(
        os.path.join(output_dir, 'Final'),
        finalValidPairs, finalForeignPairs, finalSavedPairs
    )

    if verbose:
        writeFilePairs(
            os.path.join(output_dir, "tmp", "Initial"), 
            initialValidPairs, initialForeignPairs)

        for option, choice in options.items():
            if choice:
                funcName = globals()[f'{option}Handler']
                writeFilePairs(
                    os.path.join(output_dir, "tmp", option), 
                    funcName.validPairs,
                    funcName.foreignPairs,
                    funcName.savedPairs
                )


def col(id):
	if id == 1: return "\033[32m"
	if id == 2: return "\033[33m"
	if id == 3: return "\033[31m"
	return "\033[0m"

def cleanup(dirname):
    for sub_dir in ["Final", "tmp"]:
        if os.path.isdir(os.path.join(dirname, sub_dir)):
            shutil.rmtree(os.path.join(dirname, sub_dir))

def _merge(input_files, output_file):
    with open(output_file, 'w') as outf:
        for input_file in input_files:
            with open(input_file) as inpf:
                for line in inpf:
                    print(line.strip(), file=outf)

def _train(input_file, output_prefix, coverage, vocab_size, model_type):
    cmd = [
        f"spm_train --input=\"{input_file}\"",
        f"--model_prefix=\"{output_prefix}\"",
        f"--vocab_size={vocab_size}",
        f"--character_coverage={coverage}",
        "--train_extremely_large_corpus"
    ]
    os.system(" ".join(cmd))

def main(args):
    global enPatternMap, bnPatternMap
    enPatternMap, bnPatternMap = readReplacePatterns(
        os.path.join(os.path.dirname(__file__), "replacePatterns.txt")
    )
    if os.path.isdir(args.output_dir):
        shutil.rmtree(args.output_dir)

    if args.normalize:
        iterator = tqdm(
            glob.glob(os.path.join(args.input_dir, "**", "*.bn"), recursive=True) +
            glob.glob(os.path.join(args.input_dir, "**", "*.en"), recursive=True),
            desc="Normalizing files"
        )
        for input_file in iterator:
            output_file = input_file.replace(
                os.path.normpath(args.input_dir),
                os.path.normpath(args.output_dir) 
            )
            os.makedirs(os.path.dirname(output_file), exist_ok=True)
            with open(output_file, 'w') as outf:
                with open(input_file) as inpf:
                    for line in inpf:
                        line = normalize(line)
                        _, line = applyPatterns(line, enPatternMap)
                        _, line = applyPatterns(line, bnPatternMap)
                        print(line.strip(), file=outf)
    else:
        cleanup(args.output_dir)
        os.makedirs(os.path.join(args.output_dir, "data"), exist_ok=True)

        linePair_list = []
        for bnFile in glob.glob(os.path.join(args.input_dir, "**", f"*.bn"), recursive=True):
            enFile = bnFile[:-3] + ".en"
            if not os.path.isfile(enFile):
                continue
            linePair_list.append(readFilePair(bnFile, enFile))
        
        linePairs = list(chain.from_iterable(linePair_list))

        print(col(2) + 'Starting Stage 1...' + col(0))
        cleanSentencePairs(linePairs, {'pattern': True}, args.output_dir)

        shutil.copy(
            os.path.join(args.output_dir, "Final", "cleaned.bn"),
            os.path.join(args.output_dir, "data", "data1.bn")
        )
        shutil.copy(
            os.path.join(args.output_dir, "Final", "cleaned.en"),
            os.path.join(args.output_dir, "data", "data1.en")
        )
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "pattern", "filtered.bn"),
            os.path.join(args.output_dir, "data", "stage1Filtered.bn")
        )
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "pattern", "filtered.en"),
            os.path.join(args.output_dir, "data", "stage1Filtered.en")
        )

        print(col(2) + 'Starting Stage 2...' + col(0))
        cleanup(args.output_dir)
        linePairs = readFilePair(
            os.path.join(args.output_dir, "data", "stage1Filtered.bn"),
            os.path.join(args.output_dir, "data", "stage1Filtered.en")
        )
        cleanSentencePairs(linePairs, {'ratio': True}, args.output_dir)
        
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "ratio", "cleaned.bn"),
            os.path.join(args.output_dir, "data", "data2.bn")
        )
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "ratio", "cleaned.en"),
            os.path.join(args.output_dir, "data", "data2.en")
        )
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "ratio", "filtered.bn"),
            os.path.join(args.output_dir, "data", "stage2Filtered.bn")
        )
        shutil.copy(
            os.path.join(args.output_dir, "tmp", "ratio", "filtered.en"),
            os.path.join(args.output_dir, "data", "stage2Filtered.en")
        )


        print(col(2) + 'Starting Stage 3...' + col(0))
        cleanup(args.output_dir)
        linePairs = readFilePair(
            os.path.join(args.output_dir, "data", "stage2Filtered.bn"),
            os.path.join(args.output_dir, "data", "stage2Filtered.en")
        )
        cleanSentencePairs(
            linePairs,
            {
                'patch': True,
                'alternatePatch': True,
                'transliterate': True
            },
            args.output_dir
        )
        
        shutil.copy(
            os.path.join(args.output_dir, "Final", "cleaned.bn"),
            os.path.join(args.output_dir, "data", "data3.bn")
        )
        shutil.copy(
            os.path.join(args.output_dir, "Final", "cleaned.en"),
            os.path.join(args.output_dir, "data", "data3.en")
        )
        
        _merge(
            [
                os.path.join(args.output_dir, "data", "data1.bn"),
                os.path.join(args.output_dir, "data", "data2.bn"),
                os.path.join(args.output_dir, "data", "data3.bn"),
            ],
            os.path.join(args.output_dir, "combined.bn")
        )
        _merge(
            [
                os.path.join(args.output_dir, "data", "data1.en"),
                os.path.join(args.output_dir, "data", "data2.en"),
                os.path.join(args.output_dir, "data", "data3.en"),
            ],
            os.path.join(args.output_dir, "combined.en")
        )
        _merge(
            [
                os.path.join(args.output_dir, "data", "data1.bn"),
                os.path.join(args.output_dir, "data", "data3.bn"),
            ],
            os.path.join(args.output_dir, "vocab_train.bn")
        )
        _merge(
            [
                os.path.join(args.output_dir, "data", "data1.en"),
                os.path.join(args.output_dir, "data", "data3.en"),
            ],
            os.path.join(args.output_dir, "vocab_train.en")
        )

        _train(
            os.path.join(args.output_dir, "vocab_train.bn"),
            os.path.join(args.output_dir, "bn"),
            args.bn_coverage,
            args.bn_vocab_size,
            args.bn_model_type
        )

        _train(
            os.path.join(args.output_dir, "vocab_train.en"),
            os.path.join(args.output_dir, "en"),
            args.en_coverage,
            args.en_vocab_size,
            args.en_model_type
        )

        shutil.rmtree(os.path.join(args.output_dir, "data"))
        os.remove(os.path.join(args.output_dir, "vocab_train.bn"))
        os.remove(os.path.join(args.output_dir, "vocab_train.en"))
        cleanup(args.output_dir)



if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--input_dir', '-i', type=str,
        required=True,
        metavar='PATH',
        help="Input directory")

    parser.add_argument(
        '--output_dir', '-o', type=str,
        required=True,
        metavar='PATH',
        help="Output directory")

    parser.add_argument('--normalize', action='store_true',
        help='Only normalize the files in input directory')

    parser.add_argument(
        '--bn_vocab_size', type=int, default=32000, 
        help='bengali vocab size')

    parser.add_argument(
        '--en_vocab_size', type=int, default=32000, 
        help='english vocab size')

    parser.add_argument(
        '--bn_model_type', type=str, default="unigram", 
        help='bengali sentencepiece model type')

    parser.add_argument(
        '--en_model_type', type=str, default="unigram", 
        help='english sentencepiece model type')

    parser.add_argument(
        '--bn_coverage', type=float, default=1.0, 
        help='bengali character coverage')

    parser.add_argument(
        '--en_coverage', type=float, default=1.0, 
        help='english character coverage')

    args = parser.parse_args()
    main(args)