mdpre

#!/usr/bin/env python3
"""
mdpre - Preprocesses a file into Markdown

First argument is file to write to

Reads from stdin

"""

# TODO: (later) new log instead of sys.stderr.write + verbose
# With log( ) we could replace szs.stderr.write
# With log_verbose( ) we could more the if VERBOSE inside the function

# TODO: Warning - mdpre>(node:29276) [DEP0005]
# DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the
# Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.


import os
import sys
import csv
import shutil
import zipfile
import io

# import StringIO
import re
import datetime
from functools import partial
from pathlib import Path
from enum import Enum
import calendar


mdpre_level = "0.6.8"
mdpre_date = "8 March, 2024"
banner = "mdpre Markdown Preprocessor v" + mdpre_level + " (" + mdpre_date + ")"

log = partial(print, file=sys.stderr)  # | create log to stderr


monthNames = [
    "January",
    "February",
    "March",
    "April",
    "May",
    "June",
    "July",
    "August",
    "September",
    "October",
    "November",
    "December",
]


def log_status(p_line):
    log("-" * len(banner))
    log(p_line)
    log("-" * len(banner) + "\n")


g_output = None
g_wrapInCSV = False

csv_colaligns = []
csv_colwidths = []
in_csv = False

cal_date = (0, 0, 2)
in_cal = False

# Find index of variable in variable list. -1 means "not found"
def find_variable(targetVar):
    global vars
    foundVar = -1
    cursor = 0
    for var in vars:
        if var[0] == targetVar:
            varRecord = var
            foundVar = cursor
            break
        cursor += 1
    return foundVar

# Set a variable for the first time - or update it if it already exists
def setOrUpdateVariable(varName, varValue):
    global vars
    var = parse_def(varName + " " + varValue)
    var_index = find_variable(var[0])
    if var_index > -1:
        # Replace the variable with its new definition
        vars[var_index] = var
    else:
         # Add the variable as it's new
        vars.append(var)

class OutputType(Enum):
    """preparation for later usage. support e.g. other outputs like MMD or ADOC"""

    MD = 1


class Output(object):
    NL = "\n"
    output = OutputType.MD

    def __init__(self, p_toFile):
        self.toFile = None
        if hasattr(p_toFile, "write"):
            self.toFile = p_toFile
            log(f"- opened {p_toFile.name} for writing")
        else:
            log(f"ERROR: g_output could not find an open file {p_toFile}")
            exit_script(g_output, -101)

    def close(self):
        self.toFile.close()

    def write(self, p_text):
        if self.output == OutputType.MD:
            self.toFile.write(f"{p_text}{self.NL}")

    def write_line(self, p_text):
        self.write(f"{p_text}{self.NL}")

    def formatCSV(self, CSV_lines, colalign, colwidth):
        CSV_reader = csv.reader(CSV_lines)

        # Find out how many columns and prime CSV_lines
        columns = 0
        CSV_lines = []
        for row in CSV_reader:
            columns = max(columns, len(row))
            CSV_lines.append(row)

        # Extend colalign to cover all columns - now we know how many there are
        colalign_len = len(colalign)
        for i in range(colalign_len, columns + 1):
            colalign.append("l")

        # Extend colwidth to cover all columns - now we know how many there are
        colwidth_len = len(colwidth)
        for i in range(colwidth_len, columns + 1):
            colwidth.append(1)

        # Now print the table
        first_row = True
        for row in CSV_lines:
            row_text = "|"
            cell_prefix = ""
            cell_suffix = ""
            for col_index, col_text in enumerate(row):
                if (col_index == 0) & (col_text.startswith("=rowspan ")):
                    rowspan2 = col_text.split("!!XYZZY!!")
                    spanClass = rowspan2[0][9:]
                    col_text = rowspan2[1][1:-1]
                    cell_prefix = "<span class=\"" + spanClass +"\">"
                    cell_suffix = "</span>"
                      
                row_text = row_text + cell_prefix + col_text + cell_suffix + "|"
            self.write(row_text)

            titleUnderline = "|"
            if first_row is True:
                for col_index in range(columns):
                    dashes = "-" * (colwidth[col_index])
                    # NOTE: col_width is ratio not characters
                    if colalign[col_index] == "r":
                        titleUnderline = titleUnderline + dashes + ":|"
                    elif colalign[col_index] == "c":
                        titleUnderline = titleUnderline + ":" + dashes + ":|"
                    elif colalign[col_index] == "l":
                        titleUnderline = titleUnderline + ":" + dashes + "|"
                    else:
                        titleUnderline = titleUnderline + dashes + "|"

                self.write(titleUnderline)
                first_row = False
        return

    def formatCalendar(self, cal_lines, cal_date, cal_spans, cal_keys, cal_notes):
        # Get a calendar month in a 2-dimensional array
        cal_year, cal_month, cal_cellHeight = cal_date
        c = calendar.TextCalendar(calendar.MONDAY)
        daysAndNumbers = c.formatmonth(cal_year, cal_month).split("\n")[1: -1]
        

        # Process the rows into day names and actual day numbers
        weeks = []

        for rowNumber, row in enumerate(daysAndNumbers):
            rawCells = []
            row = row.ljust(21)
            for d in range(7):
                rawCells.append(str(row[d * 3 : d * 3 +3]).lstrip().rstrip())

            if rowNumber == 0:
                dayNames = rawCells
            else:
                weeks.append(rawCells)

        # Write month line
        self.write("|" + monthNames[cal_month - 1] + " " + str(cal_year) + "|||||||")
        
        # Write the table heading
        self.write(("|:-:") * 7 + "|")

        dayLine = "|"
        for dayName in dayNames:
            dayLine = dayLine + "**" + dayName + "**|"

        self.write(dayLine)

        # Write the remaining lines
        for week in weeks:
            weekLine = "|"
            for day in week:
                # Figure out if day has a note
                if day in cal_notes.keys():
                    dayNote = "<br/>" + cal_notes[day]
                    cellLines = len(dayNote.split("<br/>"))
                else:
                    dayNote = ""
                    cellLines = 1

                # Compose day Markdown / HTML
                if day in cal_spans.keys():
                    formattedDay = (
                        "<span class='"
                        + cal_spans[day]
                        + "'>"
                        + day
                        + "</span>"
                        + dayNote
                    )
                else:
                    formattedDay = day + dayNote

                # Make blank cell appear
                if formattedDay == "":
                    formattedDay = "&nbsp;"
                    
                # Vertically pad
                formattedDay = formattedDay + "<br/>" * (cal_cellHeight - cellLines)

                # Add day cell to week line
                weekLine = weekLine + formattedDay + "|"

            # Write week line
            self.write(weekLine)

        # Write any descriptions
        self.write("")
        for key in cal_keys.keys():
            self.write("* <span class='" + key + "'>&nbsp;</span>" + cal_keys[key])
        self.write("")

    # Print Table Of Contents down to maximum level
    def printTableOfContents(self, minLevel, maxLevel, heading, toc):

        self.write_line(heading)
        for tocEntryLevel, tocEntryText in toc:
            if (tocEntryLevel >= minLevel) & (tocEntryLevel <= maxLevel):
                self.write(
                    ("\t" * (tocEntryLevel - minLevel))
                    + "* ["
                    + tocEntryText
                    + "](#"
                    + makeLink(tocEntryText)
                    + ")",
                )

    def handle_adjustGraphics(self, line):
        adjustedLine = ""
        splitStart = line.split("![")
        fragment = 0
        for s in splitStart:
            if fragment == 0:
                # Before first ![
                adjustedLine = s + "!["
            else:
                # At first or subsequent ![
                endLabelPos = s.find("](")
                beginGraphicPos = endLabelPos + 2
                adjustedLine += s[:beginGraphicPos]
                endGraphicPos = s.find(")")

                # Extract original graphic name, including subdirs
                graphic = s[beginGraphicPos:endGraphicPos]

                # Adjust graphic so it's in the assets subfolder
                lastSlash = graphic.rfind("/")
                if lastSlash == -1:
                    adjustedGraphic = "assets/" + graphic
                else:
                    adjustedGraphic = "assets/" + graphic[lastSlash + 1 :]

                adjustedLine = (
                    adjustedLine + adjustedGraphic + ")" + s[endGraphicPos + 1 :]
                )

                # Copy the file into the text bundle
                shutil.copy(graphic, realpath + "/" + adjustedGraphic)
            fragment += 1
        if wantVerbose is True:
            sys.stderr.write("Adjusting >>" + adjustedLine + "<<<\n")

        self.write(adjustedLine)

        return


##########################
# Class to control loops #
##########################

class LoopController:

    def __init__(self, loopType, parent, loopVarName, loopVarValues, loopValue = "", children = []):
        self.loopType = loopType
        self.parent = parent
        self.loopVarName = loopVarName
        self.loopVarValues = loopVarValues
        self.children = children
        self.lineValue = loopValue
        self.endForEncountered = False
        self.currentIndex = -1
        self.valueCount = len(loopVarValues)


####################
# Helper Functions #
####################

def intTryParse(value):
    try:
        return int(value), True
    except ValueError:
        return value, False


def parse_colalign(colString, p_separator="\s+"):
    cs = colString.rstrip().lstrip()

    return re.split(p_separator, cs)


def parse_colwidth(colString, p_separator="\s+"):
    cs = colString.rstrip().lstrip()
    colwidths = re.split(p_separator, cs)
    colwidthInt = []
    for c in colwidths:
        colwidthInt.append(int(c))

    return colwidthInt


def parse_calstart(calString, p_separator="\s+"):
    d = calString.rstrip().lstrip()
    ym = re.split(p_separator, d)

    if len(ym) < 3:
        cal_cellHeight = 2
    else:
        cal_cellHeight = int(ym[2])

    return (int(ym[0]), int(ym[1]), cal_cellHeight)


def parse_caldays(spanString, cal_spans, p_separator="\s+"):
    sp = spanString.rstrip().lstrip()
    words = re.split(p_separator, sp)

    for w in range(len(words)):
        if w == 0:
            # Pick up class name
            spanClass = words[w]
        else:
            # Pick up dates
            dayRange = words[w].split("-")
            if len(dayRange) == 1:
                # Single date specified
                dayNumber = dayRange[0]
                cal_spans[dayNumber] = spanClass
            else:
                # Range of dates specified
                for dayNumber in range(int(dayRange[0]), int(dayRange[1]) + 1):
                    cal_spans[str(dayNumber)] = spanClass

    return cal_spans


def parse_calkeys(keyString, cal_keys):
    keyString = keyString.rstrip().lstrip()
    spacePos = keyString.find(" ")
    keyName = keyString[:spacePos]
    keyValue = keyString[spacePos + 1 :]
    cal_keys[keyName] = keyValue

    return cal_keys


def parse_calnote(noteString, cal_notes):
    noteString = noteString.rstrip().lstrip()

    # Pick up single date or range
    spacePos = noteString.find(" ")
    noteDays = noteString[:spacePos]

    # Pick up the note string
    noteText = noteString[spacePos + 1 :]

    # Pick up dates
    dayRange = noteDays.split("-")
    if len(dayRange) == 1:
        # Single date specified
        dayNumber = dayRange[0]
        cal_notes[dayNumber] = noteText
    else:
        # Range of dates specified
        for dayNumber in range(int(dayRange[0]), int(dayRange[1]) + 1):
            cal_notes[str(dayNumber)] = noteText

    return cal_notes


def parse_def(defString):
    spacePos = defString.find(" ")
    varName = defString[:spacePos]
    residue = defString[spacePos + 1 :]
    if spacePos == -1:
        varType = "F"
        varValue = ""
        varName = defString
    else:
        varType = "T"
        varValue = residue

    return [varName, varType, varValue]


def parse_inc(line):
    global vars
    targetVar = line[5:-1].lstrip()
    foundVar = find_variable(targetVar)
    if foundVar > -1:
        varRecord = vars[foundVar]
        if varRecord[1] != "T":
            sys.stderr.write(
                "Variable " + targetVar + " is a flag. Cannot be incremented.\n"
            )
        else:
            intVal, OK = intTryParse(varRecord[2])
            if OK is False:
                sys.stderr.write(
                    "Variable "
                    + targetVar
                    + " is not an integer. Cannot be incremented.\n"
                )
            else:
                vars[foundVar] = [targetVar, "T", str(intVal + 1)]

    else:
        sys.stderr.write(
            "Variable " + targetVar + " not found. Cannot be incremented.\n"
        )


def parse_dec(line):
    global vars
    targetVar = line[5:-1].lstrip()
    foundVar = find_variable(targetVar)
    if foundVar > -1:
        varRecord = vars[foundVar]
        if varRecord[1] != "T":
            sys.stderr.write(
                "Variable " + targetVar + " is a flag. Cannot be decremented.\n"
            )
        else:
            intVal, OK = intTryParse(varRecord[2])
            if OK is False:
                sys.stderr.write(
                    "Variable "
                    + targetVar
                    + " is not an integer. Cannot be decremented.\n"
                )
            else:
                vars[foundVar] = [targetVar, "T", str(intVal - 1)]

    else:
        sys.stderr.write(
            "Variable " + targetVar + " not found. Cannot be decremented.\n"
        )


# =ifdef encountered
def handle_ifdef(ifdefString):
    global ifStack, vars
    verbosity = ""

    for varName, varType, varValue in vars:
        if varName == ifdefString:
            # Variable is defined
            if wantVerbose is True:
                verbosity = "<!--mdpre-ifdef:true:" + varName + "-->"

            ifStack.append(True)

            return verbosity

    # Variable is undefined
    ifStack.append(False)

    if wantVerbose is True:
        verbosity = "<!--mdpre-ifdef:false:" + ifdefString + "-->"

    return verbosity


# =ifndef encountered
def handle_ifndef(ifndefString):
    global ifStack, vars
    verbosity = ""

    for varName, varType, varValue in vars:
        if varName == ifndefString:
            # Variable is defined
            if wantVerbose is True:
                verbosity = "<!--mdpre-ifndef:false:" + ifndefString + "-->"

            ifStack.append(False)

            return verbosity

    # Variable is undefined

    ifStack.append(True)
    if wantVerbose is True:
        verbosity = "<!--mdpre-ifndef:true:" + ifndefString + "-->"

    return verbosity


# =endif encountered
def handle_endif():
    global ifStack
    if wantVerbose is True:
        verbosity = "<!--mdpre-endif:-->"
    else:
        verbosity = ""

    ifStack.pop()

    return verbosity


def handle_undef(name):
    global vars
    for var in vars:
        if var[0] == name:
            vars.remove(var)
            return ["<!--mdpre-undef:found:" + name + "-->"]

    return ["<!--mdpre-undef:notfound:" + name + "-->"]

def handle_for(forArguments):
    global rootLoopController, currentLoopController
    global vars

    ####################
    # Handle arguments #
    ####################
    forArgs=forArguments.split(" ")
    
    # Remove empty members because of multiple spaces
    forArgs = [i for i in forArgs if i]
    
    # Remove optional "="
    forArgs = [i for i in forArgs if i != "="]
    
    loopIsIn = False
    
    # Parse arguments
    argCount = len(forArgs)

    if argCount < 3:
        sys.stderr.write(f"Syntax error: {forArguments}\n")
        sys.stderr.write("Stopping.")
        sys.exit()

    elif forArgs[1].lower() == "in":
        # for / in variant
        loopVarname = forArgs[0]
        
        # Get remainder of statement after the variable and "in"
        afterIn = forArguments.find(" in ") + 4
        rest = forArguments[afterIn:]

        # ~Use CSV reader to parse the line
        loopValues = csv.reader(io.StringIO(rest), escapechar = "\\", skipinitialspace = True).__next__()

        loopIsIn = True

    elif argCount == 4:
        # for / to - without by
        loopVarname = forArgs[0]

        loopFirst = int(forArgs[1])

        if forArgs[2].lower() != "to":
            sys.stderr.write(f"Syntax error: {forArguments}\n")
            sys.stderr.write("Stopping.")
            sys.exit()
        
        loopLast = int(forArgs[3])

        loopBy = 1
    else:
        # for / to / by
        loopVarname = forArgs[0]

        loopFirst = int(forArgs[1])

        if forArgs[2].lower() != "to":
            sys.stderr.write(f"Syntax error: {forArguments}\n")
            sys.stderr.write("Stopping.")
            sys.exit()
        
        loopLast = int(forArgs[3])

        if forArgs[4].lower() != "by":
            sys.stderr.write(f"Syntax error: {forArguments}\n")
            sys.stderr.write("Stopping.")
            sys.exit()
        

        loopBy = int(forArgs[5])

    # Only use range to make list if not "in" form
    if loopIsIn is False:
    # Make values into a list
        loopLast = loopLast + loopBy
        loopValues = [*range(loopFirst, loopLast, loopBy)]

    ##############################
    # Set up the loop controller #
    ##############################

    # Register this for loop
    loopController = LoopController("for", None, loopVarname, loopValues,"", [])
    if rootLoopController == None:
        # Start the tree of loops
        rootLoopController = loopController
        currentLoopController = loopController
    else:
        if currentLoopController != None:
            # Add loop controller to tree as a child at parent's position
            currentLoopController.children.append(loopController)
            loopController.parent = currentLoopController
            currentLoopController = loopController
    
    # Set the loop variable to its initial value
    setOrUpdateVariable(loopVarname, str(loopValues[0]))

    return "<!--mdpre-for:" + forArguments + "-->"

def _printLoopControllers(loopController, level):
    print("    " * level + "--------------------")
    print("    " * level + "Type: " + loopController.loopType)
    print("    " * level + "Self: " + str(loopController))
    print("    " * level + "Parent: " + str(loopController.parent))
    print("    " * level + "End For: " + str(loopController.endForEncountered))
    print("    " * level + "Variable: " + loopController.loopVarName)
    for value in loopController.loopVarValues:
        print("    " * level + str(value))
    if loopController.loopType == "line":
        print("    " * level + "Line value: " +loopController.lineValue)
    elif loopController.loopType == "include":
        print("    " * level + "Filename: " +loopController.lineValue[9:])
    else:
        print("    " * level + "Children:")
        for child in loopController.children:
            print("    " * level + str(child))
    
    print("    " * level + "--------------------")
    print("\n")

    for child in loopController.children:
        _printLoopControllers(child, level + 1)
    print("\n\n")
    
def printLoopControllers():
    if rootLoopController is not None:
        _printLoopControllers(rootLoopController, 0)
    else:
        print("No loop controllers")

def interpretLoops(input_file2, loopController):
    global vars
    global rootLoopController, currentLoopController
    #printLoopControllers()

    # Loop over the loop variable's values
    for loopVarValue in loopController.loopVarValues:
        # Set the loop variable's current value
        setOrUpdateVariable(loopController.loopVarName, str(loopVarValue))

        # Add the children of this loop controller
        for child in loopController.children:
            if child.loopType == "line":
                line = substitute_variables(child.lineValue)
                input_file2.append(line)
            elif child.loopType == "include":
                line = substitute_variables(child.lineValue)
                include_name = substitute_variables(line[9:].rstrip().lstrip())

                # Check for recursive inclusion
                if include_name in includeStack:
                    sys.stderr.write(
                        f"File '{include_name}' recursively included. Stopping."
                    )
                    sys.exit()

                if wantVerbose is True:
                    input_file2.append(
                        "<!--mdpre-embed-start:"
                        + str(embedLevel)
                        + ":"
                        + include_name
                        + "-->"
                    )

                # Not recursive so add include file name to stack
                includeStack.append(include_name)

                try:
                    with open(include_name, "r") as f:
                        stopIt, include_lines = parse_include(f.readlines())
                except:
                    sys.stderr.write(f"File {include_name} missing. Terminating.\n")
                    exit()

                for line2 in include_lines:
                    input_file2.append(line2)

                if wantVerbose is True:
                    input_file2.append(
                        "<!--mdpre-embed-stop:"
                        + str(embedLevel)
                        + ":"
                        + include_name
                        + "-->"
                    )

                if stopIt is True:
                    return [stopIt, input_file2, vars]
            else:
                input_file2 = interpretLoops(input_file2, child)
    if loopController == rootLoopController:
        rootLoopController = None
        currentLoopController = None
    return input_file2

def handle_endfor(endforArguments, input_file2):
    global rootLoopController, currentLoopController
    global vars

    loopController = currentLoopController

    # Flag "end for" encountered
    currentLoopController.endForEncountered = True

    # Back to processing parent
    currentLoopController = currentLoopController.parent
    # printLoopControllers()
    
    # If all the way up to root loop controller has met "end for" interpret the loop set
    if rootLoopController.endForEncountered:
        input_file2 = interpretLoops(input_file2, rootLoopController)

    return ["<!--mdpre-endfor:" + endforArguments + "-->", input_file2]
    

def parse_include(input_file):
    global rootLoopController, currentLoopController
    global embedLevel
    global ifStack
    global vars
    
    embedLevel += 1
    input_file2 = []
    for line in input_file:
        if line.startswith("=stop") is True:
            return [True, input_file2]

        # Substitute any symbols into the included filename
        unsubstitutedLine = line
        line = substitute_variables(line)

        if ifStack[-1] is True:
            if (line.startswith("=include ")) & (rootLoopController == None):
                include_name = substitute_variables(line[9:].rstrip().lstrip())

                # Check for recursive inclusion
                if include_name in includeStack:
                    sys.stderr.write(
                        f"File '{include_name}' recursively included. Stopping."
                    )
                    sys.exit()

                # Not recursive so add include file name to stack
                includeStack.append(include_name)

                if wantVerbose is True:
                    input_file2.append(
                        "<!--mdpre-embed-start:"
                        + str(embedLevel + 1)
                        + ":"
                        + include_name
                        + "-->"
                    )

                try:
                    with open(include_name, "r") as f:
                        stopIt, include_lines = parse_include(f.readlines())
                except:
                    sys.stderr.write(f"File {include_name} missing. Terminating.\n")
                    exit()
                
                for line2 in include_lines:
                    input_file2.append(line2)
                if wantVerbose is True:
                    input_file2.append(
                        "<!--mdpre-embed-stop:"
                        + str(embedLevel + 1)
                        + ":"
                        + include_name
                        + "-->"
                    )
                if stopIt is True:
                    return [stopIt, input_file2, vars]

            elif line.startswith("=def ") is True:
                var = parse_def(line[5:-1].lstrip())
                var_index = find_variable(var[0])
                if var_index > -1:
                    # Replace the variable with its new definition
                    vars[var_index] = var
                else:
                    # Add the variable as it's new
                    vars.append(var)

                if wantVerbose is True:
                    varName, varType, varValue = var
                    
                    if var_index > -1:
                        redefined =" (redefined)"
                    else:
                        redefined =""
                        
                    input_file2.append(
                        "<!--mdpre-def:"
                        + varName
                        + ":"
                        + varType
                        + ":"
                        + varValue
                        + redefined
                        + "-->"
                    )

            elif line.startswith("=for ") is True:
                verbosity = handle_for(line[5:-1].lstrip())
                if wantVerbose is True:
                    input_file2.append(verbosity)

            elif line.startswith("=endfor") is True:
                verbosity, input_file2 = handle_endfor(line[8:-1].lstrip(), input_file2)
                if wantVerbose is True:
                    input_file2.append(verbosity)

            elif currentLoopController != None:
                # There is an active loop
                if line.startswith("=include "):
                    # Add current line to loop lines - as an include
                    lineController = LoopController("include", None, "", [], unsubstitutedLine, [])
                    currentLoopController.children.append(lineController)
                else:
                    # Add current line to loop lines - as an ordinary line
                    lineController = LoopController("line", None, "", [], unsubstitutedLine, [])
                    currentLoopController.children.append(lineController)

            elif line.startswith("=inc ") is True:
                parse_inc(line)

            elif line.startswith("=dec ") is True:
                parse_dec(line)

            elif line.startswith("=undef") is True:
                varName = line[7:-1].lstrip()
                verbosity = handle_undef(varName)

                if wantVerbose is True:
                    input_file2.append(verbosity)

            elif line.startswith("=ifdef ") is True:
                verbosity = handle_ifdef(line[7:].lstrip().rstrip())

                if wantVerbose is True:
                    input_file2.append(verbosity)

            elif line.startswith("=ifndef ") is True:
                verbosity = handle_ifndef(line[8:].lstrip().rstrip())

                if wantVerbose is True:
                    input_file2.append(verbosity)

            elif line.startswith("=endif") is False:
                input_file2.append(line)

        if line.startswith("=endif") is True:
            verbosity = handle_endif()

            if wantVerbose is True:
                input_file2.append(verbosity)

    # Pop this file from the stack
    includeStack.pop()

    embedLevel -= 1
    
    return [False, input_file2]


def replace_varname(varName):
    global vars
    for var in vars:
        if (var[0] == varName) & (var[1] == "T"):
            # A match so return the value
            return var[2]
    return "&" + varName + ";"


def substitute_variables(input_line):
    fragments = input_line.split("&")
    ampersands = len(fragments) - 1

    # Output line starts with fragment before the first ampersand
    output_line = fragments[0]

    # Go through the remaining fragments, extracting their data
    for f in range(ampersands):
        fragment = fragments[f + 1]
        semicolonPos = fragment.find(";")
        if semicolonPos == -1:
            # No terminating semicolon
            output_line += "&" + fragment
        else:
            # Terminating semicolon so extract variable name
            varName = fragment[:semicolonPos]
            tail = fragment[semicolonPos + 1 :]
            output_line += replace_varname(varName) + tail
    return output_line


def handle_linejoins(input_file):
    input_file2 = []
    output_line = ""
    pending = False
    line_count = len(input_file)

    for line_number, line in enumerate(input_file):
        output_line += line

        if line_number < line_count - 1:
            next_line = input_file[line_number + 1]
        else:
            next_line = ""

        if line.endswith("\\\n"):
            # Next line should be concatenated to this one
            output_line = output_line[:-2]
            pending = True

        elif next_line.startswith("<br/>"):
            output_line = output_line[:-1]
            pending = True
            
        else:
            # Next line shouldn't be concatenated to this one
            input_file2.append(output_line)
            output_line = ""
            pending = False
    if pending is True:
        input_file2.append(output_line)

    return input_file2


# Returns one item per potential TOC entry. First is level, Second is Text
def extractTableOfContents(p_input_file_lines, o_heading_symbol="#"):
    r_toc = []  # | init:
    r_minTocLevel = 999
    r_maxTocLevel = 0
    for line in p_input_file_lines:  # | for every line:

        if line.startswith(o_heading_symbol):  # |   if line starts with heading:
            title = line.lstrip(o_heading_symbol)  # |     remoe leading sign
            level = len(line) - len(title)  # |     calculate level
            title = title.lstrip().rstrip()  # |     flatten heading title
            r_toc.append((level, title))  # |     append to list
            # |
            if level > r_maxTocLevel:  # |     maybe update levels
                r_maxTocLevel = level
            elif level < r_minTocLevel:
                r_minTocLevel = level

    return [r_minTocLevel, r_maxTocLevel, r_toc]


def parseTocStatement(line, minTocLevel, maxTocLevel):
    TocParms = line[5:].lstrip().rstrip()
    if TocParms == "":
        # No parms so return defaults - full depth and "Contents"
        return [minTocLevel, maxTocLevel, "Contents"]
    else:
        # Have parms so parse them
        TocParmsArray = TocParms.split(" ")
        if TocParmsArray[0].isdigit() is True:
            # First parm is either max or min level spec
            level1 = int(TocParmsArray[0])
            if len(TocParmsArray) == 1:
                # First parm is only parm so it is the max level
                return [minTocLevel, level1, "Contents"]
            elif TocParmsArray[1].isdigit() is True:
                # Second parm is also a number so have both min and max
                level2 = int(TocParmsArray[1])
                if len(TocParmsArray) == 2:
                    # Have both numeric parms and no title
                    return [level1, level2, "Contents"]
                else:
                    # Have both numeric parms and a title
                    return [level1, level2, TocParms[4:]]
            elif TocParmsArray[1] == "*":
                # Top level limited by levels available
                level2 = maxTocLevel
                if len(TocParmsArray) == 2:
                    # Have both range parms and no title
                    return [level1, level2, "Contents"]
                else:
                    # Have both range parms and a title
                    return [level1, level2, TocParms[4:]]
        elif TocParms != "":
            # Have just the title
            return [minTocLevel, maxTocLevel, TocParms]


def makeLink(linkText):
    linkURL = (
        linkText.replace("-", "")
        .replace(" ", "-")
        .replace("?", "")
        .replace("`", "")
        .replace("=", "")
        .replace(",", "")
        .replace(":", "")
        .replace(".", "")
        .replace("\\", "")
        .replace('"', "")
        .replace("'", "")
        .lower()
        .rstrip("-")
    )
    return re.sub(r"-+", "-", linkURL)


def handle_verbose(line):
    noPrefixSuffix = line[10:-3]
    verboseArray = noPrefixSuffix.split(":")
    verboseType = verboseArray[0]

    if verboseType == "embed-start":
        embedLevel = int(verboseArray[1])
        sys.stderr.write(("---" * embedLevel) + "> Start of " + verboseArray[2] + " (@" + str(embedLevel) + ")\n")

    elif verboseType == "embed-stop":
        embedLevel = int(verboseArray[1])
        sys.stderr.write(("---" * embedLevel) + "> End of " + verboseArray[2] + " (@" + str(embedLevel) + ")\n")

    elif verboseType == "heading":
        sys.stderr.write(line[18:-3].replace("#", "..... ")[6:].lstrip() + "\n")

    elif verboseType == "toc":
        sys.stderr.write("Table Of Contents - spec '" + noPrefixSuffix[9:] + "'\n")

    elif verboseType == "csv":
        sys.stderr.write("CSV Start\n")

    elif verboseType == "endcsv":
        sys.stderr.write("CSV Stop\n")

    elif verboseType == "cal":
        sys.stderr.write("Calendar Start\n")

    elif verboseType == "endcal":
        sys.stderr.write("Calendar Stop\n")

    elif verboseType == "caldays":
        sys.stderr.write("Calendar Day Span\n")

    elif verboseType == "calkey":
        sys.stderr.write("Calendar Class Days\n")

    elif verboseType == "calnote":
        sys.stderr.write("Calendar Days Note\n")

    elif verboseType == "colalign":
        sys.stderr.write("Column Alignment - spec '" + noPrefixSuffix[19:] + "'\n")

    elif verboseType == "colwidth":
        sys.stderr.write("Column Width - spec '" + noPrefixSuffix[19:] + "'\n")
    
    elif verboseType == "rowspan":
        sys.stderr.write("Row span - spec '" + noPrefixSuffix[17:] + "'\n")

    elif verboseType == "def":
        if verboseArray[2] == "T":
            sys.stderr.write("Def " + verboseArray[1] + " = " + verboseArray[3] + "\n")
        else:
            sys.stderr.write("Def " + verboseArray[1] + "\n")

    elif verboseType == "undef":
        if verboseArray[1] == "found":
            sys.stderr.write("Undef successful " + verboseArray[2] + "\n")
        else:
            sys.stderr.write("Undef failed " + verboseArray[2] + "\n")

    elif verboseType == "endif":
        sys.stderr.write("Endif\n")

    elif verboseType == "ifdef":
        if verboseArray[1] == "true":
            sys.stderr.write("Ifdef true " + verboseArray[2] + "\n")
        else:
            sys.stderr.write("Ifdef nottrue " + verboseArray[2] + "\n")

    elif verboseType == "ifndef":
        if verboseArray[1] == "true":
            sys.stderr.write("Ifndef true " + verboseArray[2] + "\n")
        else:
            sys.stderr.write("Ifndef untrue " + verboseArray[2] + "\n")

    elif verboseType == "for":
        sys.stderr.write("For " + verboseArray[1] + "\n")

    elif verboseType == "endfor":
        sys.stderr.write("Endfor " + "\n")

    else:
        sys.stderr.write("Unknown " + line + "\n")

    return


# Handle a -d command line parameter
def handleCommandLineDefine(defString):
    equalsPos = defString.find("=")
    if equalsPos > -1:
        # Assign a value to the variable
        name = defString[:equalsPos]
        value = defString[equalsPos + 1 :]
        parmVars.append("=def " + name + " " + value)
    else:
        # No value to assign so just define it
        name = defString
        parmVars.append("=def " + name)


# Set up standard variables
def setupStandardVariables():
    now = datetime.datetime.now()
    runTime = now.strftime("%H&colon;%M").lstrip()
    runDate = now.strftime("%e %B&comma; %G").lstrip()

    input_file.insert(0, "=def date " + runDate + " ")
    input_file.insert(0, "=def time " + runTime + " ")
    input_file.insert(0, "=def userid " + os.getlogin() + " ")
    input_file.insert(0, "=def mdpre_level " + mdpre_level + " ")
    input_file.insert(0, "=def mdpre_date " + mdpre_date + " ")


def exit_script(p_output, p_code=0):
    if p_output != None:
        p_output.close()
        if p_code == 0:
            log_status("- Processing completed.")
        elif p_code == 1:
            # HELP output
            p_code = 0
        else:
            log_status(f"- Processing STOPPED with error {p_code}")
    else:
        p_code = 0

    sys.exit(p_code)


def create_output_file(p_file_path=""):
    """
    param: p_file_path:
        "" : use sys.stdout
    """
    r_output = None

    if p_file_path == "":
        r_output = Output(sys.stdout)
    else:
        if Path(p_file_path).exists():
            if wantVerbose:
                sys.stderr.write("WARNING: overwriting file!")
            pass
        to_file = open(p_file_path, "w")
        r_output = Output(to_file)

    return r_output


############################################################################################################
#  Main
#

BOLD = "\033[1m"
END = "\033[0m"

bannerUnderline = "=" * len(banner)

sys.stderr.write("\n" + banner + "\n" + bannerUnderline + "\n")

wantVerbose = False

wantTextBundle = False
wantTextPack = False
awaitingFilename = False
textBundleFilename = ""

rootLoopController = None
currentLoopController = None

parmVars = []

input_file = ""  # | TODO: add input file from command line for file output later

g_output = None  # | global variable for writing output
markdownFile = ""


def bold(p_text):
    return BOLD + p_text + END


arguments = len(sys.argv) - 1
for arg in range(1, arguments + 1):
    argument = sys.argv[arg]
    if argument == "-h":
        # Print help
        sys.stderr.write(
            "\n\nThe mdpre program is a Markdown preprocessor, making writing complex\nMarkdown documents easier.\n\n"
        )
        sys.stderr.write(
            "\n" + BOLD + "Usage:" + END + "\n\nmdpre < inputfile > outputfile\n\n"
        )
        sys.stderr.write("which processes from inputfile to outputfile.\n\n")

        sys.stderr.write(BOLD + "or:\n\n" + END)
        sys.stderr.write("mdpre -h\n\n")
        sys.stderr.write("which generates this help.\n\n")

        sys.stderr.write(BOLD + "or:\n\n" + END)
        sys.stderr.write("mdpre -v < inputfile > outputfile\n\n")
        sys.stderr.write('which writes additional information in "verbose mode".\n\n')

        sys.stderr.write(BOLD + "or:\n\n" + END)
        sys.stderr.write("mdpre -t < inputfile outputdir\n\n")
        sys.stderr.write(
            "which creates a textbundle directory from inputfile. -v can be used with -t.\n\n"
        )

        sys.stderr.write(BOLD + "or:\n\n" + END)
        sys.stderr.write("mdpre -z < inputfile outputfile\n\n")
        sys.stderr.write(
            "which creates a textpack (zipped textbundle) file from inputfile. -v can be used with -z.\n\n"
        )

        sys.stderr.write("\n\n" + BOLD + "Variables\n\n" + END)
        sys.stderr.write(BOLD + "---------\n\n" + END)
        sys.stderr.write(
            "\nYou can define a variable without a value with the -d flag:\n\n"
        )
        sys.stderr.write("mdpre -dfred < inputfile outputfile\n\n")
        sys.stderr.write(
            "You can define a variable with a value also with the -d flag:\n\n"
        )
        sys.stderr.write("mdpre '-djon=bon jovi' < inputfile outputfile\n\n")
        sys.stderr.write(
            "In this case quotes were used to allow a variable to take a value with spaces in.\n\n"
        )

        sys.stderr.write("\n\n" + BOLD + "Process As A CSV File\n\n" + END)
        sys.stderr.write(BOLD + "---------------------\n\n" + END)
        sys.stderr.write(
            "\nmdpre can treat the whole input file as a CSV file, creating a Markdown table from it, if you specify the -c flag:\n\n"
        )
        sys.stderr.write("mdpre -c < input.csv outputfile\n\n")

        exit_script(g_output, 1)

    elif argument == "-v":
        # Turn on "verbose mode"
        wantVerbose = True
    elif argument == "-t":
        wantTextBundle = True
        wantTextPack = False
        awaitingFilename = True
    elif argument == "-z":
        wantTextBundle = True
        wantTextPack = True
        awaitingFilename = True
    elif argument[:2] == "-d":
        handleCommandLineDefine(argument[2:])
    elif argument == "-c":
        g_wrapInCSV = True
    elif awaitingFilename is True:
        if textBundleFilename == "":
            textBundleFilename = argument
        else:
            textBundleFilename = textBundleFilename + " " + argument

# Preliminaries if creating a textbundle
if textBundleFilename != "":
    realpath = os.path.realpath(textBundleFilename)
    if wantTextPack is True:
        zippath = realpath
        realpath = realpath + "-temp"

    if wantVerbose is True:
        sys.stderr.write("Writing to Textbundle " + realpath + "\n\n")

    # Check if path exists as a directory
    if os.path.exists(realpath):
        if os.path.isfile(realpath) is True:
            # Regular file of that name exists so quitting
            sys.stderr.write(realpath + " is a regular file. Quitting.\n")
            sys.exit(0)
        else:
            # Directory already exists
            if wantVerbose is True:
                sys.stderr.write(
                    "Directory exists so not creating. It might already have files, some of which might be overwritten.\n\n"
                )
    else:
        # Directory doesn't exist so create it
        if wantVerbose is True:
            sys.stderr.write("Directory does not exist - so creating.\n\n")
        os.mkdir(realpath)

    # Check if assets path exists
    assets = realpath + "/assets"
    if os.path.exists(assets):
        if wantVerbose is True:
            sys.stderr.write("assets directory exists so not creating.\n\n")
    else:
        if wantVerbose is True:
            sys.stderr.write("Creating assets directory.\n\n")
        os.mkdir(assets)

    # Add info.json file
    infoJSON = open(realpath + "/info.json", "w")
    infoJSON.write("{\n")
    infoJSON.write(' "version": 2\n')
    # infoJSON.write(' "type": "net.daringfireball.markdown"\n')

    infoJSON.write("}\n")
    infoJSON.close()

    markdownFile = realpath + "/text.markdown"

    # Remove any prior instance of the markdown file
    if os.path.isfile(markdownFile) is True:
        os.remove(markdownFile)
        # | ## Textbundle:
    g_output = create_output_file(
        markdownFile
    )  # | Open the file where markdown goes for write

elif markdownFile != "":

    if os.path.isfile(markdownFile) is True:
        os.remove(markdownFile)
        # | ## File:
    g_output = create_output_file(
        markdownFile
    )  # | Open the file where markdown goes for write

else:  # | ## stdout
    g_output = (
        create_output_file()
    )  # | NOTE: Write to stdout as not making a textbundle

if input_file == "":
    input_filename = ""
    input_file = sys.stdin.readlines()
else:
    input_filename = input_file
    if os.path.isfile(input_file) is True:
        with open(input_file, "r") as file_handle:
            input_file = file_handle.readlines()


# If -c was specified on the command line wrap the file with =csv / =endcsv bracket
if g_wrapInCSV == True:
    input_file.insert(0, "=csv")
    input_file.append("=endcsv")

# Add any command line defines to the front
for parmVar in parmVars:
    input_file.insert(0, parmVar + " ")

# insert standard variables at the front
setupStandardVariables()


ifStack = [True]

# Prime list of referenced files
includeStack = ["top-level-file"]

embedLevel = 0

vars = []
# Pre-process - to pick up includes
stopIt, input_file = parse_include(input_file)

# Handle line joining
input_file = handle_linejoins(input_file)

# Generate Table Of Contents - even if it's not needed
minTocLevel, maxTocLevel, tableOfContents = extractTableOfContents(input_file)

# Do any other processing, such as CSV to table
csv_colaligns = []
csv_colwidths = []
in_csv = False
in_cal = False


for line in input_file:
    line = line.rstrip("\n")
    if line.startswith("<!--mdpre-"):
        # For verbose mode writing to stderr
        handle_verbose(line)

    elif line.startswith("#"):
        if wantVerbose is True:
            handle_verbose("<!--mdpre-heading:" + line + "-->")
        g_output.write(line)

    elif line.startswith("=toc"):
        if wantVerbose is True:
            handle_verbose("<!--mdpre-toc:" + line + "-->")
        minLevel, maxLevel, heading = parseTocStatement(line, minTocLevel, maxTocLevel)
        print(minLevel, maxLevel, heading, file=sys.stderr)
        g_output.printTableOfContents(
            minLevel, maxLevel, "### " + heading, tableOfContents
        )

    elif (line.startswith("=csv")) & (in_csv is False):
        in_csv = True
        in_rowspan = False
        CSV_lines = []
        if wantVerbose is True:
            handle_verbose("<!--mdpre-csv:" + line + "-->")

    elif (line.startswith("=endcsv")) & (in_csv is True):
        g_output.formatCSV(CSV_lines, csv_colaligns, csv_colwidths)
        in_csv = False
        if wantVerbose is True:
            handle_verbose("<!--mdpre-endcsv:" + line + "-->")

    elif line.startswith("=colalign "):
        csv_colaligns = parse_colalign(line[10:])
        if wantVerbose is True:
            handle_verbose("<!--mdpre-colalign:" + line + "-->")

    elif line.startswith("=colwidth "):
        csv_colwidths = parse_colwidth(line[10:])
        if wantVerbose is True:
            handle_verbose("<!--mdpre-colwidth:" + line + "-->")
    elif line.startswith("=rowspan"):
        in_rowspan = True
        if wantVerbose is True:
            handle_verbose("<!--mdpre-rowspan:" + line + "-->")
            rowspan = line.rstrip()

    elif in_csv is True:
        if in_rowspan:
            # Prepend line with rowspan
            CSV_lines.append(rowspan + "!!XYZZY!!" + line)
            in_rowspan = False
        else:
            # Don't prepend line with rowspan
            CSV_lines.append(line)

    elif (line.startswith("=cal ")) & (in_cal is False):
        in_cal = True
        cal_lines = []
        cal_spans = {}
        cal_keys = {}
        cal_notes = {}
        cal_date = parse_calstart(line[5:])
        if wantVerbose is True:
            handle_verbose("<!--mdpre-cal:" + line + "-->")

    elif (line.startswith("=endcal")) & (in_cal is True):
        g_output.formatCalendar(cal_lines, cal_date, cal_spans, cal_keys, cal_notes)
        in_cal = False
        if wantVerbose is True:
            handle_verbose("<!--mdpre-endcal:" + line + "-->")

    elif (line.startswith("=caldays")) & (in_cal is True):
        cal_spans = parse_caldays(line[9:], cal_spans)
        if wantVerbose is True:
            handle_verbose("<!--mdpre-caldays:" + line + "-->")

    elif (line.startswith("=calkey")) & (in_cal is True):
        cal_keys = parse_calkeys(line[8:], cal_keys)
        if wantVerbose is True:
            handle_verbose("<!--mdpre-calkey:" + line + "-->")

    elif (line.startswith("=calnote")) & (in_cal is True):
        cal_notes = parse_calnote(line[9:], cal_notes)
        if wantVerbose is True:
            handle_verbose("<!--mdpre-calnote:" + line + "-->")

    elif in_cal is True:
        cal_lines.append(line)

    elif (line.find("![") > -1) & (wantTextBundle is True):
        # Fix up graphics references and copy graphics to assets folder
        g_output.handle_adjustGraphics(line)

    else:
        g_output.write(line)

# NOTE: toFile.close() is handled in the new exits function
# NOTE: ! this message will self destruct during the pull request

# Issue a warning if 1 or more =endif statements are missing
if (l := len(ifStack)) > 1:
    if l == 2:
        sys.stderr.write(f'Warning 1 =endif statement missing.')
    else:
        sys.stderr.write(f'Warning {str(l - 1)} =endif statements missing.')

# Optionally zip into TextPack format
if wantTextPack is True:
    if wantVerbose is True:
        print("Archiving " + realpath + " to " + zippath + ".zip")
    shutil.make_archive(zippath, "zip", realpath)

    if wantVerbose is True:
        print("Renaming " + zippath + ".zip to " + zippath)
    os.rename(zippath + ".zip", zippath)

    if wantVerbose is True:
        print("Removing temporary files in directory " + realpath)
    shutil.rmtree(realpath)

exit_script(g_output, 0)