Python Markdown extensions: gentoc_remove, auc_headers, autoxref

#!/usr/bin/python
"""
genTOC Table of Contents remover Extension for Python-Markdown
==============================================================

Removes a text-mode Table of Contents entry from the input document and
replaces it with "[TOC]" for the 'toc' extension.

Expects to find "Table of Contents" within the first 100 lines of the input
file, with either an ATX '#'/'##' header or an Setext/AUC underline

License: [BSD](http://www.opensource.org/licenses/bsd-license.php)
"""

from __future__ import absolute_import
from __future__ import unicode_literals
from . import Extension
from ..preprocessors import Preprocessor
import re

class GenTOCRemoveExtension(Extension):
    """ GenTOCRemove Extension. """

    def extendMarkdown(self, md, md_globals):
        """ Add pieces to Markdown. """
        md.registerExtension(self)

        # Insert the preprocessor after whitespace normalisation
        md.preprocessors.add(
            "gentoc_remove", GenTOCRemovePreprocessor(self), ">normalize_whitespace"
        )

class GenTOCRemovePreprocessor(Preprocessor):
    """ Remove a Table of Contents section generated by genTOC """

    def run(self, lines):
        """
        Remove a Table of Contents section generated by genTOC

        Keywords:
        * lines: A list of lines of text

        Return: A list of lines of text with the Table of Contents replaces with '[TOC]'
        """

        RE_ToC = re.compile('Table of Contents', re.IGNORECASE)
        RE_ToC_line = re.compile(r'\s*\d+\s+.+$')
        RE_blank_line = re.compile(r'\s*$')
        i = -1
        toc_start = -1  # First available line after '[TOC]' marker
        toc_end = 0     # Last line of the ToC, befire blank lines
        skip_AUC_line = False   # AUC = "Arbitrary Underline Character"
        for l in lines:
            i = i + 1
            if RE_ToC.match(l):
                # We found a candidate ToC line
                if re.match(r'\s*#', l):
                    toc_start = i + 1
                    continue
                elif re.match(r'\s*(([^\s])(\2+))\s*$', lines[i+1]):
                    toc_start = i + 2
                    skip_AUC_line = True
                    continue

            # Stop processing if no "Table of Contents" after 100 lines
            if toc_start == -1 and i > 100:
                break

            # If we're in a Table of Contents section, check for a valid ToC line
            # (e.g. "  55  This is a heading"). If it's non-blank and not a ToC
            # line, we've found the end of the Table of Contents
            if toc_start > -1 and not RE_blank_line.match(l):
                if RE_ToC_line.match(l):
                    toc_end = i
                elif skip_AUC_line:
                    skip_AUC_line = False
                else:
                    break

        if toc_start > -1:
            lines.insert(toc_start, "[TOC]")
            del lines[toc_start+1 : toc_end+2]

        return lines

def makeExtension(*args, **kwargs):
    """ Return an instance of the GenTOCRemoveExtension """
    return GenTOCRemoveExtension(*args, **kwargs)

#!/usr/bin/python
class AutoxrefTreeProcessor(Treeprocessor):
    """ Automatically create links to headers referenced in a block """

    # Detect an inline reference
    RE = re.compile(r'(?P<lead>.*?)\[(?P<ref>[^]]+)\](?P<tail>..)?')

    def __init__(self, md):
        super(AutoxrefTreeProcessor, self).__init__(md.parser)
        self.md = md


    def _process_match(self, match, headers):
        """ Process an instance of [text that might be a header link] """
        lead = match.group('lead')
        ref = slugify(match.group('ref'), '-')
        tail = match.group('tail')
        text = match.group('lead') + '[' + match.group('ref') + ']'
        if (not tail or (tail[0:1] != '(' or tail == '[]')) \
                and ref in headers:
            text = "%s(#%s)" % (text, ref)
        if tail and tail != '[]':
            text = text + tail
        pos = tail and match.end(3) or match.end(2)+1
        return text, pos

    def run(self, doc):
        """
        Seeks out inline references in the form [reference] or [reference][]
        and attempts to resolve them to headings in the document.

        Parameters:

        * lines: A list of lines of text

        """
        # Get a list of headings, slugified
        headers = {}
        for elem in doc.iter():
            if elem.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
                headers[slugify(elem.text, '-')] = None

        # Look for [link references] in various elements.
        for elem in doc.iter():
            if not elem.tag in ('p', 'li', 'td', 'dt', 'dd', 'article', 'caption'):
                continue
            if not elem.text:
                continue

            # Loop through all occurrences of '[link reference]' in the element
            new_text = ''
            p = 0          # Last known position within elem.text
            for m in re.finditer(self.RE, elem.text):
                # (Account for text skipped from the last match)
                if p < m.start(1):
                    new_text = new_text + elem.text[p:m.start(1)]
                # Process this match
                text, p = self._process_match(m, headers)
                new_text = new_text + text
            # Append any trailing text
            if p < len(elem.text):
                new_text = "%s%s" % (new_text, elem.text[p:])
            elem.text = new_text