import dxr.plugins
import pygments
import pygments.lexers
from pygments.token import Token
import os, sys
import fnmatch

class Pygmentizer(object):
    """ Pygmentizer add syntax regions for file """
    def __init__(self, text, lexer):
        self.text   = text
        self.lexer  = lexer
    def refs(self):
        return []
    def regions(self):
        for index, token, text in self.lexer.get_tokens_unprocessed(self.text):
            cls = None
            if token is Token.Keyword:          cls = 'k'
            if token is Token.Name:             cls = None
            if token is Token.Literal:          cls = None
            if token is Token.String:           cls = 'str'
            if token is Token.Operator:         cls = None
            if token is Token.Punctuation:      cls = None
            if token is Token.Comment:          cls = 'c'
            if cls:   yield index, index + len(text), cls
    def annotations(self):
        return []
    def links(self):
        return []


def load(tree, conn):
    pass

def htmlify(path, text):
    # TODO Enable C++ highlighting using pygments, pending fix for infinite
    # looping that we don't like, see:
    # https://bitbucket.org/birkenfeld/pygments-main/issue/795/
    if any((path.endswith(e) for e in ('.c', '.cc', '.cpp', '.cxx', '.h', '.hpp'))):
        return None
    # Options and filename
    options   = {'encoding': 'utf-8'}
    filename  = os.path.basename(path)
    try:
        lexer = pygments.lexers.get_lexer_for_filename(filename, **options)
    except pygments.util.ClassNotFound:
        # Small hack for js highlighting of jsm files
        if fnmatch.fnmatchcase(filename, "*.jsm"):
            lexer = pygments.lexers.JavascriptLexer(**options)
        else:
            return None
    return Pygmentizer(text, lexer)

__all__ = dxr.plugins.htmlifier_exports()

@@ -115,16 +115,916 @@
 nmatch%0A%0A
+keyword_tokens = %5BToken.Keyword,%0A                  Token.Keyword.Constant,%0A                  Token.Keyword.Declaration,%0A                  Token.Keyword.Namespace,%0A                  Token.Keyword.Pseudo,%0A                  Token.Keyword.Reserved,%0A                  Token.Keyword.Type%5D%0Astring_tokens =  %5BToken.String,%0A                  Token.String.Backtick,%0A                  Token.String.Char,%0A                  Token.String.Doc,%0A                  Token.String.Double,%0A                  Token.String.Escape,%0A                  Token.String.Heredoc,%0A                  Token.String.Interpol,%0A                  Token.String.Other,%0A                  Token.String.Regex,%0A                  Token.String.Single,%0A                  Token.String.Symbol%5D%0Acomment_tokens = %5BToken.Comment,%0A                  Token.Comment.Multiline,%0A                  Token.Comment.Single,%0A                  Token.Comment.Special%5D%0A%0A
 class Py
@@ -1376,91 +1376,33 @@
 en i
-s Token.Keyword:          cls = 'k'%0A            if token is Token.Name:    
+n keyword_tokens:
         
  cls
@@ -1393,35 +1393,33 @@
 ens:        
- 
 cls = 
-None
+'k'
 %0A           
@@ -1433,206 +1433,86 @@
 en i
-s Token.Literal:          cls = None%0A            if token is Token.String:           cls = 'str'%0A            if token is Token.Operator:         cls = None%0A            if token is Token.Punctuation:
+n string_tokens:         cls = 'str'%0A            if token in comment_tokens:  
     
@@ -1511,36 +1511,35 @@
 s:        cls = 
-None
+'c'
 %0A            if 
@@ -1564,26 +1564,25 @@
 ment
-:         
+.Preproc:
  cls = '
 c'%0A 
@@ -1569,33 +1569,33 @@
 Preproc: cls = '
-c
+p
 '%0A            if

Fix js highlighting