try:
    from cStringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

from chardet import detect
from lxml.html import parse, HTMLParser
from libextract.coretools import highest_scoring


NODES_WITH_TEXT = '//*[not(self::script or self::style)]/text()/..'
TEXT_IN_NODE = './/text()[normalize-space()]'

FILTER_TEXT = './/*[not(self::script or self::style or \
        self::figure or self::span or self::time)]/\
        text()[normalize-space()]'


def node_text_length(node):
    return len(''.join(node.xpath(TEXT_IN_NODE)))


def get_etree(document):
    return parse(BytesIO(document),
                 HTMLParser(encoding=detect(document)['encoding'],
                            remove_blank_text=True))


def get_pairs(etree):
    for node in etree.xpath(NODES_WITH_TEXT):
        yield node.getparent(), node_text_length(node)


def get_final_text(node):
    return ' '.join(node.xpath(FILTER_TEXT))


STRATEGY = (get_etree, get_pairs, highest_scoring, get_final_text)

@@ -508,80 +508,294 @@
     
-return len(''.join(node.xpath(TEXT_IN_NODE)))%0A%0A%0Adef get_etree(document):
+%22%22%22%0A    Returns the length of the text contained within%0A    a given *node*.%0A    %22%22%22%0A    return len(''.join(node.xpath(TEXT_IN_NODE)))%0A%0A%0Adef get_etree(document):%0A    %22%22%22%0A    Get an ElementTree instance from a given XML/HTML%0A    *document*. The encoding is automatically detected.%0A    %22%22%22
 %0A   
@@ -967,24 +967,123 @@
 irs(etree):%0A
+    %22%22%22%0A    Given an *etree*, returns an iterable of parent%0A    to node text length pairs.%0A    %22%22%22%0A
     for node
@@ -1195,24 +1195,135 @@
 text(node):%0A
+    %22%22%22%0A    Gets the text contained within the children node%0A    of a given *node*, joined by a space.%0A    %22%22%22%0A
     return '

document html module