#!/usr/bin/env python
import os
from tsa.lib import tabular, html

xlsx_filepath = '%s/ohio/sb5-b.xlsx' % os.getenv('CORPORA', '.')
label_keys = ['For', 'Against', 'Neutral', 'Broken Link', 'Not Applicable']


def read(limit=None):
    '''Yields dicts with at least 'Labels' and 'Tweet' fields.'''
    for row in tabular.read_xlsx(xlsx_filepath, limit=limit):
        for label_key in label_keys:
            row[label_key] = bool(row[label_key])

        row['Labels'] = [label_key for label_key in label_keys if row[label_key]]
        row['Label'] = (row['Labels'] + ['NA'])[0]
        row['Tweet'] = html.unescape(row['Tweet'])

        yield row

@@ -60,16 +60,70 @@
 , html%0A%0A
+import logging%0Alogger = logging.getLogger(__name__)%0A%0A%0A
 xlsx_fil
@@ -698,8 +698,688 @@
 eld row%0A
+%0A%0Adef read_cached(limit=None):%0A    import cPickle as pickle%0A    pickle_filepath = '%25s.pickled-%25s' %25 (xlsx_filepath, limit or 'all')%0A%0A    if os.path.exists(pickle_filepath):%0A        logger.info('Loading pickled sb5b tweets file from %25s', pickle_filepath)%0A        pickle_file = open(pickle_filepath, 'rb')%0A        for item in pickle.load(pickle_file):%0A            yield item%0A    else:%0A        logger.info('Reading fresh sb5b tweets')%0A        items = list(read(limit=limit))%0A%0A        logger.info('Pickling sb5b tweets to %25s', pickle_filepath)%0A        pickle_file = open(pickle_filepath, 'wb')%0A        pickle.dump(items, pickle_file)%0A        for item in items:%0A            yield item%0A

Add specific iterable-like pickling handler for sb5b tweet data