

import json
import sqlite3
import collections

database = 'data/data.db'
Record = collections.namedtuple('Record', ['business_id', 'reviews', 'categories'])

class DataLoader():
    def __init__(self):
        self._conn = sqlite3.connect(database)
        self._datasize = self._conn.execute('SELECT count(business_id) FROM business;').fetchone()[0]
        print "{0} business loaded".format(self._datasize)

    def alldata(self):
        sql = 'SELECT business_id, cat FROM business;'
        l = []
        for business_id, cat in self._conn.execute(sql):
            sql = 'SELECT content FROM review WHERE business_id=?'
            for content in self._conn.execute(sql, (business_id,)):
                l.append(content[0])

            yield Record(business_id, l, json.loads(cat)['list'])


if __name__ == '__main__':
    d = DataLoader()
    from pprint import pprint
    c = 0
    for i in d.alldata():
        pprint((i.business_id, i.categories))
        c = c+1

        if c > 20:
            break

@@ -796,16 +796,414 @@
 ist'%5D)%0A%0A
+    def first_100(self):%0A        r = %5B%5D%0A        c = 0%0A        for business in self.alldata():%0A            c += 1%0A            if u%22Restaurants%22 in business.categories:%0A                label = %22Restaurants%22%0A            else:%0A                label = %22Not Restaurants%22%0A%0A            r.append(Record(business.business_id, %22%22.join(business.reviews), label))%0A            if c%3E100:%0A                return r%0A
 %0Aif __na

add first_100