import sqlite3
import json
import sys

conn = sqlite3.connect('jose.db')

def do_stmt(stmt, params=None):
    global conn
    cur = conn.cursor()
    cur.execute(stmt, params)
    conn.commit()
    return cur

def server_messages(serverid, limit=None):
    cur = do_stmt('SELECT message FROM markovdb WHERE serverid=?', (serverid,))
    rows = [row[0] for row in cur.fetchall()]

    if limit is not None:
        pos = len(rows) - limit
        rows = rows[pos:]

    return rows

def get_all_serverids():
    server_ids = do_stmt('SELECT serverid FROM markovdb')
    return set(server_ids)

def main(args):
    resfile = args[1]
    serverids = get_all_serverids()
    amnt = do_stmt("SELECT COUNT(*) FROM markovdb")

    print("%d servers")

    msgs = 0
    with open(resfile, 'w') as sqlfile:
        for serverid in serverids:
            messages = server_messages(serverid, 10000)

            for message in messages:
                sqlfile.write("INSERT INTO markovdb (serverid, message) VALUES (%s, %s);\n" % \
                    (serverid, json.dumps(message)))
                msgs += 1

    print("Done! from %d to %d messages." % (amnt, msgs))

if __name__ == '__main__':
    sys.exit(main(sys.argv))

@@ -139,16 +139,79 @@
 ursor()%0A
+    if params is None:%0A        cur.execute(stmt)%0A    else:%0A    
     cur.
@@ -567,26 +567,19 @@
 ():%0A    
-server_ids
+cur
  = do_st
@@ -614,16 +614,68 @@
 kovdb')%0A
+    server_ids = %5Brow%5B0%5D for row in cur.fetchall()%5D%0A
     retu
@@ -736,44 +736,8 @@
 %5B1%5D%0A
-    serverids = get_all_serverids()%0A
     
@@ -787,32 +787,93 @@
 db%22)
-%0A%0A    print(%22%25d servers%22
+.fetchall()%5B0%5D%0A    serverids = get_all_serverids()%0A%0A    print(%22%25d messages%22 %25 amnt%5B0%5D
 )%0A%0A 
@@ -1278,16 +1278,19 @@
  %25 (amnt
+%5B0%5D
 , msgs))

fix some issues with jspeak-10k-limiting