<jupyter_start><jupyter_text>Word map databases<jupyter_code>#
# We need to be able to rapidly look at tweets 
# down to the 1, 2, and 3 gram level. Thus the 
# tools in this section notebook create and 
# maintain the necessary databases.

%cd twitteranalysis
%run -i environment.py

# TEST = False

#General tools
import sys
import locale
import json
import time
from random import shuffle
import itertools #For set operations
# from urllib2 import URLError
import datetime
from datetime import time
from datetime import date
import string
import shelve

#Data storage
import redis
import couchdb
from couchdb.design import ViewDefinition

#Display
from CustomDisplayTools import Table
from IPython.display import HTML
%config InlineBackend.figure_format = 'svg'

# Pandas
from pandas import DataFrame, Series
import pandas as pd
pd.options.display.max_rows = 999 #let pandas dataframe listings go long

#Network x
import networkx as nx
from networkx.algorithms import bipartite as bi


#Plotting 
%matplotlib inline
from matplotlib import pyplot as plt
import seaborn as sns
sns.set(style="whitegrid")

# Timing etc tools (e.g., @timefn)
from OptimizationTools import *

#SqlAlchemy
import sqlalchemy
from sqlalchemy import create_engine

#Twitter specific
%cd twittermining
import twitter
# import twitter_text # easy_install twitter-text-py
# from TwitterLogin import login
# # from TwitterUtilities import makeTwitterRequest
# # t = login()
# import TwitterSQLService
# #%%bash couchdb
# import TwitterServiceClasses as TSC
# import TwitterSearcher as TS
# import TwitterDataProcessors as TDP
# #Data manipulation and cleaning
# from TwitterDataProcessors import Tags
# from TwitterDataProcessors import Extractors
# from TwitterDataProcessors import TagsFromSearch
%cd twitteranalysis
import sphinxapi

# Create your connection.

# engine = create_engine('mysql://root:''@localhost:3306/twitter_wordsTEST')
#engine = create_engine('mysql://testuser:testpass@localhost/twitter_data')<jupyter_output><empty_output><jupyter_text>To do- Build in text encoding normalization to the tweet object as a getter so that accessing tweet.text will yield expected encoding.<jupyter_code>sentences = ['a', 'b']
[(idx, txt) for idx, txt in enumerate(sentences)]
# for idx, txt in enumerate(sentences):
#     print(idx, txt)<jupyter_output><empty_output><jupyter_text>Process into single word database<jupyter_code># Create tables in database
from DataTools.WordORM import create_db_tables
create_db_tables()

#%cd /Users/adam/Dropbox/PainNarrativesLab/TwitterDataAnalysis
#%bookmark twitteranalysis
%cd twitteranalysis
%run -i environment.py
%run -i ConstantsAndUtilities.py
%run -i TestingTools/DataAndFunctionsForTesting.py
%run -i "DataTools/DataStructures.py"
%run -i "DataTools/DataConnections.py"
%run -i "DataTools/WordORM.py"
%run -i "DataTools/DataRepositories.py" 

# Initialize the tools for filtering and modifying the individual tweet words
from TextProcessors.Filters import *
from TextProcessors.Modifiers import *
%cd texttools
%run -i TextProcessors/Processors
%cd twitteranalysis
%run -i ConstantsAndUtilities.py
# word = session.query(Word).filter( Word.word == 'taco1' ).first( )
# if isinstance(word, Word):
#     print ('j')
# mysql cursor
# # get a connection
# import sqlalchemy.pool as pool
#  'mysql://root:''@localhost:3306/twitter_wordsTEST' 
# def getconn():
#     c = psycopg2.connect(username='ed', host='127.0.0.1', dbname='test')
#     return c

# mypool = pool.QueuePool(getconn, max_overflow=10, pool_size=5)
# conn = mypool.connect()

# # use it
# cursor = conn.cursor()
# cursor.execute("select foo")
# for _ in range(0, 10):
#     print(cursor.next_tweet().tweetID)
# Session.rollback()<jupyter_output><empty_output><jupyter_text>cursors<jupyter_code>%cd twitteranalysis

%run -i DataTools/Cursors
u = UserCursor(language='fr')
t = TweetCursor( )

k = u.next()
k.lang, k.userID

k = t.next()
k.tweetID
k.tweetText<jupyter_output><empty_output><jupyter_text>Load and process actual tweets<jupyter_code>%cd twitteranalysis

# %run -i profiling/optimizing_string_processor.py
# from threading import Thread
# # session.rollback()
# # session.no_autoflush()

# RUNS = 5
# MAX_THREADS = 71
# START_THREADS= 69
# DATASTRUCTURES = ('tuple', 'dawg')
# NOTE = 'programmatic run 69-71 threads'
# MODULE = 'stringProcessingWorker'
# NUM_TWEETS = 1181 #don't rely on other stuff

# def add_run(frame, module, threads, tweets, time, datastructure, note):
#     """Adds a record to benchmarks"""
#     d = {
#         'id': frame.id.max() + 1,
#         'timestamp': datetime.datetime.now(),
#         'module': module,
#         'numThreads': threads,
#         'numTweets': tweets,
#         'totalTime': time,
#         'tweetTime': time / tweets,
#         'dataStructure': datastructure,
#         'note': note
#     }
#     d = pd.DataFrame([d])
#     frame = pd.concat([frame, d])
#     return frame

# benchmarks = pd.read_csv("%s/tests/benchmarks/StringProcessingWorker.csv" % TEXT_TOOLS_PATH)

# for THREADS in range(START_THREADS, MAX_THREADS):
#     for DATASTRUCTURE in DATASTRUCTURES:
#         for _ in range(RUNS):
#             print("threads: %s   datastructure: %s   run: %s" % (THREADS, DATASTRUCTURE, _))

#             # Create queue and listeners for processed tokens
#             Queue = SaveQueueHandler()
#             Queue.register_listener(SaveListener())

#             # Load cursor for tweet ids
#             cursor = DataTools.Cursors.TweetCursor()

#             #Add regular or dawg filter
#             if DATASTRUCTURE is 'dawg':
#                 print('dawg')
#                 word_processor.add_to_filters( ignoreDawgFilter )
#             else:
#                 word_processor.add_to_filters( ignoreListFilter )
                
#             #Initialize everything we need
#             StringProcessingWorker.initialize(cursor, Queue, word_processor)
#             threads = []
#             time = Timer()

#             #Go!
#             time.start() 

#             if THREADS is 0:
#                 worker = StringProcessingWorker()

# #                 if DATASTRUCTURE is 'dawg':
# #                     worker.do_it_dawg()
# #                 else:
#                 worker.do_it()

#             else:
#                 # if this run is using threads, we go this way
#                 for _ in range(THREADS):
#                     worker = StringProcessingWorker()
                    
# #                     if DATASTRUCTURE is 'dawg':
# #                         thread = Thread(target=worker.do_it_dawg)
# #                     else:
#                     thread = Thread(target=worker.do_it)
#                     threads.append(thread)
#                     thread.start()

#                 for t in threads:
#                     t.join()

#             #....and stop!
#             time.stop() 

#             #Good job! Now, write it down and get back out there, tiger!
#             benchmarks = add_run(frame=benchmarks, 
#                                  module=MODULE, 
#                                  threads=THREADS, 
#                                  tweets=NUM_TWEETS, 
#                                  time=time.elapsed_seconds, 
#                                  datastructure=DATASTRUCTURE, 
#                                  note=NOTE)
#             benchmarks.set_index(['id']).to_csv("%s/tests/benchmarks/StringProcessingWorker.csv" % TEXT_TOOLS_PATH)

THREADS
datetime.datetime.now()
#benchmarks.timestamp = pd.to_datetime(benchmarks.timestamp)
benchmarks
worker.totalProcessed<jupyter_output><empty_output><jupyter_text>analysis<jupyter_code>#load from file
data = pd.read_csv("%s/tests/benchmarks/StringProcessingWorker.csv" % TEXT_TOOLS_PATH)
data.set_index(['id'], inplace=True)
#data.tweetTime = data.totalTime / data.numTweets
# data.drop(['date', 'module', 'note', 'time', 'numTweets', 'timestamp'], axis=1, inplace=True)
data<jupyter_output><empty_output><jupyter_text>key will be adding a dawg to hold the word ids so don't have to hit the db with every word'<jupyter_code>len(data)

def estimate_times(totalProcessed, elapsed_seconds):
    tweets_to_process = [690000, 4000000]
    print("Processed %s tweets in %s seconds" % (totalProcessed, elapsed_seconds))
    time_per_tweet = elapsed_seconds/totalProcessed
    for num in tweets_to_process:
        estimated_sec = time_per_tweet * num
        print("estimated time to process %s tweets: %s minutes \n" % (num, int( estimated_sec/60 ) ))
        
# import ProcessingTools.Workers.StringProcessingWorker

from threading import Thread

time = Timer()
time.start()

# Create queue and listeners for processed tokens
Queue = SaveQueueHandler()
Queue.register_listener(SaveListener())

# Load cursor for tweet ids
cursor = DataTools.Cursors.TweetCursor()

StringProcessingWorker.initialize(cursor, Queue, word_processor)
threads = []

# for _ in range(1):
worker = StringProcessingWorker()
worker.do_it()

# thread = Thread(target=worker.do_it)
# threads.append(thread)
# thread.start()

# for t in threads:
#     t.join()

time.stop()


estimate_times(worker.totalProcessed, time.elapsed_seconds)
benchmarks['hours'] = benchmarks.est_4m_min.apply(lambda x: x/60)
benchmarks.groupby('dataStructure').hours.plot(legend=True, title='Est hours for 4 million tweets')<jupyter_output><empty_output><jupyter_text>Dawg<jupyter_code>import dawg
from DataTools import DataStructures
words = [('text %s' % i, tuple(make_result(i, i, 'text %s' % i, i))) for i in range(0, 5)]
words
# base_dawg = dawg.DAWG(words)
# completion_dawg = dawg.CompletionDAWG(words)

# record_dawg = dawg.RecordDAWG(words)
format = DataStructures.Result
keys = [u'text%s' % i for i in range(0, 5)]
values = [(i, i, keys[i], i) for i in range(0, 5)]
data = zip(keys, values)
print([r for r in data])
record_dawg = dawg.RecordDAWG(format, data)
data = [ (u'foo', 1), (u'bar', 2) ]
int_dawg = dawg.IntDAWG(data)
int_dawg.
[print (k) for k in int_dawg.keys()]
for r in record_dawg.items():
    print(r)
'text0' in record_dawg
[print (k) for k in record_dawg.keys()]
u'foo' in base_dawg
completion_dawg.keys(u'foo')
u'baz' in completion_dawg
to test whether some key begins with a given prefix:

>>> completion_dawg.has_keys_with_prefix(u'foo')
>>> True

and to find all prefixes of a given key:

>>> base_dawg.prefixes(u'foobarz')
[u'foo', u'foobar']

Iterator versions are also available:

>>> for key in completion_dawg.iterkeys(u'foo'):
...     print(key)
foo
foobar
>>> for prefix in base_dawg.iterprefixes(u'foobarz'):
...     print(prefix)
foo
foobar

It is possible to find all keys similar to a given key (using a one-way char translation table):

>>> replaces = dawg.DAWG.compile_replaces({u'o': u'ö'})
>>> base_dawg.similar_keys(u'foo', replaces)
[u'foo', u'foö']
>>> base_dawg.similar_keys(u'foö', replaces)
[u'foö']
>>> base_dawg.similar_keys(u'bor', replaces)
[u'bör']


>>> keys = [u'foo', u'bar', u'foobar', u'foo']
>>> values = [(1, 2, 3), (2, 1, 0), (3, 3, 3), (2, 1, 5)]
>>> data = zip(keys, values)
>>> record_dawg = RecordDAWG(format, data)

estimate_times(totalProcessed, time.elapsed_seconds)
2090/60
#subprocesses setup
import subprocess

proc = subprocess.Popen(['echo', 'jjjj'], stdout=subprocess.PIPE)

out, err = proc.communicate()
print(out.decode('utf-8'))
# session = Session()

# w = session.query(Word).limit(1).all()
# ww = w[0].word
# type(ww.encode('utf-8'))
j = "b'jip"
j[2:]<jupyter_output><empty_output><jupyter_text>dawg 1 threadProcessed 1181 tweets in 33.9468 secondsestimated time to process 690000 tweets: 331 minutes estimated time to process 4000000 tweets: 1916 minutes  dawg no threadProcessed 1181 tweets in 34.3619 secondsestimated time to process 690000 tweets: 334 minutes estimated time to process 4000000 tweets: 1939 minutes reg 1 threadProcessed 1181 tweets in 34.9496 secondsestimated time to process 690000 tweets: 340 minutes estimated time to process 4000000 tweets: 1973 minutes  run 2Processed 1181 tweets in 33.6475 secondsestimated time to process 690000 tweets: 327 minutes estimated time to process 4000000 tweets: 1899 minutes  reg no threadProcessed 1181 tweets in 34.4375 secondsestimated time to process 690000 tweets: 335 minutes estimated time to process 4000000 tweets: 1943 minutes<jupyter_code># %run -i ProcessingTools/ProcessingControllers.py
# %run -i ProcessingTools/Listeners.py
# %run -i ProcessingTools/Workers.py


# import DataTools.Cursors
# #Mission Control

# # Create queue and listeners for processed tokens
# Queue = SaveQueueHandler()
# Queue.register_listener(SaveListener())

# # Create controller which will oversee the process
# # control = TweetProcessingController(Queue)

# # Initialize the tools for filtering and modifying the individual tweet words
# word_processor = TextProcessors.Processors.SingleWordProcessor( )
# ignore = ConstantsAndUtilities.Ignore( )
# ignore._construct()
# merge = ConstantsAndUtilities.Merge( )

# #reg filter
# ignoreListFilter = TextProcessors.Filters.IgnoreListFilter( )
# ignoreListFilter.add_to_ignorelist( ignore.get_list( ) )
# ignoreListFilter.add_to_ignorelist( nltk.corpus.stopwords.words( 'english' ) ) # or do we keep them?

# #dawg filter
# ignoreDawgFilter = TextProcessors.Filters.IgnoreDawgFilter()
# ignoreDawgFilter.add_to_ignorelist( ignore.get_list( ) )
# ignoreDawgFilter.add_to_ignorelist( nltk.corpus.stopwords.words( 'english' ) ) # or do we keep them?

# word_processor.add_to_filters( TextProcessors.Filters.UsernameFilter( ) )
# word_processor.add_to_filters( TextProcessors.Filters.PunctuationFilter( ) )
# word_processor.add_to_filters( TextProcessors.Filters.URLFilter( ) )
# word_processor.add_to_filters( TextProcessors.Filters.NumeralFilter( ) )
# word_processor.add_to_modifiers( TextProcessors.Modifiers.WierdBPrefixConverter() )
# # processor.add_to_modifiers( TextProcessors.Modifiers.UnicodeConverter() )
# word_processor.add_to_modifiers( TextProcessors.Modifiers.CaseConverter( ) )<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup