<jupyter_start><jupyter_text>1. Load data<jupyter_code># Let's read in our document-term matrix
speech_df = pd.read_pickle('../dump/speech_clean_lemma')
data = pd.read_pickle('../dump/data_dtm_lemma.pkl')
tdm = data.transpose()
tdm.shape
tdm.head(10)
speech_df.head()<jupyter_output><empty_output><jupyter_text>2. Prepare for topic modelingCreate gensim corpus and dictionarry<jupyter_code>import pickle

with open("../dump/common_words.txt", "rb") as f:   # Unpickling
    common_words = pickle.load(f)
# Bag of words with CountVectorizer
# add_stop_words selected from after lemmatization
# will also remove common_words (most commonly used words in all speeches)
# will also remove boring words (words that do not add much insight to topic modeling)
add_stop_words = ['like','youre','ive','im','really','id','ve','just','dont','thi','wa',
                  'say','know','make','people']

boring_words = ['say','like','just','dont','don','im',
                  'ive','youll','youve','things','thing','youre','right','really','lot',
                  'make','know','people','way','day','class']


add_stop_words = add_stop_words + common_words + boring_words

stop_words = text.ENGLISH_STOP_WORDS.union(add_stop_words)

cv = CountVectorizer(stop_words=stop_words)
data_cv = cv.fit_transform(speech_df.transcript)
# If add_stop_words is modified, update tdm
data_dtm = cv_dtm(speech_df,'transcript',add_stop_words)
tdm = data_dtm.transpose()
# We're going to put the term-document matrix into a new gensim format
# From df --> sparse matrix --> gensim corpus
sparse_counts = scipy.sparse.csr_matrix(tdm)
corpus = matutils.Sparse2Corpus(sparse_counts)
# Gensim also requires dictionary of the all terms and their respective location in the term-document matrix
# {dictionsry of location: word}
id2word = dict((v, k) for k, v in cv.vocabulary_.items())
len(id2word)<jupyter_output><empty_output><jupyter_text>3. Topic modeling - LDALatent Dirichlet Allocation (**LDA**): specifically designed for text data with a probabilistic approach.  Use two probability values: **P(word | topics)** and **P(topics | documents)**  The two values arre calculated based on an initial random assignment, after which they are repeated for each word in each document to decide their topic assignment. In the iterative procedure, the probabilities are calculated until converged.Input:* document-term matrix* number of topics* number of iterstions.<jupyter_code>def get_lda_topics(model, num_topics):
    """Print lda topics with pd.DataFrame"""
    
    word_dict = {}
    for i in range(num_topics):
        words = model.show_topic(i, topn = 10)
        word_dict['Topic #' + '{:02d}'.format(i+1)] = [i[0] for i in words]
        
    return pd.DataFrame(word_dict).transpose()<jupyter_output><empty_output><jupyter_text>A. All text<jupyter_code># We need to specify two parameters: the number of topics and the number of passes
num_topics = 2
lda = models.LdaModel(corpus=corpus, id2word=id2word, num_topics=num_topics, passes=10)
get_lda_topics(lda, num_topics)
# LDA for num_topics = 3
num_topics = 3
lda = models.LdaModel(corpus=corpus, id2word=id2word, num_topics=num_topics, passes=10)
get_lda_topics(lda, num_topics)
# LDA for num_topics = 4
num_topics = 4
lda = models.LdaModel(corpus=corpus, id2word=id2word, num_topics=num_topics, passes=10)
get_lda_topics(lda, num_topics)<jupyter_output><empty_output><jupyter_text>B. Nouns only<jupyter_code># Let's create a function to pull out nouns from a string of text
from nltk import word_tokenize, pos_tag

def nouns(text):
    '''Given a string of text, tokenize the text and pull out only the nouns.'''
    is_noun = lambda pos: pos[:2] == 'NN'
    tokenized = word_tokenize(text)
    all_nouns = [word for (word, pos) in pos_tag(tokenized) if is_noun(pos)] 
    return ' '.join(all_nouns)
# Apply the nouns function to the transcripts to filter only on nouns
speech_df['nouns'] = speech_df.transcript.apply(nouns)
speech_df.head()
# Create dtm_n (document-term matrix with nouns only)
cv_n = CountVectorizer(stop_words=stop_words)
data_cv_n = cv_n.fit_transform(speech_df.nouns)
dtm_n = cv_dtm(speech_df,'nouns',add_stop_words)
# dtm_n
# Create the gensim corpus
corpus_n = matutils.Sparse2Corpus(scipy.sparse.csr_matrix(dtm_n.transpose()))

# Create the vocabulary dictionary
id2word_n = dict((v, k) for k, v in cv_n.vocabulary_.items())
# Let's start with 2 topics
num_topics = 2
lda_n = models.LdaModel(corpus=corpus_n, num_topics=num_topics, id2word=id2word_n, passes=10)
get_lda_topics(lda_n, num_topics)
# Let's start with 3 topics
num_topics = 3
lda_n = models.LdaModel(corpus=corpus_n, num_topics=num_topics, id2word=id2word_n, passes=10)
get_lda_topics(lda_n, num_topics)
# Let's start with 4 topics
num_topics = 4
lda_n = models.LdaModel(corpus=corpus_n, num_topics=num_topics, id2word=id2word_n, passes=10)
get_lda_topics(lda_n, num_topics)
# Test for network analysis
topic_df = get_lda_topics(lda_n, num_topics)
topic_df
topic_df.to_pickle('../dump/topic_df')
# Let's start with 10 topics
num_topics = 10
lda_n = models.LdaModel(corpus=corpus_n, num_topics=num_topics, id2word=id2word_n, passes=30)
get_lda_topics(lda_n, num_topics)
# Let's start with 20 topics
num_topics = 20
lda_n = models.LdaModel(corpus=corpus_n, num_topics=num_topics, id2word=id2word_n, passes=40)
get_lda_topics(lda_n, num_topics)<jupyter_output><empty_output><jupyter_text>B. Nouns and adjective<jupyter_code># Let's create a function to pull out nouns from a string of text
from nltk import word_tokenize, pos_tag

def nouns_adj(text):
    '''Given a string of text, tokenize the text and pull out only the nouns.'''
    is_noun = lambda pos: pos[:2] == 'NN' or pos[:2] == 'JJ'
    tokenized = word_tokenize(text)
    all_nouns = [word for (word, pos) in pos_tag(tokenized) if is_noun(pos)] 
    return ' '.join(all_nouns)
# Apply the nouns function to the transcripts to filter only on nouns
speech_df['nouns_adj'] = speech_df.transcript.apply(nouns_adj)
speech_df.head()
# Add boring words that do not add much insight to topic modeling

boring_words = ['say','like','just','dont','don','im',
                  'ive','youll','youve','things','thing','youre','right','really','lot',
                  'make','know','people','way','day','class']
add_stop_words = add_stop_words + boring_words
stop_words = text.ENGLISH_STOP_WORDS.union(add_stop_words)
# Create dtm_n (document-term matrix with nouns only)
cv_na = CountVectorizer(stop_words=stop_words)
data_cv_na = cv_na.fit_transform(speech_df.nouns_adj)
dtm_na = cv_dtm(speech_df,'nouns_adj',add_stop_words)
# dtm_na
# Create the gensim corpus
corpus_na = matutils.Sparse2Corpus(scipy.sparse.csr_matrix(dtm_na.transpose()))

# Create the vocabulary dictionary
id2word_na = dict((v, k) for k, v in cv_na.vocabulary_.items())
# Let's start with 2 topics
num_topics=2
lda_na = models.LdaModel(corpus=corpus_na, num_topics=num_topics, id2word=id2word_na, passes=10)
get_lda_topics(lda_na, num_topics)
# Let's start with 3 topics
num_topics=3
lda_na = models.LdaModel(corpus=corpus_na, num_topics=num_topics, id2word=id2word_na, passes=10)
get_lda_topics(lda_na, num_topics)
# Let's start with 4 topics
num_topics=4
lda_na = models.LdaModel(corpus=corpus_na, num_topics=num_topics, id2word=id2word_na, passes=10)
get_lda_topics(lda_na, num_topics)
# Let's start with 10 topics
num_topics=10
lda_na = models.LdaModel(corpus=corpus_na, num_topics=num_topics, id2word=id2word_na, passes=30)
get_lda_topics(lda_na, num_topics)
# Let's start with 20 topics
num_topics=20
lda_na = models.LdaModel(corpus=corpus_na, num_topics=num_topics, id2word=id2word_na, passes=40)
get_lda_topics(lda_na, num_topics)
# lda_na.print_topics()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup