<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 Make word-doc matrix
Step1:  Run model
Step2:  Questions
Step3:  Now let's look at the variations of the respective clusters, nmf topic and epithets
Question
Step4:  Visualize topic clusters
Step5:  Kmeans based tfidf matrix
Step6:  Kmeans based on nmf
<ASSISTANT_TASK:>
Python Code:
import datetime as dt
import os
import time
from cltk.corpus.greek.tlg.parse_tlg_indices import get_epithet_index
from cltk.corpus.greek.tlg.parse_tlg_indices import get_epithets
from cltk.corpus.greek.tlg.parse_tlg_indices import select_authors_by_epithet
from cltk.corpus.greek.tlg.parse_tlg_indices import get_epithet_of_author
from cltk.corpus.greek.tlg.parse_tlg_indices import get_id_author
from cltk.stop.greek.stops import STOPS_LIST as greek_stops
from cltk.tokenize.word import nltk_tokenize_words
from greek_accentuation.characters import base
import pandas  # pip install pandas
from sklearn.decomposition import NMF  # pip install scikit-learn scipy
from sklearn.externals import joblib
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
def stream_lemmatized_files(corpus_dir):
    # return all docs in a dir
    user_dir = os.path.expanduser('~/cltk_data/user_data/' + corpus_dir)
    files = os.listdir(user_dir)
    for file in files:
        filepath = os.path.join(user_dir, file)
        with open(filepath) as fo:
            yield fo.read()
def map_file_order_to_name(corpus_dir):
    # read contents of dir, order and their tlg name
    user_dir = os.path.expanduser('~/cltk_data/user_data/' + corpus_dir)
    files = os.listdir(user_dir)
    
    map_id_author = get_id_author()  # {…, '1648': 'Pyrgion Hist.', '4017': 'Syrianus Phil.', …}
    map_count_name = {}
    for count, file in enumerate(files):
        file_id = str(file[3:-4])
        name = map_id_author[file_id]
        map_count_name[count] = name
    
    return map_count_name
map_file_count_name = map_file_order_to_name('tlg_lemmatized_no_accents_no_stops')
t0 = dt.datetime.utcnow()
data_samples = []
for text in stream_lemmatized_files('tlg_lemmatized_no_accents_no_stops'):
    data_samples.append(text)
print('... finished in {}'.format(dt.datetime.utcnow() - t0))
print('Number of texts:', len(data_samples))
# view all epithets:
get_epithets()
t0 = dt.datetime.utcnow()
# tf-idf features
n_samples = 2000
n_features = 1000  # TODO: increase
n_topics = len(get_epithets())
n_top_words = 20
tfidf_vectorizer = TfidfVectorizer(max_df=1.0, 
                                   min_df=1,
                                   max_features=n_features,
                                   stop_words=None)
tfidf = tfidf_vectorizer.fit_transform(data_samples)
# save features
vector_fp = os.path.expanduser('~/cltk_data/user_data/tlg_lemmatized_no_accents_no_stops_tfidf_{0}features.pickle'.format(n_features))
joblib.dump(tfidf, vector_fp)
print('... finished in {}'.format(dt.datetime.utcnow() - t0))
# time on good server:
# 1000 features: 0:01:22
Explanation: Make word-doc matrix
End of explanation
t0 = dt.datetime.utcnow()
print("Fitting the NMF model with tf-idf features, "
      "n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
nmf = NMF(n_components=n_topics, random_state=1,
          alpha=.1, l1_ratio=.5).fit(tfidf)
# save model
nmf_fp = os.path.expanduser('~/cltk_data/user_data/tlg_lemmatized_no_accents_no_stops_tfidf_{0}features_nmf.pickle'.format(n_features))
joblib.dump(nmf, nmf_fp)
print('... finished in {}'.format(dt.datetime.utcnow() - t0))
def print_top_words(model, feature_names, n_top_words):
    for topic_id, topic in enumerate(model.components_):
        print('Topic #{}:'.format(int(topic_id)))
        print(''.join([feature_names[i] + ' ' + str(round(topic[i], 2))
              +' | ' for i in topic.argsort()[:-n_top_words - 1:-1]]))
        print()
print("Topics in NMF model:")
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
print_top_words(nmf, tfidf_feature_names, n_top_words)
tfidf.shape
doc_topic_distrib = nmf.transform(tfidf)  # numpy.ndarray
doc_topic_distrib.shape
df = pandas.DataFrame(doc_topic_distrib)
df = df.rename(map_file_count_name)
Explanation: Run model
End of explanation
df.head(100)
# for each topic (col), which author (row) has the highest value?
# TODO: get top 5 authors
for count in range(n_topics):
    print('Top author of topic {0}: {1}'.format(count, df[count].idxmax()))
# Now, transpose df and get top topic of each author
# for each topic (col), which author (row) has the highest value?
# TODO: get top 5 authors
df_t = df.T
df_t.head(10)
map_name_epithet_id = {}
for curr_name in df_t.columns:
    print(curr_name)
    try:
        top_topic = int(df_t[curr_name].idxmax())
    except TypeError:  # there are some duplicate names, just take one value for now
        top_topic = int(df_t[curr_name].idxmax().iloc[0])    
    print('    NMF topic:', top_topic)
    for _id, name in get_id_author().items():
        if curr_name == name:
            epithet = get_epithet_of_author(_id)
            print('    Traditional epithet:', epithet)
            map_name_epithet_id[name] = {'id': _id,
                                        'top_topic': top_topic,
                                        'epithet': epithet}
    print()
Explanation: Questions:
to what topic does each author most belong? (and how to determine cutoff?)
what authors most exemplify a topic?
End of explanation
map_name_epithet_id['Silenus Trag.']
# Group by epithet, collect topics
# {<epithet>: [<topics>]}
from collections import defaultdict
map_epithet_topics = defaultdict(list)
for name, _dict in map_name_epithet_id.items():
    epithet = _dict['epithet']
    top_topic = _dict['top_topic']
    map_epithet_topics[epithet].append(top_topic)
# import pprint
# pp = pprint.PrettyPrinter(indent=4)
# pp.pprint(dict(map_epithet_topics))
print(dict(map_epithet_topics))
# which epithet has the most topics associated with it?
map_epithet_count_topics = {}
for epithet, topic_list in map_epithet_topics.items():
    map_epithet_count_topics[epithet] = len(topic_list)
sorted(map_epithet_count_topics.items(), key=lambda x:x[1], reverse=True)
# Group by topic, collect epithets
# {<topic>: [<epithets>]}
from collections import defaultdict
map_topic_epithets = defaultdict(list)
for name, _dict in map_name_epithet_id.items():
    epithet = _dict['epithet']
    top_topic = _dict['top_topic']
    map_topic_epithets[top_topic].append(epithet)
dict(map_topic_epithets)
# least, most cohesive epithets
# which epithet has the most topics associated with it?
map_topics_count_epithet = {}
for topic, epithet_list in map_topic_epithets.items():
    map_topics_count_epithet[topic] = len(epithet_list)
# map_topics_count_epithet
sorted_list_tuples = sorted(map_topics_count_epithet.items(), key=lambda x:x[1], reverse=True)
for topic_freq in sorted_list_tuples:
    topic_number = str(topic_freq[0])
    doc_freq = str(topic_freq[1])
    print('Topic #{0} has {1} author-documents in it'.format(topic_number, doc_freq))
# also do this all w/ countvectorizer?
# http://scikit-learn.org/stable/modules/clustering.html
dataset_array = df.values
print(dataset_array.dtype)  # kmeans needs to be homogeneous data type (here, float64)
print(dataset_array)
# do I need to normalize
# sklearn.preprocessing.StandardScaler
from sklearn import preprocessing
# http://scikit-learn.org/stable/modules/preprocessing.html
# first load scaler and train on given data set
scaler = preprocessing.StandardScaler().fit(df)
scaler.mean_
scaler.scale_
t0 = dt.datetime.utcnow()
# actually do normalization; can be reused for eg a training set
df_scaled = pandas.DataFrame(scaler.transform(df))
print('... finished in {}'.format(dt.datetime.utcnow() - t0))
Explanation: Now let's look at the variations of the respective clusters, nmf topic and epithets
Question: Which topics are found within each epithet?
Question: Which epithets are found within each topic? And how many?
End of explanation
from sklearn import cluster
# Convert DataFrame to matrix (numpy.ndarray)
matrix = df_scaled.as_matrix()
km = cluster.KMeans(n_clusters=n_topics)
km.fit(matrix)
# Get cluster assignment labels
labels = km.labels_  # these are the topics 0-54; array([53, 53, 16, ..., 42, 16, 13]
# Format results as a DataFrame
df_clusters = pandas.DataFrame([df_scaled.index, labels]).T  # add author names to the 0 col
df_clusters.head(5)
%matplotlib inline
import matplotlib.pyplot as plt  # pip install matplotlib
import matplotlib
matplotlib.style.use('ggplot')
# from pandas.tools.plotting import table
# this is a cluseter of the already-clustered kmeans topics; not very informative
plt.figure()
df_clusters.plot.scatter(x=0, y=1)  # y is topics no., x is doc id
Explanation: Visualize topic clusters
End of explanation
# try clustering the original tfidf
# tfidf_dense = tfidf.toarray()
scaler = preprocessing.StandardScaler(with_mean=False).fit(tfidf) # either with_mean=False or make dense
# save scaler
scaler_fp = os.path.expanduser('~/cltk_data/user_data/tlg_lemmatized_no_accents_no_stops_tfidf_{0}features_scaler.pickle'.format(n_features))
joblib.dump(df_scaled, scaler_fp)
import numpy as np
# direct Pandas conversion of sparse scipy matrix not supported
# Following http://stackoverflow.com/a/17819427
# df_scaled_tfidf = pandas.DataFrame(scaler.transform(tfidf))
# df_scaled_tfidf = pandas.DataFrame()
scaler_tfidf = scaler.transform(tfidf)  # sparse matrix of type '<class 'numpy.float64'>
pandas.SparseDataFrame([pandas.SparseSeries(scaler_tfidf[i].toarray().ravel()) for i in np.arange(scaler_tfidf.shape[0])])
df_scaled_tfidf = pandas.SparseDataFrame([pandas.SparseSeries(scaler_tfidf[i].toarray().ravel()) for i in np.arange(scaler_tfidf.shape[0])])
# type(df)  # pandas.sparse.frame.SparseDataFrame
# Convert DataFrame to matrix (numpy.ndarray)
matrix_tfidf = df_scaled_tfidf.as_matrix()
km_tfidf = cluster.KMeans(n_clusters=n_topics)
km_tfidf.fit(matrix_tfidf)
# Get cluster assignment labels
labels = km_tfidf.labels_  # these are the topics 0-54; array([53, 53, 16, ..., 42, 16, 13]
# Format results as a DataFrame
df_clusters_tfidf = pandas.DataFrame([df_scaled_tfidf.index, labels]).T  # add author names to the 0 col
df_clusters_tfidf.head(10)
plt.figure()
df_clusters_tfidf.plot.scatter(x=0, y=1)  # y is topics no., x is doc id
Explanation: Kmeans based tfidf matrix
End of explanation
nmf_array = nmf.components_
t0 = dt.datetime.utcnow()
# nmf_dense = nmf_array.toarray()
scaler = preprocessing.StandardScaler().fit(nmf_array) # either with_mean=False or make dense
# save features
tfidf_matrix_scaler_fp = os.path.expanduser('~/cltk_data/user_data/tlg_lemmatized_no_accents_no_stops_tfidf_matrix_{0}features.pickle'.format(n_features))
joblib.dump(scaler, tfidf_matrix_scaler_fp)
print('... finished in {}'.format(dt.datetime.utcnow() - t0))
df_scaled_nmf = pandas.DataFrame(scaler.transform(nmf_array))
# Convert DataFrame to matrix (numpy.ndarray)
matrix_nmf = df_scaled_nmf.as_matrix()
km_nmf = cluster.KMeans(n_clusters=n_topics)
km_nmf.fit(matrix_nmf)
# Get cluster assignment labels
labels = km_nmf.labels_  # these are the clusters 0-54; array([ 1,  4, 11, 14, 28,  9, 30,
# Format results as a DataFrame
df_clusters_nmf = pandas.DataFrame([df_scaled_nmf.index, labels]).T  # add author names to the 0 col
df_clusters_nmf.head(10)
plt.figure()
df_clusters_nmf.plot.scatter(x=0, y=1)  # y is topics no., x is doc id
Explanation: Kmeans based on nmf
End of explanation
<END_TASK>