<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
Step1:  Vanilla Generative Adversarial <img src="http
Step3:  State of art weight Initialization strategy
Step4:  Discriminator
Step5:  Generator
Step6:  Check if the generator is saved properly
Step7:  Reasons why sentences seem nonsensical
Step8:  Missed out on 5 words
Step9:  Function to find nearest neighbours
<ASSISTANT_TASK:>
Python Code:
import tensorflow as tf
import numpy as np
import os
from collections import Counter
from nltk.tokenize import TweetTokenizer
import codecs
from random import randint
tf.__version__
with codecs.open(os.path.join('../data', 'sent.csv'), 'r', encoding='utf-8') as f:
    corpus_line_by_line = f.read().lower().split("\n")
corpus_line_by_line = corpus_line_by_line[:1000]
corpus_line_by_line = [line.rstrip('\r').split(',') for line in corpus_line_by_line]
corpus_line_by_line[0]
tw = TweetTokenizer()
corpus_tokenized = list(map(lambda line: (line[0], tw.tokenize(line[1])), corpus_line_by_line))
corpus_tokenized[0]
corpus = []
for sent in corpus_tokenized:
    corpus.extend([word for word in sent[1]])
def build_vocab(words, vocab_size):
     Build vocabulary of VOCAB_SIZE most frequent words 
    dictionary = dict()
    count = [('<UNK>', -1)]
    count.extend(Counter(words).most_common(vocab_size - 1))
    index = 0
    for word, _ in count:
        dictionary[word] = index
        index += 1
    index_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return dictionary, index_dictionary
vocabulary_size = len(set(corpus))
vocabulary_size
vocabulary, reverse_vocabulary = build_vocab(corpus, vocabulary_size)
for word, index in zip(reverse_vocabulary.values(), vocabulary.values()):
    print(word, index)
def index_words_in_corpus(corpus):
    return [vocabulary[token] if token in vocabulary else 0 for token in corpus]
corpus_indexed = [(line[0], index_words_in_corpus(line[1])) for line in corpus_tokenized]
len(corpus_indexed)
corpus_indexed[0]
vocabulary_size
def one_hot_encode(review):
    temp = np.zeros(vocabulary_size)
    for indx in review:
        temp[indx] = 1
    return temp
def one_hot_encode_class(sentiment):
    temp = np.zeros(2)
    if sentiment == 1:
        temp[1] = 1
    else:
        temp[0] = 1
    return temp
# data = np.array([(one_hot_encode(word), vocabulary.get(word)) for word in corpus])
data = np.array([one_hot_encode(word[1]) for word in corpus_indexed])
print("TRAIN: (Total number of words, Vocabulary size):", data.shape)
sample = data[0]
np.where(sample == 1)[0]
def decode_sentence(sample):
    sentence = []
    for i in range(sample.shape[0]):
        if sample[i] == 1:
            sentence.append(reverse_vocabulary[i])
    return ' '.join([i for i in sentence])
print(sample, decode_sentence(sample))
def get_batches(batch_size):
    idx = randint(data.shape[0]-3)
    return data[idx:idx+batch_size]
Explanation: Vanilla Generative Adversarial <img src="http://creationkinetica.com/wp-content/uploads/2015/03/new-twitter-logo-vector-400x400.jpg" height="30" width="30" style="display: inline-block"> Networks
End of explanation
def xavier_init(n_inputs, n_outputs, uniform=True):
  Set the parameter initialization using the method described.
  This method is designed to keep the scale of the gradients roughly the same
  in all layers.
  Xavier Glorot and Yoshua Bengio (2010):
           Understanding the difficulty of training deep feedforward neural
           networks. International conference on artificial intelligence and
           statistics.
  Args:
    n_inputs: The number of input nodes into each output.
    n_outputs: The number of output nodes for each input.
    uniform: If true use a uniform distribution, otherwise use a normal.
  Returns:
    An initializer.
  
  if uniform:
    # 6 was used in the paper.
    init_range = tf.sqrt(6.0 / (n_inputs + n_outputs))
    return tf.random_uniform_initializer(-init_range, init_range)
  else:
    # 3 gives us approximately the same limits as above since this repicks
    # values greater than 2 standard deviations from the mean.
    stddev = tf.sqrt(3.0 / (n_inputs + n_outputs))
    return tf.truncated_normal_initializer(stddev=stddev)
'''A recent paper by He, Rang, Zhen and Sun they build on Glorot & Bengio and suggest using 2/size_of_input_neuron
''' 
def xavier_init(size):
    in_dim = size[0]
#     xavier_stddev = 1. / in_dim
#     xavier_stddev = 2. / in_dim
    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)
    return tf.random_normal(shape=size, stddev=xavier_stddev)
Explanation: State of art weight Initialization strategy
End of explanation
with tf.name_scope('Input'):
    X = tf.placeholder(tf.float32, shape=[None, vocabulary_size])
with tf.name_scope('Discriminator_weights_biases'):    
    D_W1 = tf.Variable(xavier_init([vocabulary_size, 128]))
    D_b1 = tf.Variable(tf.zeros(shape=[128]))
    D_W2 = tf.Variable(xavier_init([128, 1]))
    D_b2 = tf.Variable(tf.zeros(shape=[1]))
    theta_D = [D_W1, D_W2, D_b1, D_b2]
def discriminator(x):
    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)
    D_logit = tf.matmul(D_h1, D_W2) + D_b2
    D_prob = tf.nn.sigmoid(D_logit)
    return D_prob, D_logit
Explanation: Discriminator
End of explanation
with tf.name_scope('Latent_space'):
    Z = tf.placeholder(tf.float32, shape=[None, 100])
with tf.name_scope('Generator_weights_biases'):
    G_W1 = tf.Variable(xavier_init([100, 128]))
    G_b1 = tf.Variable(tf.zeros(shape=[128]))
    G_W2 = tf.Variable(xavier_init([128, vocabulary_size]))
    G_b2 = tf.Variable(tf.zeros(shape=[vocabulary_size]))
    theta_G = [G_W1, G_W2, G_b1, G_b2]
def sample_Z(m, n):
    return np.random.uniform(0., 1., size=[m, n])
def generator(z):
    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)
    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2
    G_prob = tf.nn.sigmoid(G_log_prob)
    return G_prob
G_sample = generator(Z)
D_real, D_logit_real = discriminator(X)
D_fake, D_logit_fake = discriminator(G_sample)
with tf.name_scope('cost'):
    D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
    D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
    D_loss = D_loss_real + D_loss_fake
    G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))
with tf.name_scope('train'):
    D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)
    G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)
minibatch_size = 128
Z_dim = 100
saver = tf.train.Saver(max_to_keep=1)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
i = 0
for it in range(100000):
    if it % 1000 == 0:
        samples = sess.run(G_sample, feed_dict={Z: sample_Z(1, Z_dim)})
        for sample in samples:
            sample[sample > 0.95] = 1
            print(samples.shape, decode_sentence(sample))
                  
    X_mb = get_batches(minibatch_size)
    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(minibatch_size, Z_dim)})
    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(minibatch_size, Z_dim)})
    
    if it % 1000 == 0:
        print('Iter: {}'.format(it))
        print('D loss: {:.4}'. format(D_loss_curr))
        print('G_loss: {:.4}'.format(G_loss_curr))
        print()
saver.save(sess, './tf_model/generator', global_step=1000000)
Explanation: Generator
End of explanation
saver.restore(sess,  './tf_model/generator-1000000')
samples = sess.run(G_sample, feed_dict={Z: sample_Z(1, Z_dim)})
for sample in samples:
    print(sample)
    sample[sample == 1.] = 1
    sample[sample < 1.] = 0
    print(np.where(sample == 0)[0])
    print(decode_sentence(sample))
Explanation: Check if the generator is saved properly
End of explanation
with codecs.open("../data/sentiment.vec", 'r', encoding='utf-8') as f:
    next(f)
    word_vectors_twitter = f.read().split('\n')
word_vectors_twitter[0]
word_vectors = {}
for lin in word_vectors_twitter:
    line = lin.split()
#     print(line)
    try:
        word_vectors[line[0]] = np.array(line[1:])
    except:
        print(line, lin)
len(list(word_vectors.keys()))
Explanation: Reasons why sentences seem nonsensical:
We are representing our words in discreet 1 or 0, hint: Use pretrained word embeddings which we can do via our Word2Vec notebook
Notice how we were passing in the sentences data without sequence so A B C would be represented similiar to C A B
We are storing any information over time in our network (hint: LSTM) which we should do if we are to generate sequential data
Training word2vec using skipgram on twitter dataset
<img src="https://i.imgur.com/5tOMur7.png">
End of explanation
word_vectors['to']
Explanation: Missed out on 5 words
End of explanation
from sklearn.metrics.pairwise import cosine_similarity
cosine_similarity([[1, 0, -1]], [[-1,-1, 0]])
def nn(word):
    nearest = 0
    max = (0, 'max')
    for w in word_vectors.keys():
        try:
            if cosine_similarity([word_vectors[word]], [word_vectors[w]])[0][0] > max[0] and w != word:
                max = (cosine_similarity([word_vectors[word]], [word_vectors[w]])[0][0], w)
        except:
            pass
#             print(w, word_vectors[word], word_vectors[w])
    print(max[1])
    
nn('hate')
Explanation: Function to find nearest neighbours
End of explanation
<END_TASK>