<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 Import
Step1:  file_path
Step2:  LSTM - Hyper Params
Step3:  vgg16
Step4:  load_vgg16
Step5:  File Info
Step6:  Text Reader
def input_pipeline(filenames, batch_size, num_epochs=None)
Step7:  LSTM First Layer
Step8:  matrix_concat
Step9:  LSTM Second Layer
Step10:  Test
Step11:  Result Check
<ASSISTANT_TASK:>
Python Code:
import tensorflow as tf
from PIL import Image
import numpy as np
from scipy.misc import imread, imresize
from imagenet_classes import class_names
import os
Explanation: Import
End of explanation
#File Path
# filepath_input  = "./data/run/" #input csv file path
filepath_ckpt  = "./ckpt/model_weight.ckpt" #weight saver check point file path
filepath_pred = "./output/predicted.csv" #predicted value file path
filename_queue_description = tf.train.string_input_producer(['./data/description/raw_data.csv'])
num_record = 50
Explanation: file_path
End of explanation
label_vec_size = 5
input_vec_size = 27
batch_size = 50
state_size_1 = 100
state_size_2 = 4096 + state_size_1
hidden = 15
learning_rate = 0.01
Explanation: LSTM - Hyper Params
End of explanation
class vgg16:
    def __init__(self, imgs, weights=None, sess=None):
        self.imgs = imgs
        self.convlayers()
        self.fc_layers()
        self.probs = tf.nn.softmax(self.fc3l)
        if weights is not None and sess is not None:
            self.load_weights(weights, sess)
    def convlayers(self):
        self.parameters = []
        # zero-mean input
        with tf.name_scope('preprocess') as scope:
            mean = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32, shape=[1, 1, 1, 3], name='img_mean')
            images = self.imgs-mean
        # conv1_1
        with tf.name_scope('conv1_1') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv1_1 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv1_2
        with tf.name_scope('conv1_2') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv1_1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv1_2 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # pool1
        self.pool1 = tf.nn.max_pool(self.conv1_2,
                               ksize=[1, 2, 2, 1],
                               strides=[1, 2, 2, 1],
                               padding='SAME',
                               name='pool1')
        # conv2_1
        with tf.name_scope('conv2_1') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.pool1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv2_1 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv2_2
        with tf.name_scope('conv2_2') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv2_1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv2_2 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # pool2
        self.pool2 = tf.nn.max_pool(self.conv2_2,
                               ksize=[1, 2, 2, 1],
                               strides=[1, 2, 2, 1],
                               padding='SAME',
                               name='pool2')
        # conv3_1
        with tf.name_scope('conv3_1') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.pool2, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv3_1 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv3_2
        with tf.name_scope('conv3_2') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv3_1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv3_2 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv3_3
        with tf.name_scope('conv3_3') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv3_2, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv3_3 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # pool3
        self.pool3 = tf.nn.max_pool(self.conv3_3,
                               ksize=[1, 2, 2, 1],
                               strides=[1, 2, 2, 1],
                               padding='SAME',
                               name='pool3')
        # conv4_1
        with tf.name_scope('conv4_1') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.pool3, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv4_1 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv4_2
        with tf.name_scope('conv4_2') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv4_1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv4_2 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv4_3
        with tf.name_scope('conv4_3') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv4_2, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv4_3 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # pool4
        self.pool4 = tf.nn.max_pool(self.conv4_3,
                               ksize=[1, 2, 2, 1],
                               strides=[1, 2, 2, 1],
                               padding='SAME',
                               name='pool4')
        # conv5_1
        with tf.name_scope('conv5_1') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.pool4, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv5_1 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv5_2
        with tf.name_scope('conv5_2') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv5_1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv5_2 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # conv5_3
        with tf.name_scope('conv5_3') as scope:
            kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
                                                     stddev=1e-1), name='weights')
            conv = tf.nn.conv2d(self.conv5_2, kernel, [1, 1, 1, 1], padding='SAME')
            biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
                                 trainable=True, name='biases')
            out = tf.nn.bias_add(conv, biases)
            self.conv5_3 = tf.nn.relu(out, name=scope)
            self.parameters += [kernel, biases]
        # pool5
        self.pool5 = tf.nn.max_pool(self.conv5_3,
                               ksize=[1, 2, 2, 1],
                               strides=[1, 2, 2, 1],
                               padding='SAME',
                               name='pool4')
    def fc_layers(self):
        # fc1
        with tf.name_scope('fc1') as scope:
            shape = int(np.prod(self.pool5.get_shape()[1:]))
            fc1w = tf.Variable(tf.truncated_normal([shape, 4096],
                                                         dtype=tf.float32,
                                                         stddev=1e-1), name='weights')
            fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
                                 trainable=True, name='biases')
            pool5_flat = tf.reshape(self.pool5, [-1, shape])
            fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b)
            self.fc1 = tf.nn.relu(fc1l)
            self.parameters += [fc1w, fc1b]
        # fc2
        with tf.name_scope('fc2') as scope:
            fc2w = tf.Variable(tf.truncated_normal([4096, 4096],
                                                         dtype=tf.float32,
                                                         stddev=1e-1), name='weights')
            fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
                                 trainable=True, name='biases')
            fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b)
            self.fc2 = tf.nn.relu(fc2l)
            self.parameters += [fc2w, fc2b]
        # fc3
        with tf.name_scope('fc3') as scope:
            fc3w = tf.Variable(tf.truncated_normal([4096, 1000],
                                                         dtype=tf.float32,
                                                         stddev=1e-1), name='weights')
            fc3b = tf.Variable(tf.constant(1.0, shape=[1000], dtype=tf.float32),
                                 trainable=True, name='biases')
            self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
            self.parameters += [fc3w, fc3b]
    def load_weights(self, weight_file, sess):
        weights = np.load(weight_file)
        keys = sorted(weights.keys())
        for i, k in enumerate(keys):
            print(i, k, np.shape(weights[k]))
            sess.run(self.parameters[i].assign(weights[k]))
Explanation: vgg16
End of explanation
with tf.Session() as sess_vgg:
    imgs = tf.placeholder(tf.float32, [None, 200, 200, 3])
    vgg = vgg16(imgs, 'vgg16_weights.npz', sess_vgg)
    img_files = ['./data/img/cropped/' + i for i in os.listdir('./data/img/cropped')]
    imgs = [imread(file, mode='RGB') for file in img_files]
    temps = [sess_vgg.run(vgg.fc1, feed_dict={vgg.imgs: [imgs[i]]})[0] for i in range(50)]
    reimgs= np.reshape(a=temps, newshape=[50,-1])
    sess_vgg.close()
Explanation: load_vgg16
End of explanation
reader = tf.TextLineReader()
key,value = reader.read(filename_queue_description)
record_defaults =[[-1], [-1], [-1], [-1], [-1], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2]]
lab1, lab2, lab3, lab4, lab5, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15 = tf.decode_csv(value, record_defaults)  
feature_label = tf.stack([lab1, lab2, lab3, lab4, lab5])
feature_word = tf.stack([w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15])
with tf.Session() as sess_data:
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)
    img_queue = []
    for i in range(num_record):
#         image = sess.run(images)
        
        label, raw_word = sess_data.run([feature_label, feature_word])
        onehot = tf.one_hot(indices=raw_word, depth=27)
        if i == 0:
            full_input = onehot
            full_label = label
        else:
            full_input = tf.concat([full_input, onehot], 0)
            full_label = tf.concat([full_label, label], 0)
#         print(sess.run(tf.shape(image)))
#         batch = tf.train.batch([image, label], 1)
#         print(sess.run(batch))
        
    coord.request_stop()
    coord.join(threads)
    sess_data.close()
Explanation: File Info
End of explanation
with tf.name_scope('batch') as scope:
    # full_label = tf.reshape(full_label, [batch_size, hidden, label_vec_size])
    full_input = tf.reshape(full_input, [batch_size, hidden, input_vec_size])
    input_batch, label_batch = tf.train.batch([full_input, full_input], batch_size=1)
Explanation: Text Reader
def input_pipeline(filenames, batch_size, num_epochs=None):
    filename_queue = tf.train.string_input_producer(filenames, num_epochs=num_epochs, shuffle=False)
    images = tf.image.decode_png(value, channels=3, dtype=tf.uint8)
    reader = tf.TextLineReader()
    key,value = reader.read(filename_queue_description)
    record_defaults =[[-1], [-1], [-1], [-1], [-1], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2], [-2]]
    lab1, lab2, lab3, lab4, lab5, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15 = tf.decode_csv(value, record_defaults)  
feature_label = tf.stack([lab1, lab2, lab3, lab4, lab5])
feature_word = tf.stack([w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15])
example_batch, label_batch = tf.train.batch([images, feature_label], batch_size=batch_size)
return example_batch, label_batch
with tf.Session() as sess:
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)
input_pipeline(filename_queue_description, num_epochs=1, batch_size=10)
coord.request_stop()
coord.join(threads)
sess.close()
Batching
End of explanation
with tf.name_scope('lstm_layer_1') as scope:
    with tf.variable_scope('lstm_layer_1'):
        rnn_cell_1 = tf.contrib.rnn.BasicLSTMCell(state_size_1, reuse=None)
        output_1, _ = tf.contrib.rnn.static_rnn(rnn_cell_1, tf.unstack(full_input, axis=1), dtype=tf.float32)
#         output_w_1 = tf.Variable(tf.truncated_normal([hidden, state_size_1, input_vec_size]))
#         output_b_1 = tf.Variable(tf.zeros([input_vec_size]))
#         pred_temp = tf.matmul(output_1, output_w_1) + output_b_1
with tf.Session() as sess_temp:
    print(sess_temp.run(tf.shape(output_1)))
Explanation: LSTM First Layer
End of explanation
input_2 = [tf.concat([out, reimgs], axis=1) for out in output_1]
Explanation: matrix_concat
End of explanation
with tf.name_scope('lstm_layer_2') as scope:
    with tf.variable_scope('lstm_layer_2'):
        rnn_cell_2 = tf.contrib.rnn.BasicLSTMCell(state_size_2, reuse=None)
        output_2, _ = tf.contrib.rnn.static_rnn(rnn_cell_2, tf.unstack(input_2, axis=0), dtype=tf.float32)
        output_w_2 = tf.Variable(tf.truncated_normal([hidden, state_size_2, input_vec_size]))
        output_b_2 = tf.Variable(tf.zeros([input_vec_size]))
        pred = tf.nn.softmax(tf.matmul(output_2, output_w_2) + output_b_2)
with tf.name_scope('loss') as scope:
    loss = tf.constant(0, tf.float32)
    for i in range(hidden):
        loss += tf.losses.softmax_cross_entropy(tf.unstack(full_input, axis=1)[i], tf.unstack(pred, axis=0)[i])
    train = tf.train.AdamOptimizer(learning_rate).minimize(loss)
with tf.Session() as sess_train:
    sess_train.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    save_path = saver.save(sess_train, filepath_ckpt)
    
    for i in range(31):
        sess_train.run(train)
        if i % 5 == 0:
            print("loss : ", sess_train.run(loss))
#             print("pred : ", sess.run(pred))
    save_path = saver.save(sess_train, filepath_ckpt)
    print("= Weigths are saved in " + filepath_ckpt)
    sess_train.close()
Explanation: LSTM Second Layer
End of explanation
with tf.Session() as sess_vgg_test:
    imgs = tf.placeholder(tf.float32, [None, 200, 200, 3])
    vgg = vgg16(imgs, 'vgg16_weights.npz', sess_vgg_test)
    test_img_files = ['./data/img/cropped/001.png']
    test_imgs = [imread(file, mode='RGB') for file in test_img_files]
#     bilinear_test_imgs = [imresize(arr=img,interp='bilinear') for img in test_imgs]
    temps = [sess_vgg_test.run(vgg.fc1, feed_dict={vgg.imgs: [img]})[0] for img in test_imgs]
    test_reimgs= np.reshape(a=temps, newshape=[1,-1])
    sess_vgg_test.close()
start_input = tf.zeros([1,15,27])
with tf.Session() as sess_init_generator:
    input_init = sess_init_generator.run(start_input)
sos = [0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
input_init[0][0] = sos
with tf.name_scope('lstm_layer_1') as scope:
    with tf.variable_scope('lstm_layer_1'):
        rnn_cell_1 = tf.contrib.rnn.BasicLSTMCell(state_size_1, reuse=True)
        output_test_1, _ = tf.contrib.rnn.static_rnn(rnn_cell_1, tf.unstack(input_init, axis=1), dtype=tf.float32)
# output_t_1 = tf.contrib.rnn.static_rnn(rnn_cell, tf.unstack(full_input, axis=1), dtype=tf.float32)
# pred = tf.nn.softmax(tf.matmul(output1, output_w[0]) + output_b[0])
input_2 = [tf.concat([out, test_reimgs], axis=1) for out in output_test_1]
with tf.name_scope('lstm_layer_2') as scope:
    with tf.variable_scope('lstm_layer_2'):
        rnn_cell_2 = tf.contrib.rnn.BasicLSTMCell(state_size_2, reuse=None)
        output_2, _ = tf.contrib.rnn.static_rnn(rnn_cell_2, tf.unstack(input_2, axis=0), dtype=tf.float32)
        output_w_2 = tf.Variable(tf.truncated_normal([hidden, state_size_2, input_vec_size]))
        output_b_2 = tf.Variable(tf.zeros([input_vec_size]))
        pred = tf.nn.softmax(tf.matmul(output_2, output_w_2) + output_b_2)
sess_model = tf.Session()
saver = tf.train.Saver(allow_empty=True)
saver.restore(sess_model, filepath_ckpt)
for i in range(hidden):
    result = sess_model.run(pred)
    result_temp = result[i]
    if i == hidden -1:
        pass
    else:
        input_init[0][i+1] = result_temp
Explanation: Test
End of explanation
print(result.shape)
decoded_result = np.argmax(a=result, axis=2)
print(result)
print(decoded_result)
Explanation: Result Check
End of explanation
<END_TASK>