<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 TensorFlow
Step1:  Download the Data
Step2:  Dataset Metadata
Step3:  Building a TensorFlow Custom Estimator
Creating feature columns
Creating model_fn
Create estimator using the model_fn
Define data input_fn
Define Train and evaluate experiment
Run experiment with parameters
1. Create feature columns
Step4:  2. Create model_fn
Use feature columns to create input_layer
Use tf.keras.layers to define the model architecutre and output
Use binary_classification_head for create EstimatorSpec
Step5:  3. Create estimator
Step6:  4. Data Input Function
Step7:  5. Experiment Definition
Step8:  6. Run Experiment with Parameters
<ASSISTANT_TASK:>
Python Code:
import math
import os
import pandas as pd
import numpy as np
from datetime import datetime
import tensorflow as tf
from tensorflow import data
print "TensorFlow : {}".format(tf.__version__)
SEED = 19831060
Explanation: TensorFlow: Optimizing Learning Rate
End of explanation
DATA_DIR='data'
# !mkdir $DATA_DIR
# !gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR
# !gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR
TRAIN_DATA_FILE = os.path.join(DATA_DIR, 'adult.data.csv')
EVAL_DATA_FILE = os.path.join(DATA_DIR, 'adult.test.csv')
TRAIN_DATA_SIZE = 32561
EVAL_DATA_SIZE = 16278
Explanation: Download the Data
End of explanation
HEADER = ['age', 'workclass', 'fnlwgt', 'education', 'education_num',
               'marital_status', 'occupation', 'relationship', 'race', 'gender',
               'capital_gain', 'capital_loss', 'hours_per_week',
               'native_country', 'income_bracket']
HEADER_DEFAULTS = [[0], [''], [0], [''], [0], [''], [''], [''], [''], [''],
                       [0], [0], [0], [''], ['']]
NUMERIC_FEATURE_NAMES = ['age', 'education_num', 'capital_gain', 'capital_loss', 'hours_per_week']
CATEGORICAL_FEATURE_NAMES = ['gender', 'race', 'education', 'marital_status', 'relationship', 
                             'workclass', 'occupation', 'native_country']
FEATURE_NAMES = NUMERIC_FEATURE_NAMES + CATEGORICAL_FEATURE_NAMES
TARGET_NAME = 'income_bracket'
TARGET_LABELS = [' <=50K', ' >50K']
WEIGHT_COLUMN_NAME = 'fnlwgt'
NUM_CLASSES = len(TARGET_LABELS)
def get_categorical_features_vocabolary():
    data = pd.read_csv(TRAIN_DATA_FILE, names=HEADER)
    return {
        column: list(data[column].unique()) 
        for column in data.columns if column in CATEGORICAL_FEATURE_NAMES
    }
feature_vocabolary = get_categorical_features_vocabolary()
print(feature_vocabolary)
Explanation: Dataset Metadata
End of explanation
def create_feature_columns():
    
    feature_columns = []
    
    for column in NUMERIC_FEATURE_NAMES:
        feature_column = tf.feature_column.numeric_column(column)
        feature_columns.append(feature_column)
        
    for column in CATEGORICAL_FEATURE_NAMES:
        vocabolary = feature_vocabolary[column]
        embed_size = round(math.sqrt(len(vocabolary)) * 1.5)
        feature_column = tf.feature_column.embedding_column(
            tf.feature_column.categorical_column_with_vocabulary_list(column, vocabolary), 
            embed_size)
        feature_columns.append(feature_column)
        
    return feature_columns
Explanation: Building a TensorFlow Custom Estimator
Creating feature columns
Creating model_fn
Create estimator using the model_fn
Define data input_fn
Define Train and evaluate experiment
Run experiment with parameters
1. Create feature columns
End of explanation
from tensorflow.python.ops import math_ops
def find_learning_rate(params):
    
    training_step = tf.cast(tf.train.get_global_step(), tf.float32)
    factor = tf.cast(tf.multiply(1.e-5, training_step*training_step), tf.float32)
    learning_rate = tf.add(params.learning_rate, factor)
    return learning_rate
    
def update_learning_rate(params):
    
    training_step = tf.cast(tf.train.get_global_step(), tf.int32)
    base_cycle = tf.floordiv(training_step, params.cycle_length)
    current_cycle = tf.cast(tf.round(tf.sqrt(tf.cast(base_cycle, tf.float32))) + 1, tf.int32)
    current_cycle_length = tf.cast(tf.multiply(current_cycle, params.cycle_length), tf.int32)
    cycle_step = tf.mod(training_step, current_cycle_length)
    learning_rate = tf.cond(
        tf.equal(cycle_step, 0),
        lambda: params.learning_rate,
        lambda: tf.train.cosine_decay(
            learning_rate=params.learning_rate,
            global_step=cycle_step,
            decay_steps=current_cycle_length,
            alpha=0.0,
        )
    )
    
    tf.summary.scalar('base_cycle', base_cycle)
    tf.summary.scalar('current_cycle', current_cycle)
    tf.summary.scalar('current_cycle_length', current_cycle_length)
    tf.summary.scalar('cycle_step', cycle_step)
    tf.summary.scalar('learning_rate', learning_rate)
    return learning_rate
def model_fn(features, labels, mode, params):
    
    is_training = True if mode == tf.estimator.ModeKeys.TRAIN else False
    
    # model body
    def _inference(features, mode, params):
        
        feature_columns = create_feature_columns()
        input_layer = tf.feature_column.input_layer(features=features, feature_columns=feature_columns)
        dense_inputs = input_layer
        for i in range(len(params.hidden_units)):
            dense = tf.keras.layers.Dense(params.hidden_units[i], activation='relu')(dense_inputs)
            dense_dropout = tf.keras.layers.Dropout(params.dropout_prob)(dense, training=is_training)
            dense_inputs = dense_dropout
        fully_connected = dense_inputs  
        logits = tf.keras.layers.Dense(units=1, name='logits', activation=None)(fully_connected)
        return logits
    
    # model head
    head = tf.contrib.estimator.binary_classification_head(
        label_vocabulary=TARGET_LABELS,
        weight_column=WEIGHT_COLUMN_NAME
    )
    
    learning_rate = find_learning_rate(params) if params.lr_search else update_learning_rate(params)
    
    return head.create_estimator_spec(
        features=features,
        mode=mode,
        logits=_inference(features, mode, params),
        labels=labels,
        optimizer=tf.train.AdamOptimizer(learning_rate)
    )
    
Explanation: 2. Create model_fn
Use feature columns to create input_layer
Use tf.keras.layers to define the model architecutre and output
Use binary_classification_head for create EstimatorSpec
End of explanation
def create_estimator(params, run_config):
    
    feature_columns = create_feature_columns()
    
    estimator = tf.estimator.Estimator(
        model_fn,
        params=params,
        config=run_config
    )
    
    return estimator
Explanation: 3. Create estimator
End of explanation
def make_input_fn(file_pattern, batch_size, num_epochs, 
                  mode=tf.estimator.ModeKeys.EVAL):
    
    def _input_fn():
        dataset = tf.data.experimental.make_csv_dataset(
            file_pattern=file_pattern,
            batch_size=batch_size,
            column_names=HEADER,
            column_defaults=HEADER_DEFAULTS,
            label_name=TARGET_NAME,
            field_delim=',',
            use_quote_delim=True,
            header=False,
            num_epochs=num_epochs,
            shuffle=(mode==tf.estimator.ModeKeys.TRAIN)
        )
        
        iterator = dataset.make_one_shot_iterator()
        features, target = iterator.get_next()
        return features, target
    
    return _input_fn
Explanation: 4. Data Input Function
End of explanation
def train_and_evaluate_experiment(params, run_config):
    
    # TrainSpec ####################################
    train_input_fn = make_input_fn(
        TRAIN_DATA_FILE,
        batch_size=params.batch_size,
        num_epochs=None,
        mode=tf.estimator.ModeKeys.TRAIN
    )
    
    train_spec = tf.estimator.TrainSpec(
        input_fn = train_input_fn,
        max_steps=params.traning_steps
    )
    ###############################################    
    
    # EvalSpec ####################################
    eval_input_fn = make_input_fn(
        EVAL_DATA_FILE,
        num_epochs=1,
        batch_size=params.batch_size,
    )
    eval_spec = tf.estimator.EvalSpec(
        name=datetime.utcnow().strftime("%H%M%S"),
        input_fn = eval_input_fn,
        steps=None,
        start_delay_secs=0,
        throttle_secs=params.eval_throttle_secs
    )
    ###############################################
    tf.logging.set_verbosity(tf.logging.INFO)
    
    if tf.gfile.Exists(run_config.model_dir):
        print("Removing previous artefacts...")
        tf.gfile.DeleteRecursively(run_config.model_dir)
            
    print ''
    estimator = create_estimator(params, run_config)
    print ''
    
    time_start = datetime.utcnow() 
    print("Experiment started at {}".format(time_start.strftime("%H:%M:%S")))
    print(".......................................") 
#     tf.estimator.train_and_evaluate(
#         estimator=estimator,
#         train_spec=train_spec, 
#         eval_spec=eval_spec
#     )
    estimator.train(train_input_fn, steps=params.traning_steps)
    time_end = datetime.utcnow() 
    print(".......................................")
    print("Experiment finished at {}".format(time_end.strftime("%H:%M:%S")))
    print("")
    time_elapsed = time_end - time_start
    print("Experiment elapsed time: {} seconds".format(time_elapsed.total_seconds()))
Explanation: 5. Experiment Definition
End of explanation
MODELS_LOCATION = 'models/census'
MODEL_NAME = 'dnn_classifier-01'
model_dir = os.path.join(MODELS_LOCATION, MODEL_NAME)
BATCH_SIZE = 64
NUM_EPOCHS = 10
steps_per_epoch = int(math.ceil((TRAIN_DATA_SIZE / BATCH_SIZE)))
training_steps = int(steps_per_epoch * NUM_EPOCHS)
print("Training data size: {}".format(TRAIN_DATA_SIZE))
print("Btach data size: {}".format(BATCH_SIZE))
print("Steps per epoch: {}".format(steps_per_epoch))
print("Traing epochs: {}".format(NUM_EPOCHS))
print("Training steps: {}".format(training_steps))
params  = tf.contrib.training.HParams(
    batch_size=BATCH_SIZE,
    traning_steps=training_steps,
    hidden_units=[64, 32],
    learning_rate=1.e-3,
    cycle_length=500,
    dropout_prob=0.1,
    eval_throttle_secs=0,
    lr_search=False
)
run_config = tf.estimator.RunConfig(
    tf_random_seed=SEED,
    save_checkpoints_steps=steps_per_epoch,
    log_step_count_steps=100,
    save_summary_steps=1,
    keep_checkpoint_max=3,
    model_dir=model_dir,
)
train_and_evaluate_experiment(params, run_config)
Explanation: 6. Run Experiment with Parameters
End of explanation
<END_TASK>