<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 http
Step1:  p
Step2:  Train - Test
<ASSISTANT_TASK:>
Python Code:
from __future__ import division
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from sklearn.metrics import r2_score
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from pandas import read_csv
from pandas import datetime
from pandas.plotting import autocorrelation_plot
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.tsa.arima_model import ARIMA
from pandas import DataFrame
from sklearn.metrics import mean_squared_error
%matplotlib inline
data_path = '../../../../../Dropbox/data'
score_dic_filepath = data_path + "/arima/scoredic_testing.npy"
test = np.load(score_dic_filepath)[()]
len(test.keys())
ph_data_path = data_path + '/price_history'
npz_path = ph_data_path + '/price_history_mobattrs_date_dp_60to30_62020_trimmed'
npz_train = npz_path + '_train.npz'
assert path.isfile(npz_train)
path.abspath(npz_train)
dic = np.load(npz_train)
dic.keys()
dic = {
    "inputs": dic['inputs'][:, :, 0],
    "targets": dic['targets'],
    #"full": np.hstack((dic['inputs'][:, :, 0], dic['targets'])),
}
for key, val in dic.iteritems():
    print key, val.shape
#fix targets
fulls = np.array([np.concatenate( (item_in, item_tar + item_in[-1]) ) 
 for item_in, item_tar in zip(dic['inputs'], dic['targets'])])
fulls.shape
sns.tsplot(fulls[100])
autocorrelation_plot(fulls[100])
plot_pacf(fulls[100])
plt.show()
Explanation: http://machinelearningmastery.com/arima-for-time-series-forecasting-with-python/
http://ucanalytics.com/blogs/arima-models-manufacturing-case-study-example-part-3/
https://stats.stackexchange.com/questions/23036/estimating-same-model-over-multiple-time-series
The idea with ARIMA models is that the final residual should look like white noise otherwise, there is juice or information available in the data to extract
End of explanation
target_len = 30
XX = fulls[:, :-target_len]
XX.shape
YY = fulls[:, -target_len:]
YY.shape
from arima_estimator import ArimaEstimator
ae = ArimaEstimator(p_auto_regression_order=5, d_integration_level=2, q_moving_average=5, easy_mode=True)
ae
#duration estimation in hours!
55800 * (1.09 /12) * 30 / 3600
15*60 / (13.2 / 3) #secs each
import warnings
%%time
# with warnings.catch_warnings():
#     warnings.filterwarnings("ignore")
#     ae.fit(inputs=XX[:3], targets=YY[:3])
from collections import OrderedDict
parameters = OrderedDict([
    ('p_auto_regression_order', range(6)), #0-5
    ('d_integration_level', range(3)), #0-2
    ('q_moving_average', range(6)), #0-5
])
def cartesian_coord(*arrays):
    grid = np.meshgrid(*arrays)        
    coord_list = [entry.ravel() for entry in grid]
    points = np.vstack(coord_list).T
    return points
cart = cartesian_coord(*parameters.values())
cart.shape
108 * 15 / 60 / 2 #in hours!
count_inds=100
random_state = np.random.RandomState(seed=16011984)
random_inds = random_state.choice(range(len(XX)), count_inds, replace=False)
xx = XX[random_inds]
yy = YY[random_inds]
filepath = 'scoredic.npy'
%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    for pp, dd, qq in cart:
        scoredic = np.load(filepath)[()] if path.isfile(filepath) else OrderedDict()
        cur_tuple = (pp, dd, qq)
        if cur_tuple in scoredic:
            continue
        else:
            ae = ArimaEstimator(p_auto_regression_order=pp, d_integration_level=dd, q_moving_average=qq,
                                easy_mode=True)
            scoredic[cur_tuple] = ae.fit(inputs=xx, targets=yy).score(xx, yy)
            np.save(filepath, scoredic)
from sklearn.model_selection import GridSearchCV
# Pick hyperparameters
# There is NO separation of training and testing set because there is no transfer of knowledge from the training 
# that could be useful for validation
# Run Arima prediction steps for all instances of training dataset and get a fastdtw score from each case
# Get the mean of the fastdtw and this is the CV score
# 
# Do this for all possible parameters
fitted = model.fit()
fitted.
model_fit = model.fit(disp=0)
model_fit.summary()
residuals = DataFrame(model_fit.resid)
residuals.plot()
plt.show()
residuals.plot(kind='kde')
plt.show()
residuals.describe()
Explanation: p: The number of lag observations included in the model, also called the lag order.
d: The number of times that the raw observations are differenced, also called the degree of differencing.
q: The size of the moving average window, also called the order of moving average.
End of explanation
XX = series.values
XX.shape
#splitting
train_size = int(len(XX)*2/3)
train_size
train_set = XX[:train_size]
test_set = XX[train_size:]
train_set.shape, test_set.shape
train_set[0]
history = list(train_set)
history[0] = 3333
history = list(train_set)
len(history)
predictions = []
for tt in range(len(test_set)):
    output = ARIMA(history, order=(5,1,0)).fit(disp=0).forecast()
    y_hat = output[0] #0th is the index of the prediction
    predictions.append(y_hat)
    
    observation = test_set[tt]
    #history.append(observation)
    history.append(y_hat)
    
    print "predicted: {}, expected: {}".format(y_hat, observation)
error = mean_squared_error(predictions, test_set)
error
plt.figure(figsize=(15,6))
plt.plot(predictions, label='preds')
plt.plot(test_set, label='test set')
plt.legend()
plt.show()
Explanation: Train - Test
End of explanation
<END_TASK>