content: 
  from sklearn import *
  import sklearn
  import pandas as pd
  import numpy as np
  import xgboost as xgb
  import lightgbm as lgb
  from time import gmtime, strftime
  import numpy.random as rng
  from multiprocessing.dummy import Pool
  import h5py
  import concurrent.futures
  import tensorflow as tf
  import multiprocessing as mp
  
  from sklearn.cross_validation import KFold
  from keras.models import Sequential, Model
  from keras.layers.core import Dense, Dropout, Flatten, Reshape
  from keras.layers.normalization import BatchNormalization
  from keras.layers.embeddings import Embedding
  from keras.layers import Input, concatenate, merge
  from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2D
  from keras.optimizers import SGD, RMSprop, Adam
  from keras.callbacks import EarlyStopping
  from keras.utils import np_utils
  from keras import backend as K
  from sklearn.metrics import log_loss
  from keras import __version__ as keras_version
  
  graph = tf.get_default_graph()
  
  HIDDEN_UNITS = [64, 16, 8]
  DNN_EPOCHS = 40
  BATCH_SIZE = 5
  DNN_BN = True
  DROPOUT_RATE = 0.5
  SIAMESE_PAIR_SIZE = 100000
  MAX_WORKERS = 8
  EMBEDDING_SIZE = 6
  
  full_feature = True
  
  data_folder = '../Data/'
  train = pd.read_csv(data_folder + 'training_variants')
  #print train.dtypes
  test = pd.read_csv(data_folder + 'test_variants')
  trainx = pd.read_csv(data_folder + 'training_text', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])
  #print trainx.dtypes
  testx = pd.read_csv(data_folder + 'test_text', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])
  
  train = pd.merge(train, trainx, how='left', on='ID').fillna('')
  #train = train.iloc[1:1000]
  y = train['Class'].values
  train = train.drop(['Class'], axis=1)
  
  test = pd.merge(test, testx, how='left', on='ID').fillna('')
  pid = test['ID'].values
  
  #df_all = pd.concat((train, test), axis=0, ignore_index=True)
  #df_all['Gene_Share'] = df_all.apply(lambda r: sum([1 for w in r['Gene'].split(' ') if w in r['Text'].split(' ')]), axis=1).astype(np.int8)
  #df_all['Variation_Share'] = df_all.apply(lambda r: sum([1 for w in r['Variation'].split(' ') if w in r['Text'].split(' ')]), axis=1).astype(np.int8)
  #
  #print df_all[['Gene_Share', 'Variation_Share']].max()
  ## exit(0)
  #if full_feature:
  #    #commented for Kaggle Limits
  #    for i in range(5):
  #        df_all['Gene_'+str(i)] = df_all['Gene'].map(lambda x: str(x[i]) if len(x)>i else '')
  #        df_all['Variation'+str(i)] = df_all['Variation'].map(lambda x: str(x[i]) if len(x)>i else '')
  #    print df_all.dtypes
  #
  #    gen_var_lst = sorted(list(train.Gene.unique()) + list(train.Variation.unique()))
  #    print(len(gen_var_lst))
  #    gen_var_lst = [x for x in gen_var_lst if len(x.split(' '))==1]
  #    print(len(gen_var_lst))
  #    i_ = 0
  #    #commented for Kaggle Limits
  #    for gen_var_lst_itm in gen_var_lst:
  #        if i_ % 100 == 0: print(i_)
  #        df_all['GV_'+str(gen_var_lst_itm)] = df_all['Text'].map(lambda x: str(x).count(str(gen_var_lst_itm))).astype(np.int8)
  #        i_ += 1
  #        if i_ == 5:
  #            break
  #
  #for c in df_all.columns:
  #    if df_all[c].dtype == 'object':
  #        if c in ['Gene','Variation']:
  #            lbl = preprocessing.LabelEncoder()
  #            df_all[c+'_lbl_enc'] = lbl.fit_transform(df_all[c].values)
  #            df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
  #            df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x).split(' ')))
  #        elif c != 'Text':
  #            lbl = preprocessing.LabelEncoder()
  #            df_all[c] = lbl.fit_transform(df_all[c].values)
  #        if c=='Text':
  #            df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
  #            df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x).split(' ')))
  #
  #train = df_all.iloc[:len(train)]
  #print "... train dtypes before svd ..."
  #print train.dtypes
  #print train.head()
  #exit(0)
  #test = df_all.iloc[len(train):]
  #
  #class cust_regression_vals(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
  #    def fit(self, x, y=None):
  #        return self
  #    def transform(self, x):
  #        x = x.drop(['Gene', 'Variation','ID','Text'],axis=1).values
  #        return x
  #
  #class cust_txt_col(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
  #    def __init__(self, key):
  #        self.key = key
  #    def fit(self, x, y=None):
  #        return self
  #    def transform(self, x):
  #        return x[self.key].apply(str)
  #
  #print('Pipeline...')
  #fp = pipeline.Pipeline([
  #    ('union', pipeline.FeatureUnion(
  #        n_jobs = -1,
  #        transformer_list = [
  #            ('standard', cust_regression_vals()),
  #            ('pi1', pipeline.Pipeline([('Gene', cust_txt_col('Gene')), ('count_Gene', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))), ('tsvd1', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12))])),
  #            ('pi2', pipeline.Pipeline([('Variation', cust_txt_col('Variation')), ('count_Variation', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))), ('tsvd2', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12))])),
  #            #commented for Kaggle Limits
  #            ('pi3', pipeline.Pipeline([('Text', cust_txt_col('Text')), ('tfidf_Text', feature_extraction.text.TfidfVectorizer(ngram_range=(1, 2))), ('tsvd3', decomposition.TruncatedSVD(n_components=50, n_iter=25, random_state=12))]))
  #        ])
  #    )])
  #
  #train = fp.fit_transform(train);
  #print type(train)
  #print(train.shape)
  #print (train.nbytes)
  #np.save("train_array", train)
  ## print(df.dtypes)
  ## print(df.memory_usage())
  #test = fp.transform(test); print(test.shape)
  #np.save("test_array", test)
  #exit(0)
  train = np.load("./train_array.npy")
  test = np.load("./test_array.npy")
  # siamese_features_array = np.load("./siamese_features_array_2017_09_15_07_57_44.npy")
  y = y - 1 #fix for zero bound array
  
  CONTINUOUS_INDICES = []
  SPARSE_INDICES = []
  
  for i in range((train.shape)[1]):
      if (i >= 3205 and i <= 3212):
          pass
      elif (i >= 2 and i <= 113): # or (i >= 114 and i <= 3204):
          SPARSE_INDICES.append(i)
      else:
          CONTINUOUS_INDICES.append(i)
  #train = train[:, CONTINUOUS_INDICES]
  #test = test[:, CONTINUOUS_INDICES]
  
  print('train shape after loading and selecting trainging columns: %s' % str(train.shape))
  
  siamese_train_len = len(train) // 3
  print('siamese_train_len is %d' % (siamese_train_len))
  siamese_train_data = train[:siamese_train_len]
  siamese_train_label = y[:siamese_train_len]
  
  lgbm_train_data = train[siamese_train_len:]
  lgbm_train_label = y[siamese_train_len:]
  #train = train[:200]
  #y = y[:200]
  #test = test[:200]
  #pid = pid[:200]
  
  def xgbTrain(train_data, train_label, flod = 5):
      """
      """
      denom = 0
      fold = 5 #Change to 5, 1 for Kaggle Limits
      models = []
      for i in range(fold):
          params = {
              'eta': 0.03333,
              'max_depth': 4,
              'objective': 'multi:softprob',
              'eval_metric': 'mlogloss',
              'num_class': 9,
              'seed': i,
              'silent': True
          }
          x1, x2, y1, y2 = model_selection.train_test_split(train_data, train_label, test_size=0.18, random_state=i)
          watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]
          model = xgb.train(params, xgb.DMatrix(x1, y1), 1000,  watchlist, verbose_eval=50, early_stopping_rounds=100)
          score1 = metrics.log_loss(y2, model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit), labels = list(range(9)))
          #print(score1)
  
          models.append((model, 'x'))
  
      return models
  
  
  def lgbm_train(train_data, train_label, fold = 5):
      """
      LGB Training
      """
     # print train.shape
     # print siamese_features_array.shape
     # train_merge = siamese_features_array #np.concatenate((train, siamese_features_array), axis = 1)
     # print train_merge.shape
     # # exit(0)
      print("Over all training size:")
      print(train_data.shape)
     # train_data = train_merge#[:train_len * 3 / 10]
     # train_label = lgbm_train_label#[:train_len * 3 / 10]
      #valide_data = train_merge[train_len * 9 / 10:]
      #valide_label = y[train_len * 9 / 10:]
  
      models = []
      for i in range(fold):
          d_train = lgb.Dataset(train_data, train_label) #, categorical_feature = SPARCE_INDICES)
          #d_valide = lgb.Dataset(valide_data, valide_label)
  
          params = {
              'task': 'train',
              'boosting_type': 'gbdt',
              'objective': 'multiclass',
              'metric': {'multi_logloss'},
              'num_class': 9,
            #  'num_leaves': 256,
            #  'max_depth': 12,
            #  'feature_fraction': 0.9,
            #  'bagging_fraction': 0.95,
            #  'bagging_freq': 5,
              'num_leaves': 60, # 60,
            #  'min_sum_hessian_in_leaf': 20,
              'max_depth': 10, # 10,
              'learning_rate': 0.02, # 0.02,
              'feature_fraction': 0.5,
              'verbose': 0,
            #   'valid_sets': [d_valide],
              'num_boost_round': 327,
              'feature_fraction_seed': i,
              # 'bagging_fraction': 0.9,
              # 'bagging_freq': 15,
              # 'bagging_seed': i,
              # 'early_stopping_round': 10
              # 'random_state': 10
              # 'verbose_eval': 20
              #'min_data_in_leaf': 665
          }
  
          # ROUNDS = 1
          print('fold: %d th light GBM train :-)' % (i))
          # params['feature_fraction_seed'] = i
          #bst = lgb.train(
          #                params ,
          #                d_train,
          #                verbose_eval = False
          #                # valid_sets = [d_valide]
          #                #num_boost_round = 1
          #                )
          cv_result = lgb.cv(params, d_train, nfold=10)
          pd.DataFrame(cv_result).to_csv('cv_result', index = False)
          exit(0)
          # pred = model_eval(bst, 'l', test)
          #print pred.shape
          #print pred[0, :]
          models.append((bst, 'l'))
  
      return models
  
  
  def create_model(input_len):
      model = Sequential()
      model.add(Dense(HIDDEN_UNITS[0], activation='sigmoid', input_dim = input_len))
      if DNN_BN:
          model.add(BatchNormalization())
      if DROPOUT_RATE > 0:
          model.add(Dropout(DROPOUT_RATE))
      model.add(Dense(HIDDEN_UNITS[1], activation='sigmoid'))
      if DNN_BN:
          model.add(BatchNormalization())
      if DROPOUT_RATE > 0:
          model.add(Dropout(DROPOUT_RATE))
      # model.add(Dropout(0.1))
      #model.add(Dense(32, activation='relu'))
      #model.add(Dropout(0.2))
      model.add(Dense(9, activation='softmax'))
  
      # optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
      optimizer = RMSprop(lr=1e-3, rho = 0.9, epsilon = 1e-8)
      model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics = ['accuracy'])
  
      return model
  
  
  def create_embedding_model(CONTINUE_SIZE, SPARSE_SIZE):
      """
      """
      print('CONTINUOUS_SIZE = %d' % CONTINUE_SIZE)
      print('SPARSE_SIZE = %d' % SPARSE_SIZE)
      sparse_feature = Input(shape=(SPARSE_SIZE,))
      sparse_embedding = Embedding(55, EMBEDDING_SIZE, input_length = SPARSE_SIZE)(sparse_feature)
      sparse_embedding = Reshape((EMBEDDING_SIZE * SPARSE_SIZE,))(sparse_embedding)
  
      # print "model input size: %d" % CONTINUOUS_COLUMNS
      dense_input = Input(shape=(CONTINUE_SIZE,))
      merge_input = concatenate([dense_input, sparse_embedding], axis = 1)
  
      merge_len = CONTINUE_SIZE + EMBEDDING_SIZE * SPARSE_SIZE
      output = create_model(merge_len)(merge_input)
  
      model = Model([dense_input, sparse_feature], output)
      optimizer = RMSprop(lr=1e-3, rho = 0.9, epsilon = 1e-8)
      # optimizer = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
      model.compile(optimizer = Adam(),
              loss='categorical_crossentropy', metrics = ['accuracy'])
  
      return model
  
  
  def keras_train(train_data, train_target, nfolds = 10):
      """
      Detect Fish or noFish
      """
      print("Start gen training data, shuffle and normalize!")
  
      #train_data = train
      train_target = np_utils.to_categorical(train_target)
  
      # train_data, train_target, siamese_data_loader = siamese_train(siamese_train_data, siamese_train_label)
      kf = KFold(len(train_target), n_folds=nfolds, shuffle=True)
  
      num_fold = 0
      models = []
      for train_index, test_index in kf:
          # model = create_model(classes = 2)
          model = create_embedding_model(len(CONTINUOUS_INDICES), len(SPARSE_INDICES))
          # model = create_siamese_net((train.shape)[1])
  
          X_train = train_data[train_index]
          Y_train = train_target[train_index]
          print('Positive samples in train: %d' % np.sum(Y_train))
          print('Negative samples in train: %d' % (len(Y_train) - np.sum(Y_train)))
  
          X_valid = train_data[test_index]
          Y_valid = train_target[test_index]
          print('Positive samples in valide: %d' % np.sum(Y_valid))
          print('Negative samples in valide: %d' % (len(Y_valid) - np.sum(Y_valid)))
  
          num_fold += 1
          print('Start KFold number {} from {}'.format(num_fold, nfolds))
          print('Split train: ', len(X_train), len(Y_train))
          print('Split valid: ', len(X_valid), len(Y_valid))
  
          callbacks = [
              EarlyStopping(monitor='val_loss', patience=5, verbose=0),
          ]
          model.fit([X_train[:, CONTINUOUS_INDICES], X_train[:, SPARSE_INDICES]],
                  Y_train, batch_size=BATCH_SIZE, epochs=DNN_EPOCHS,
                  shuffle=True, verbose=2,
                  validation_data=([X_valid[:, CONTINUOUS_INDICES], X_valid[:, SPARSE_INDICES]], Y_valid)
                  , callbacks=callbacks)
  
          model_name = 'keras' + strftime('_%Y_%m_%d_%H_%M_%S', gmtime())
          #model.save_weights(model_name)
          #siamese_features_array = gen_siamese_features(model, lgbm_train_data, siamese_train_data, siamese_train_label)
          models.append((model, 'k'))
          break
  
      return models #, siamese_features_array
  
  
  def model_eval(model, model_type, data_frame):
      """
      """
      if model_type == 'l':
          preds = model.predict(data_frame)
      elif model_type == 'k':
          preds = model.predict(data_frame, batch_size=BATCH_SIZE, verbose=2)
      elif model_type == 't':
          print("ToDO")
      elif model_type == 'x':
          preds = model.predict(xgb.DMatrix(data_frame), ntree_limit=model.best_ntree_limit+80)
  
      return preds
  
  
  def gen_sub(models, merge_features):
      """
      Evaluate single Type model
      """
      print('Start generate submission!')
      preds = None
      for (model, model_type) in models:
          pred = model_eval(model, model_type, merge_features)
          #print pred.shape
          #print pred[0, :]
          if preds is None:
              preds = pred.copy()
          else:
              preds += pred
  
      preds /= len(models)
      submission = pd.DataFrame(preds, columns=['class'+str(c+1) for c in range(9)])
      submission['ID'] = pid
      sub_name = "submission" + strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) + ".csv"
      print('Output to ' + sub_name)
      submission.to_csv(sub_name, index=False)
  
  
  def create_siamese_net(input_size):
      """
      """
      input_shape = (input_size, )
      left_input = Input(input_shape)
      right_input = Input(input_shape)
      #build model to use in each siamese 'leg'
      model = Sequential()
      model.add(Dense(HIDDEN_UNITS[0], activation='sigmoid', input_dim = input_size))
      if DNN_BN:
          model.add(BatchNormalization())
      if DROPOUT_RATE > 0:
          model.add(Dropout(DROPOUT_RATE))
      model.add(Dense(HIDDEN_UNITS[1], activation='sigmoid'))
      if DNN_BN:
          model.add(BatchNormalization())
      if DROPOUT_RATE > 0:
          model.add(Dropout(DROPOUT_RATE))
      #encode each of the two inputs into a vector with the convnet
      encoded_l = model(left_input)
      encoded_r = model(right_input)
      #merge two encoded inputs with the l1 distance between them
      L1_distance = lambda x: K.abs(x[0]-x[1])
      both = merge([encoded_l,encoded_r], mode = L1_distance, output_shape=lambda x: x[0])
      merge_layer = Dense(HIDDEN_UNITS[2],activation='sigmoid')(both)
      prediction = Dense(1,activation='sigmoid')(merge_layer)
      siamese_net = Model(input=[left_input,right_input],output=prediction)
      #optimizer = SGD(0.0004,momentum=0.6,nesterov=True,decay=0.0003)
  
      optimizer = Adam()
      #//TODO: get layerwise learning rates and momentum annealing scheme described in paperworking
      siamese_net.compile(loss="binary_crossentropy",optimizer=optimizer)
  
      # print siamese_net.count_params()
      return siamese_net
  
  
  class Siamese_Loader:
      #For loading batches and testing tasks to a siamese net
      def __init__(self,Xtrain,Xval = None):
          self.Xval = Xval
          self.Xtrain = Xtrain
          self.n_classes = Xtrain.shape[0]
          self.feature_size = (Xtrain[0].shape)[1]
          self.n_examples = np.array([x.shape[0] for x in Xtrain])
          self.n_tot_examples = np.sum(self.n_examples)
          print('examples of different classes: %s' % str(self.n_examples))
          # self.n_val,self.n_ex_val,_,_ = Xval.shape
  
      def get_batch(self,n):
          #Create batch of pairs, half same class, half different class
          categories = rng.choice(self.n_classes,size=(n,),replace=True)
          pairs=np.zeros((2, n, self.feature_size))
          targets=np.zeros((n,))
          positive_begin_pos = n * 1 // 2
          targets[positive_begin_pos:] = 1
          categories_list = []
          for i in range(n):
              category = categories[i]
              idx_1 = rng.randint(0, self.n_examples[category])
              pairs[0][i] = self.Xtrain[category][idx_1] #.reshape(self.feature_size)
              #pick images of same class for 1st half, different for 2nd
              category_2 = category if i >= positive_begin_pos else (category + rng.randint(1,self.n_classes)) % self.n_classes
              idx_2 = rng.randint(0,self.n_examples[category_2])
              while i >= positive_begin_pos and idx_2 == idx_1:
                  idx_2 = rng.randint(0,self.n_examples[category_2])
              pairs[1][i] = self.Xtrain[category_2][idx_2] #.reshape(self.w,self.h,1)
              categories_list.append((category, category_2))
          #pd.DataFrame(categories_list).to_csv('categories', index=False)
          #exit(0)
          # shuflle pairs to mix positive and negative
          rng.shuffle(pairs)
          return pairs, targets
  
  
  def gen_test_on_support_data(Xsupport, Xtest):
      """
      """
      n_support, feature_size = Xsupport.shape
      pairs = np.zeros((2, n_support, feature_size))
      pairs[0] = Xtest
      pairs[1] = Xsupport
      return list(pairs)
  
  
  def siamese_train(siamese_train_data, siamese_train_label):
      """
      """
      train_data = [[] for i in range(9)]
      label_ind = 0
      for feature in siamese_train_data:
          train_data[siamese_train_label[label_ind]].append(feature)
          label_ind += 1
      train_data = np.array([np.array(xi) for xi in train_data])
      print("train data shape before gen pair")
      print(train_data.shape)
      siamese_data_loader = Siamese_Loader(train_data, test)
      pairs, targets = siamese_data_loader.get_batch(SIAMESE_PAIR_SIZE)
      return pairs, targets, siamese_data_loader
  
  
  def gen_siamese_features_meta(model, Xsupport_label, Xsupport, Xtest):
      """
      """
      siamese_pair = gen_test_on_support_data(Xsupport, Xtest)
      global graph
      with graph.as_default():
          preds = model.predict(siamese_pair, batch_size=BATCH_SIZE, verbose=2)
      preds = np.insert(preds, 1, Xsupport_label, axis = 1)
      preds = pd.DataFrame(preds, columns = ['sim', 'class'])
      siamese_features = preds.groupby('class', sort = False) \
              .agg({'sim': ['max', 'min', 'median', 'mean', 'std']})
      max_class = siamese_features['sim']['max'].idxmax()
      siamese_features = np.insert(siamese_features.values.flatten(), 0, max_class, axis = 0)
      return siamese_features
  
  def gen_siamese_features(siamese_model, Xtest, Xsupport, Xsupport_label):
      """
      """
      if MAX_WORKERS <= 0:
          print("MAX_WORKERS should >= 1", file=sys.stderr)
          exit(1)
      siamese_features_array = list(range(len(Xtest)))
      test_begin = 0
      while test_begin < len(Xtest):
          test_end = min(test_begin + MAX_WORKERS, len(Xtest))
          with concurrent.futures.ThreadPoolExecutor(max_workers = MAX_WORKERS) as executor:
              future_predict = {executor.submit(gen_siamese_features_meta, siamese_model,
                          Xsupport_label,
                          Xsupport,
                          Xtest[ind]): ind for ind in range(test_begin, test_end)}
              for future in concurrent.futures.as_completed(future_predict):
                  ind = future_predict[future]
                  try:
                      siamese_features = future.result()
                      siamese_features_array[ind] = siamese_features
                  except Exception as exc:
                      print('%dth feature generated an exception: %s' % (ind, exc))
          test_begin = test_end
          if test_begin % 100 == 0:
              print('Gen %d siamsese features' % test_begin)
      if test_begin != len(Xtest):
          print("Only gen %d siamese features" % test_begin, file=sys.stderr)
          exit(1)
      siamese_features_array = np.array(siamese_features_array)
      pd.DataFrame(siamese_features_array[:, 0]).astype(np.int8).to_csv('pred_label', index = False)
      return siamese_features_array
  
  
  if __name__ == "__main__":
      model_k = keras_train(train, y, 10)
      #np.save("siamese_features_array" + \
      #        strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) , siamese_features_array)
     #  gen_sub(model_k, 'k', th, F1)
      # ind = np.array([i * 5 for i in range(9)])
      # xgbTrain(siamese_features_array[:, ind], lgbm_train_label);
      #lgbm_features = siamese_features_array #np.concatenate((lgbm_train_data, siamese_features_array),
      # model_l = lgbm_train(train, y, 10) #lgbm_features, lgbm_train_label, 10)#model_k)
      # siamese_features_test_array = siamese_test(model_k[0][0], test)
      #np.save("siamese_features_test_array" + \
      #        strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) , siamese_features_test_array)
      ##model_x = xgbTrain(5)#model_k)
      #gen_sub(model_l, siamese_features_test_array) #model_k)
  