def transformer_tall_pretrain_lm_tpu_adafactor ( ) : hparams = transformer_tall_pretrain_lm ( ) update_hparams_for_tpu ( hparams ) hparams . max_length = 1024 # For multi-problem on TPU we need it in absolute examples. hparams . batch_size = 8 hparams . multiproblem_vocab_size = 2 ** 16 return hparams
mnist multi gpu training python tensorflow
Hparams for transformer on LM pretraining ( with 64k vocab ) on TPU .
def transformer_tall_pretrain_lm_tpu_adafactor():
  """Hparams for transformer on LM pretraining (with 64k vocab) on TPU."""
  hparams = transformer_tall_pretrain_lm()
  update_hparams_for_tpu(hparams)
  hparams.max_length = 1024
  # For multi-problem on TPU we need it in absolute examples.
  hparams.batch_size = 8
  hparams.multiproblem_vocab_size = 2**16
  return hparams