iioSnail commited on
Commit
7eea323
·
verified ·
1 Parent(s): 119d519
Files changed (1) hide show
  1. csc_tokenizer.py +4 -0
csc_tokenizer.py CHANGED
@@ -114,6 +114,10 @@ class ReaLiSeTokenizer(BertTokenizerFast):
114
  pho_idx = pho_idx_list
115
  pho_lens = pho_lens_list
116
  if return_tensors == 'pt':
 
 
 
 
117
  pho_idx = torch.vstack(pho_idx)
118
  pho_lens = torch.LongTensor(pho_lens)
119
 
 
114
  pho_idx = pho_idx_list
115
  pho_lens = pho_lens_list
116
  if return_tensors == 'pt':
117
+ max_pho_len = max(pho_lens_list)
118
+ for i in range(len(pho_idx)):
119
+ pho_idx[i] = torch.nn.functional.pad(pho_idx[i], (0, max_pho_len - pho_idx[i].shape[1]))
120
+
121
  pho_idx = torch.vstack(pho_idx)
122
  pho_lens = torch.LongTensor(pho_lens)
123