Fix bug
Browse files- csc_tokenizer.py +4 -0
csc_tokenizer.py
CHANGED
@@ -114,6 +114,10 @@ class ReaLiSeTokenizer(BertTokenizerFast):
|
|
114 |
pho_idx = pho_idx_list
|
115 |
pho_lens = pho_lens_list
|
116 |
if return_tensors == 'pt':
|
|
|
|
|
|
|
|
|
117 |
pho_idx = torch.vstack(pho_idx)
|
118 |
pho_lens = torch.LongTensor(pho_lens)
|
119 |
|
|
|
114 |
pho_idx = pho_idx_list
|
115 |
pho_lens = pho_lens_list
|
116 |
if return_tensors == 'pt':
|
117 |
+
max_pho_len = max(pho_lens_list)
|
118 |
+
for i in range(len(pho_idx)):
|
119 |
+
pho_idx[i] = torch.nn.functional.pad(pho_idx[i], (0, max_pho_len - pho_idx[i].shape[1]))
|
120 |
+
|
121 |
pho_idx = torch.vstack(pho_idx)
|
122 |
pho_lens = torch.LongTensor(pho_lens)
|
123 |
|