microsoft
/

Phi-3.5-mini-instruct

Text Generation

text-generation-inference

Model card Files Files and versions Community

sylwia-kuros commited on Mar 26

Commit

5edc21d

·

verified ·

1 Parent(s): 277e139

Update modeling_phi3.py

Files changed (1) hide show

modeling_phi3.py +1 -1

modeling_phi3.py CHANGED Viewed

@@ -1296,7 +1296,7 @@ class Phi3ForCausalLM(Phi3PreTrainedModel):
             if isinstance(past_key_values, Cache):
                 cache_length = past_key_values.get_seq_length()
                 past_length = past_key_values.seen_tokens
-                max_cache_length = past_key_values.get_max_length()
             else:
                 cache_length = past_length = past_key_values[0][0].shape[2]
                 max_cache_length = None

             if isinstance(past_key_values, Cache):
                 cache_length = past_key_values.get_seq_length()
                 past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length() if hasattr(past_key_values, "get_max_length") else past_key_values.get_max_cache_shape()
             else:
                 cache_length = past_length = past_key_values[0][0].shape[2]
                 max_cache_length = None