|
from transformers import PretrainedConfig |
|
from typing import List |
|
|
|
|
|
|
|
|
|
class ModularStarEncoderConfig(PretrainedConfig): |
|
model_type = "ModularStarEncoder" |
|
keys_to_ignore_at_inference = ["past_key_values"] |
|
|
|
def __init__( |
|
self, |
|
attention_dropout= 0.1, |
|
residual_dropout= 0.1, |
|
embedding_dropout= 0.1, |
|
bos_token_id= 0, |
|
eos_token_id= 0, |
|
hidden_act= "gelu_pytorch_tanh", |
|
_attn_implementation="flash_attention_2", |
|
hidden_size= 1024, |
|
conditional_size= 4, |
|
initializer_range= 0.018042, |
|
intermediate_size= 12288, |
|
max_position_embeddings= 2048, |
|
mlp_type= "default", |
|
model_type= "starcoder2", |
|
torch_dtype= "bfloat16", |
|
layer_matryoshka_loss= True, |
|
matryoshka_layers= [4,9,18,27,36], |
|
norm_epsilon= 1e-05, |
|
layer_norm_eps=1e-05, |
|
norm_type= "layer_norm", |
|
num_attention_heads= 16, |
|
num_hidden_layers= 36, |
|
num_key_value_heads= 4, |
|
rope_theta= 999999.4420358813, |
|
sliding_window= None, |
|
transformers_version= "4.39.3", |
|
use_bias= True, |
|
use_cache= False, |
|
vocab_size= 49156, |
|
pad_token_id=0, |
|
**kwargs, |
|
): |
|
if _attn_implementation not in ["flash_attention_2", "sdpa"]: |
|
raise ValueError(f"`_attn_implementation` must be 'flash_attention_2', 'sdpa', got {_attn_implementation}.") |
|
|
|
self.attention_dropout=attention_dropout , |
|
self.residual_dropout= residual_dropout, |
|
self.embedding_dropout= embedding_dropout, |
|
self.bos_token_id= bos_token_id, |
|
self.eos_token_id= eos_token_id, |
|
self.hidden_act= hidden_act, |
|
self._attn_implementation=_attn_implementation, |
|
self.hidden_size= hidden_size, |
|
self.conditional_size= conditional_size, |
|
self.initializer_range= initializer_range, |
|
self.intermediate_size= intermediate_size, |
|
self.max_position_embeddings= max_position_embeddings, |
|
self.mlp_type= mlp_type, |
|
self.model_type= model_type, |
|
self.torch_dtype= torch_dtype, |
|
self.layer_matryoshka_loss= layer_matryoshka_loss, |
|
self.matryoshka_layers= matryoshka_layers, |
|
self.norm_epsilon= norm_epsilon, |
|
self.layer_norm_eps=layer_norm_eps, |
|
self.norm_type= norm_type, |
|
self.num_attention_heads= num_attention_heads, |
|
self.num_hidden_layers= num_hidden_layers, |
|
self.num_key_value_heads= num_key_value_heads, |
|
self.rope_theta= rope_theta, |
|
self.sliding_window= sliding_window, |
|
self.transformers_version= transformers_version, |
|
self.use_bias= use_bias, |
|
self.use_cache= use_cache, |
|
self.vocab_size= vocab_size, |
|
self.pad_token_id=pad_token_id, |
|
super().__init__( |
|
bos_token_id=bos_token_id, |
|
eos_token_id=eos_token_id, |
|
**kwargs) |
|
|
