hardcode hack

huggingface · Oct 20, 2022 · 6ede608 · 6ede608
1 parent 2e2be49
commit 6ede608
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/transformers/models/switch_transformers/modeling_switch_transformers.py b/src/transformers/models/switch_transformers/modeling_switch_transformers.py
@@ -221,7 +221,7 @@ def __init__(self, config: SwitchTransformersConfig, has_relative_attention_bias
         self.o = nn.Linear(self.inner_dim, self.d_model, bias=False)
 
         if self.has_relative_attention_bias:
-            self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads)
+            self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, 32 ) #self.n_heads)
         self.pruned_heads = set()
         self.gradient_checkpointing = False