Skip to content

Commit 4932e97

Browse files
Fix resized LM head weights being overwritten by post_init (#45079)
When `tie_word_embeddings=False`, `_get_resized_lm_head()` creates a new `nn.Linear` without `_is_hf_initialized`, causing `post_init()` to reinitialize its weights. Set the flag after weight copying is done. Fixes #35141
1 parent 57e8413 commit 4932e97

2 files changed

Lines changed: 50 additions & 0 deletions

File tree

src/transformers/modeling_utils.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2984,6 +2984,7 @@ def _get_resized_lm_head(
29842984
new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
29852985
)
29862986

2987+
new_lm_head._is_hf_initialized = True
29872988
return new_lm_head
29882989

29892990
def _init_added_embeddings_weights_with_mean(

tests/test_modeling_common.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2398,6 +2398,55 @@ def test_resize_embeddings_untied_with_deepspeed_multi_gpu(self):
23982398
with _deepspeed_zero3(ds_config):
23992399
self.test_resize_embeddings_untied()
24002400

2401+
def test_resize_embeddings_untied_no_reinit_on_post_init(self):
2402+
if not self.test_resize_embeddings:
2403+
self.skipTest(reason="test_resize_embeddings is set to `False`")
2404+
2405+
original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
2406+
original_config.tie_word_embeddings = False
2407+
try:
2408+
original_config.get_text_config().tie_word_embeddings = False
2409+
except Exception as e:
2410+
model_type = getattr(original_config, "model_type", "unknown")
2411+
print(f"Could not set text config's `tie_word_embeddings` for model type `{model_type}`: {e}")
2412+
2413+
if original_config.tie_word_embeddings:
2414+
self.skipTest(reason="Model cannot untie embeddings")
2415+
2416+
for model_class in self.all_model_classes:
2417+
with self.subTest(model_class):
2418+
config = copy.deepcopy(original_config)
2419+
model = model_class(config).to(torch_device)
2420+
model.eval()
2421+
2422+
# The bug only affects nn.Linear LM heads created by _get_resized_lm_head
2423+
output_embeds = model.get_output_embeddings()
2424+
if not isinstance(output_embeds, nn.Linear):
2425+
continue
2426+
2427+
model_vocab_size = config.get_text_config().vocab_size
2428+
try:
2429+
model.resize_token_embeddings(model_vocab_size + 10)
2430+
except (NotImplementedError, AttributeError):
2431+
continue
2432+
2433+
output_embeds = model.get_output_embeddings()
2434+
weights_before = output_embeds.weight.data.clone()
2435+
bias_before = output_embeds.bias.data.clone() if output_embeds.bias is not None else None
2436+
2437+
model.post_init()
2438+
2439+
output_embeds_after = model.get_output_embeddings()
2440+
self.assertTrue(
2441+
torch.equal(weights_before, output_embeds_after.weight.data),
2442+
"Output embedding weights were reinitialized by post_init() after resize_token_embeddings()",
2443+
)
2444+
if bias_before is not None:
2445+
self.assertTrue(
2446+
torch.equal(bias_before, output_embeds_after.bias.data),
2447+
"Output embedding bias was reinitialized by post_init() after resize_token_embeddings()",
2448+
)
2449+
24012450
def test_model_get_set_embeddings(self):
24022451
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
24032452

0 commit comments

Comments
 (0)