@@ -2398,6 +2398,55 @@ def test_resize_embeddings_untied_with_deepspeed_multi_gpu(self):
23982398 with _deepspeed_zero3 (ds_config ):
23992399 self .test_resize_embeddings_untied ()
24002400
2401+ def test_resize_embeddings_untied_no_reinit_on_post_init (self ):
2402+ if not self .test_resize_embeddings :
2403+ self .skipTest (reason = "test_resize_embeddings is set to `False`" )
2404+
2405+ original_config , inputs_dict = self .model_tester .prepare_config_and_inputs_for_common ()
2406+ original_config .tie_word_embeddings = False
2407+ try :
2408+ original_config .get_text_config ().tie_word_embeddings = False
2409+ except Exception as e :
2410+ model_type = getattr (original_config , "model_type" , "unknown" )
2411+ print (f"Could not set text config's `tie_word_embeddings` for model type `{ model_type } `: { e } " )
2412+
2413+ if original_config .tie_word_embeddings :
2414+ self .skipTest (reason = "Model cannot untie embeddings" )
2415+
2416+ for model_class in self .all_model_classes :
2417+ with self .subTest (model_class ):
2418+ config = copy .deepcopy (original_config )
2419+ model = model_class (config ).to (torch_device )
2420+ model .eval ()
2421+
2422+ # The bug only affects nn.Linear LM heads created by _get_resized_lm_head
2423+ output_embeds = model .get_output_embeddings ()
2424+ if not isinstance (output_embeds , nn .Linear ):
2425+ continue
2426+
2427+ model_vocab_size = config .get_text_config ().vocab_size
2428+ try :
2429+ model .resize_token_embeddings (model_vocab_size + 10 )
2430+ except (NotImplementedError , AttributeError ):
2431+ continue
2432+
2433+ output_embeds = model .get_output_embeddings ()
2434+ weights_before = output_embeds .weight .data .clone ()
2435+ bias_before = output_embeds .bias .data .clone () if output_embeds .bias is not None else None
2436+
2437+ model .post_init ()
2438+
2439+ output_embeds_after = model .get_output_embeddings ()
2440+ self .assertTrue (
2441+ torch .equal (weights_before , output_embeds_after .weight .data ),
2442+ "Output embedding weights were reinitialized by post_init() after resize_token_embeddings()" ,
2443+ )
2444+ if bias_before is not None :
2445+ self .assertTrue (
2446+ torch .equal (bias_before , output_embeds_after .bias .data ),
2447+ "Output embedding bias was reinitialized by post_init() after resize_token_embeddings()" ,
2448+ )
2449+
24012450 def test_model_get_set_embeddings (self ):
24022451 config , inputs_dict = self .model_tester .prepare_config_and_inputs_for_common ()
24032452
0 commit comments