From 5ff8352634aa4c1dcf8124b3294ac90b10c5ce8d Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Mon, 27 Jun 2022 10:00:21 -0400 Subject: [PATCH] remove obsolete code --- scripts/lang_adapt/madx_run_clm.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/scripts/lang_adapt/madx_run_clm.py b/scripts/lang_adapt/madx_run_clm.py index 2f5cf5c..06bfcfe 100644 --- a/scripts/lang_adapt/madx_run_clm.py +++ b/scripts/lang_adapt/madx_run_clm.py @@ -433,11 +433,6 @@ def group_texts(examples): return lm_datasets def modify_model(adapter_args, data_args, model_args, tokenizer, model): - #if "emb" in model_args.lang_adapt_strategies: - # if "replace" in model_args.embedding_strategies: - # for name, param in model.named_parameters(): - # if "wte" not in name and "wpe" not in name and "lm_head" not in name: - # param.requires_grad = False def get_adapter_config(adapter_args, model_args): if adapter_args.adapter_config == "prefix_tuning": @@ -566,16 +561,6 @@ def zero_grad(grad): embedding_layer.weight.register_hook(lambda grad: zero_grad(grad)) - #if model_args.embedding_strategies == "overlap-replace": - # if not tokenizer.name_or_path == model_args.model_name_or_path: - # orig_tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path) - # model.add_embeddings('lng_emb', tokenizer, reference_embedding='default', reference_tokenizer=orig_tokenizer ) - # model._active_embedding = "lng_emb" - # model.delete_embeddings('default') - # model.tie_weights() - #elif model_args.embedding_strategies == "replace": - # model.resize_token_embeddings(len(tokenizer)) - trainable_params = 0 frozen_params = 0 emb_params = 0 @@ -688,9 +673,7 @@ def main(): print("Model: 👇") print(model) - - # print("Embeddings at start of run:", model.get_input_embeddings().weight[250880:,:]) # get original weight for embedding layer - # orig_embeddings = model.get_input_embeddings().weight.detach().clone() # clone original weight for embedding layer + # Training if training_args.do_train: checkpoint = None @@ -725,17 +708,6 @@ def main(): trainer.log_metrics("train", metrics) trainer.save_metrics("train", metrics) trainer.save_state() - - # uncomment to test whether extending vocab gradient masking is working correctly. - # if model_args.embedding_strategies == "extend": - # print("Unsliced, post-training:", model.get_input_embeddings().weight) # get updated weight - # if not torch.equal(orig_embeddings[:250880, :], model.get_input_embeddings().weight[:250880, :]): - # raise ValueError("embedding layer is updated where it shouldn't....") - - # if torch.equal(orig_embeddings[250880:, :], model.get_input_embeddings().weight[250880:, :]): - # print("original embeddings:", orig_embeddings[250880:, :]) - # print("updated embeddings:", model.get_input_embeddings().weight[250880:, :]) - # raise ValueError("embedding layer is not updated where it should....") # Evaluation