Reference values are updated

Signed-off-by: Ann Kuruvilla <[email protected]>

Author: quic-akuruvil

tests/finetune/reference_data.py

@@ -13,206 +13,206 @@
     "llama_3.2_1B_config_alpaca_single_device": {
         "description": "Baseline for Llama on Alpaca single-device",
         "train_step_losses": [
-            1.5112206935882568,
-            1.2211230993270874,
-            1.9942185878753662,
-            2.093623161315918,
-            0.9168124198913574,
-            1.2125635147094727,
-            0.3648962676525116,
-            1.6231939792633057,
-            0.8259601593017578,
-            0.7741442918777466,
-            1.7359141111373901,
-            2.118462085723877,
-            2.061161994934082,
-            0.8256913423538208,
-            0.8088029623031616,
-            1.761340618133545,
-            1.6828027963638306,
-            1.3538823127746582,
-            2.0672550201416016,
-            3.1532647609710693,
+            1.5110896825790405,
+            1.2206485271453857,
+            1.9950776100158691,
+            2.091615676879883,
+            0.9182446599006653,
+            1.1993569135665894,
+            0.36413607001304626,
+            1.6241482496261597,
+            0.8270177245140076,
+            0.7749958634376526,
+            1.73696768283844,
+            2.120077610015869,
+            2.061460256576538,
+            0.8267984390258789,
+            0.8105809688568115,
+            1.7627557516098022,
+            1.6819559335708618,
+            1.3528242111206055,
+            2.0654125213623047,
+            3.156151294708252,
         ],
         "eval_step_losses": [
-            1.462059736251831,
-            0.24527676403522491,
-            1.046107292175293,
-            1.6403586864471436,
-            1.395291805267334,
-            2.8664817810058594,
-            1.035412311553955,
-            1.8670039176940918,
-            3.8079662322998047,
-            0.6516809463500977,
+            1.4607517719268799,
+            0.24302150309085846,
+            1.0471211671829224,
+            1.642044186592102,
+            1.3949533700942993,
+            2.8850066661834717,
+            1.0366586446762085,
+            1.8661959171295166,
+            3.81632924079895,
+            0.6577113270759583,
         ],
         "train_step_metrics": [
-            4.532259941101074,
-            3.390994071960449,
-            7.34645938873291,
-            8.114261627197266,
-            2.5013046264648438,
-            3.3620924949645996,
-            1.4403645992279053,
-            5.069255828857422,
-            2.2840728759765625,
-            2.1687355041503906,
-            5.674112319946289,
-            8.318334579467773,
-            7.855090141296387,
-            2.283458948135376,
-            2.2452187538146973,
-            5.820234775543213,
-            5.380615711212158,
-            3.872429847717285,
-            7.903097629547119,
-            23.412376403808594,
+            4.531666278839111,
+            3.389385223388672,
+            7.352773189544678,
+            8.09798812866211,
+            2.504889488220215,
+            3.3179824352264404,
+            1.43927001953125,
+            5.074095249176025,
+            2.286489486694336,
+            2.1705832481384277,
+            5.680093288421631,
+            8.33178424835205,
+            7.857433319091797,
+            2.2859883308410645,
+            2.2492144107818604,
+            5.828476905822754,
+            5.376060962677002,
+            3.8683345317840576,
+            7.8885498046875,
+            23.480052947998047,
         ],
         "eval_step_metrics": [  # steps 0-9
-            4.31483793258667,
-            1.2779749631881714,
-            2.8465487957000732,
-            5.157018661499023,
-            4.036152362823486,
-            17.575077056884766,
-            2.816267251968384,
-            6.468885898590088,
-            45.05870819091797,
-            1.9187631607055664,
+            4.309197902679443,
+            1.27509605884552,
+            2.8494362831115723,
+            5.1657185554504395,
+            4.034786224365234,
+            17.9036865234375,
+            2.819779396057129,
+            6.463661193847656,
+            45.437110900878906,
+            1.9303690195083618,
         ],
     },
     # Scenario 2: Single-device llama 3.2-1B training on GSM8k dataset.
     "llama_3.2_1B_config_gsm8k_single_device": {
         "description": "Baseline for Llama on GSM8k single-device",
         "train_step_losses": [
-            2.250276803970337,
-            2.3231687545776367,
-            1.9379945993423462,
-            1.5981022119522095,
-            1.9867562055587769,
-            1.4573354721069336,
-            1.8969658613204956,
-            1.2177824974060059,
-            1.6489791870117188,
-            1.5380687713623047,
-            1.4025083780288696,
-            1.5301083326339722,
-            1.6858205795288086,
-            1.383747935295105,
-            1.7968919277191162,
-            1.4075607061386108,
-            1.6447738409042358,
-            1.2807793617248535,
-            0.8450672030448914,
-            1.5795941352844238,
+            2.250361204147339,
+            2.3252110481262207,
+            1.9360781908035278,
+            1.5984115600585938,
+            1.9874038696289062,
+            1.4579044580459595,
+            1.8975679874420166,
+            1.2175723314285278,
+            1.6473736763000488,
+            1.537960410118103,
+            1.4019465446472168,
+            1.5310447216033936,
+            1.6878201961517334,
+            1.3849903345108032,
+            1.7976438999176025,
+            1.4060133695602417,
+            1.646375060081482,
+            1.2835280895233154,
+            0.8465587496757507,
+            1.5783095359802246,
         ],
         "eval_step_losses": [
-            1.7081595659255981,
-            1.719305157661438,
-            1.153528094291687,
-            2.0051634311676025,
-            1.3372926712036133,
-            1.3009852170944214,
-            1.2207027673721313,
-            1.3452664613723755,
-            1.329830288887024,
-            1.307450532913208,
+            1.707140326499939,
+            1.7226355075836182,
+            1.1531383991241455,
+            2.0035903453826904,
+            1.3362350463867188,
+            1.3013248443603516,
+            1.2195535898208618,
+            1.3454742431640625,
+            1.3299248218536377,
+            1.3073854446411133,
         ],
         "train_step_metrics": [
-            9.490362167358398,
-            10.207969665527344,
-            6.944809913635254,
-            4.943641662597656,
-            7.291841506958008,
-            4.294501304626465,
-            6.6656389236450195,
-            3.3796849250793457,
-            5.201667308807373,
-            4.655590534210205,
-            4.065384864807129,
-            4.618677139282227,
-            5.396877765655518,
-            3.989826202392578,
-            6.030873775482178,
-            4.0859761238098145,
-            5.179838180541992,
-            3.5994436740875244,
-            2.328134298324585,
-            4.852985858917236,
+            9.49116325378418,
+            10.228837966918945,
+            6.93151330947876,
+            4.945170879364014,
+            7.296566009521484,
+            4.296945571899414,
+            6.66965389251709,
+            3.378974676132202,
+            5.193322658538818,
+            4.655086040496826,
+            4.063101291656494,
+            4.623003959655762,
+            5.407680034637451,
+            3.994786262512207,
+            6.0354108810424805,
+            4.0796589851379395,
+            5.188138961791992,
+            3.60935115814209,
+            2.3316092491149902,
+            4.846755504608154,
         ],
         "eval_step_metrics": [  # steps 0-9
-            5.518795013427734,
-            5.580649375915527,
-            3.1693549156188965,
-            7.42730712890625,
-            3.8087174892425537,
-            3.672913074493408,
-            3.38956880569458,
-            3.8392088413238525,
-            3.7804012298583984,
-            3.6967368125915527,
+            5.5131731033325195,
+            5.599266052246094,
+            3.1681201457977295,
+            7.415632247924805,
+            3.8046915531158447,
+            3.674160957336426,
+            3.3856759071350098,
+            3.8400065898895264,
+            3.7807586193084717,
+            3.69649600982666,
         ],
     },
     # Scenario 3: Single-device google-bert/bert-base-uncased training on IMDB dataset.
     "bert_base_uncased_config_imdb_single_device": {
         "description": "Baseline for google-bert/bert-base-uncased on IMDB single-device",
         "train_step_losses": [
-            0.357421875,
-            0.546875,
-            0.98486328125,
-            0.35302734375,
-            1.23828125,
-            0.60791015625,
-            0.44384765625,
-            0.791015625,
-            0.7861328125,
-            0.51318359375,
-            0.50244140625,
-            0.90087890625,
-            0.8818359375,
-            0.86279296875,
-            0.6396484375,
-            0.49267578125,
-            0.97119140625,
-            0.7451171875,
-            0.798828125,
-            0.7080078125,
+            0.390625,
+            0.51220703125,
+            0.9208984375,
+            0.4052734375,
+            1.1640625,
+            0.6533203125,
+            0.5087890625,
+            0.76171875,
+            0.63525390625,
+            0.50146484375,
+            0.5439453125,
+            0.947265625,
+            0.89013671875,
+            0.80419921875,
+            0.6533203125,
+            0.4580078125,
+            0.92041015625,
+            0.7412109375,
+            0.7197265625,
+            0.62158203125,
         ],
         "eval_step_losses": [
-            0.634765625,
-            0.8173828125,
+            0.6044921875,
+            0.798828125,
             0.9072265625,
-            0.7177734375,
-            0.59423828125,
-            0.69921875,
-            0.7109375,
-            0.7216796875,
-            0.6064453125,
-            0.7041015625,
+            0.70361328125,
+            0.59912109375,
+            0.66357421875,
+            0.6962890625,
+            0.75390625,
+            0.61328125,
+            0.6806640625,
         ],
         "train_step_metrics": [
             1.0,
             1.0,
             0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.5,
-            0.449951171875,
-            0.4091796875,
+            0.49999988079071045,
+            0.49999988079071045,
+            0.5,
+            0.5000002384185791,
+            0.5000002384185791,
+            0.6250002384185791,
+            0.6249998807907104,
+            0.625,
+            0.6000000238418579,
+            0.5833332538604736,
+            0.5714285373687744,
+            0.5714285373687744,
+            0.5714285373687744,
+            0.5625,
+            0.555555522441864,
+            0.5055557489395142,
+            0.5101010203361511,
         ],
-        "eval_step_metrics": [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0],
+        "eval_step_metrics": [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0],
     },
     # Scenario 4: Distributed google-bert/bert-base-uncased training  (world_size=2)
     "bert_base_uncased_config_imdb_distributed_ws2": {

tests/finetune/test_finetune.py

@@ -140,7 +140,7 @@ def assert_list_close(ref_list, actual_list, atol, name, scenario_key, current_w
 ]
 
 
-@pytest.mark.skip()  # remove when it's clear why diff val_step_loss values are observed in diff runs on existing code (even without PR #478 changes)
+# @pytest.mark.skip()  # remove when it's clear why diff val_step_loss values are observed in diff runs on existing code (even without PR #478 changes)
 @pytest.mark.cli
 @pytest.mark.on_qaic
 @pytest.mark.finetune